free-coding-models 0.5.88 → 0.5.89

package/src/core/router-v2/failure-classifier.js

@@ -0,0 +1,231 @@
+/**
+ * @file failure-classifier.js
+ * @description Typed failure classification for the Router v2 failover engine.
+ *
+ * @details
+ *   📖 Router v1 treated almost every non-2xx the same way: increment
+ *   `consecutiveFailures` and fail over. That punishes healthy models when the
+ *   CLIENT payload is the problem (400/413/422 would burn 3 or 4 healthy
+ *   models toward circuit-open for nothing), and it merges fundamentally
+ *   different situations (dead key vs quota pause vs busy model) into one
+ *   counter.
+ *
+ *   📖 v2 classifies every failure into a typed kind, and each kind carries an
+ *   explicit policy: does the attempt fail over to the next candidate, does it
+ *   damage the model's health (circuit breaker), does it block the whole
+ *   provider (dead key), or does it pause the model for a quota window
+ *   (Retry-After aware). The classifier is pure: it never mutates runtime
+ *   state, so it is trivially unit-testable.
+ *
+ * @functions
+ *   → classifyStatus(status) - Map an HTTP status code to a failure kind
+ *   → classifyFailure(input) - Full verdict for a failure (kind + policy flags)
+ *   → clientStatusForKind(kind) - Status code to send the client when every
+ *     model in the set failed with that kind
+ *
+ * @exports FAILURE_KINDS, classifyStatus, classifyFailure, clientStatusForKind
+ */
+
+/**
+ * 📖 Every failure kind the v2 router can produce. Kept as a plain frozen
+ * object so tests and dashboards can enumerate them without relying on
+ * `Object.keys` order of a TS enum-like structure.
+ */
+export const FAILURE_KINDS = Object.freeze({
+  AUTH: 'auth_error',
+  RATE_LIMIT: 'rate_limit',
+  QUOTA: 'quota_exhausted',
+  TIMEOUT: 'timeout',
+  NETWORK: 'network_error',
+  SERVER: 'provider_server_error',
+  OVERLOADED: 'model_overloaded',
+  INVALID_REQUEST: 'invalid_request',
+  INVALID_JSON: 'invalid_json',
+  EMPTY_CHOICES: 'empty_choices',
+  EMPTY_CONTENT: 'empty_content',
+  ERROR_PAYLOAD: 'error_payload',
+  HTML: 'html_maintenance',
+  EMPTY_STREAM: 'empty_stream',
+  STREAM_STALL: 'stream_stall',
+  PROVIDER_URL: 'provider_url_unresolvable',
+})
+
+// 📖 HTTP status → kind. 529 is the non-standard "Overloaded" status used by
+// several inference providers: it is model-scoped (that specific model is
+// busy), not a provider-wide outage.
+const STATUS_KIND_MAP = new Map([
+  [401, FAILURE_KINDS.AUTH],
+  [403, FAILURE_KINDS.AUTH],
+  [408, FAILURE_KINDS.TIMEOUT],
+  [429, FAILURE_KINDS.RATE_LIMIT],
+  [500, FAILURE_KINDS.SERVER],
+  [502, FAILURE_KINDS.SERVER],
+  [503, FAILURE_KINDS.SERVER],
+  [504, FAILURE_KINDS.SERVER],
+  [529, FAILURE_KINDS.OVERLOADED],
+])
+
+/**
+ * 📖 Map an HTTP status code to a coarse failure kind.
+ * @param {number} status
+ * @returns {string} one of FAILURE_KINDS values
+ */
+export function classifyStatus(status) {
+  const kind = STATUS_KIND_MAP.get(status)
+  if (kind) return kind
+  if (status >= 500) return FAILURE_KINDS.SERVER
+  if (status >= 400) return FAILURE_KINDS.INVALID_REQUEST
+  return FAILURE_KINDS.NETWORK
+}
+
+/**
+ * 📖 Build the full policy verdict for a failure.
+ *
+ * @param {object} input
+ * @param {string|null} [input.kind] - explicit kind (content-level failures);
+ *   when omitted the kind is derived from `status`.
+ * @param {number|null} [input.status] - upstream HTTP status
+ * @param {number|null} [input.retryAfterMs] - parsed Retry-After from upstream
+ * @returns {{
+ *   kind: string,
+ *   blame: 'provider'|'model'|'client',
+ *   failover: boolean,
+ *   healthDamage: boolean,
+ *   blockProvider: boolean,
+ *   quotaPauseMs: number|null,
+ *   clientStatus: number,
+ * }}
+ */
+export function classifyFailure({ kind = null, status = null, retryAfterMs = null } = {}) {
+  const resolvedKind = kind || (status != null ? classifyStatus(status) : FAILURE_KINDS.NETWORK)
+
+  switch (resolvedKind) {
+    case FAILURE_KINDS.AUTH:
+      // 📖 Dead or unauthorized key: fail over AND block the rest of this
+      // provider for the request, but do NOT spin the circuit breaker: the
+      // model itself is fine, the credential is not. markAuthError handles it.
+      return {
+        kind: resolvedKind,
+        blame: 'provider',
+        failover: true,
+        healthDamage: false,
+        blockProvider: true,
+        quotaPauseMs: null,
+        clientStatus: 401,
+      }
+    case FAILURE_KINDS.RATE_LIMIT:
+    case FAILURE_KINDS.QUOTA: {
+      // 📖 Rate limited: fail over and pause THIS model for the Retry-After
+      // window (capped) so it stops eating traffic it cannot serve. Health
+      // damage stays on: repeated 429s on every attempt legitimately mean
+      // the model is not usable right now.
+      const pauseMs = retryAfterMs != null ? Math.min(Math.max(0, retryAfterMs), 15 * 60 * 1000) : null
+      return {
+        kind: resolvedKind,
+        blame: 'model',
+        failover: true,
+        healthDamage: true,
+        blockProvider: false,
+        quotaPauseMs: pauseMs,
+        clientStatus: 429,
+      }
+    }
+    case FAILURE_KINDS.INVALID_REQUEST:
+      // 📖 Blame attribution fix: a 400/404/413/422 usually means the CLIENT
+      // payload does not fit this model (unsupported tools, too-large body).
+      // v2 still fails over (another model may accept the format) but never
+      // counts it toward the circuit breaker, so one oversized payload can no
+      // longer open circuits on three healthy models.
+      return {
+        kind: resolvedKind,
+        blame: 'client',
+        failover: true,
+        healthDamage: false,
+        blockProvider: false,
+        quotaPauseMs: null,
+        clientStatus: status || 400,
+      }
+    case FAILURE_KINDS.OVERLOADED:
+      // 📖 529 "Overloaded" is scoped to the model, never the provider.
+      return {
+        kind: resolvedKind,
+        blame: 'model',
+        failover: true,
+        healthDamage: true,
+        blockProvider: false,
+        quotaPauseMs: null,
+        clientStatus: 529,
+      }
+    case FAILURE_KINDS.TIMEOUT:
+    case FAILURE_KINDS.STREAM_STALL:
+      return {
+        kind: resolvedKind,
+        blame: 'provider',
+        failover: true,
+        healthDamage: true,
+        blockProvider: false,
+        quotaPauseMs: null,
+        clientStatus: 504,
+      }
+    case FAILURE_KINDS.NETWORK:
+    case FAILURE_KINDS.SERVER:
+    case FAILURE_KINDS.PROVIDER_URL:
+      return {
+        kind: resolvedKind,
+        blame: 'provider',
+        failover: true,
+        healthDamage: true,
+        blockProvider: false,
+        quotaPauseMs: null,
+        clientStatus: 502,
+      }
+    case FAILURE_KINDS.HTML:
+      return {
+        kind: resolvedKind,
+        blame: 'provider',
+        failover: true,
+        healthDamage: true,
+        blockProvider: false,
+        quotaPauseMs: null,
+        clientStatus: 503,
+      }
+    case FAILURE_KINDS.INVALID_JSON:
+    case FAILURE_KINDS.EMPTY_CHOICES:
+    case FAILURE_KINDS.EMPTY_CONTENT:
+    case FAILURE_KINDS.ERROR_PAYLOAD:
+    case FAILURE_KINDS.EMPTY_STREAM:
+      // 📖 The notorious "HTTP 200 but garbage" family: the provider answered
+      // with a usable transport envelope but no usable answer. These MUST fail
+      // over and MUST count as real failures, otherwise the router "succeeds"
+      // with empty output (v1's biggest blind spot).
+      return {
+        kind: resolvedKind,
+        blame: 'provider',
+        failover: true,
+        healthDamage: true,
+        blockProvider: false,
+        quotaPauseMs: null,
+        clientStatus: 502,
+      }
+    default:
+      return {
+        kind: resolvedKind,
+        blame: 'provider',
+        failover: true,
+        healthDamage: true,
+        blockProvider: false,
+        quotaPauseMs: null,
+        clientStatus: 502,
+      }
+  }
+}
+
+/**
+ * 📖 Status code to send the client when every candidate failed with the same
+ * kind. Used to refine the generic 503 "all models failed" response.
+ * @param {string} kind
+ * @returns {number}
+ */
+export function clientStatusForKind(kind) {
+  return classifyFailure({ kind }).clientStatus
+}

package/src/core/router-v2/request-history.js

@@ -0,0 +1,137 @@
+/**
+ * @file request-history.js
+ * @description Persisted request history for Router v2.
+ *
+ * @details
+ *   📖 v1 kept the last 20 request log entries in memory only: restart the
+ *   daemon and the evidence was gone, which made "the router feels flaky"
+ *   impossible to debug after the fact. v2 persists every routed request
+ *   (bounded ring, atomic writes, debounced flush) with its full decision
+ *   trace so the TUI overlay and the web page can show a durable fallback
+ *   chain per request.
+ *
+ *   📖 Storage shape: a single JSON file `{ version, entries: [...] }` written
+ *   with `atomicWriteJson`. Entries are capped (oldest dropped) and each one
+ *   is routing metadata only: model keys, statuses, timings, error kinds.
+ *   No prompts, no completions, no credentials.
+ *
+ * @functions
+ *   → new RequestHistory({ path, logger, maxEntries }) - Load persisted history
+ *   → history.append(entry) - Add one request record (debounced flush)
+ *   → history.recent(limit) - Newest-first slice
+ *   → history.stats() - Aggregate counters over the retained window
+ *   → history.flush({ force }) / history.clear()
+ *
+ * @exports RequestHistory
+ */
+
+import { existsSync, readFileSync } from 'node:fs'
+import { atomicWriteJson, safeJsonParse } from '../shared-helpers.js'
+
+const STATE_VERSION = 1
+const FLUSH_DEBOUNCE_MS = 2000
+export const DEFAULT_MAX_ENTRIES = 500
+
+export class RequestHistory {
+  constructor({ path, logger, maxEntries = DEFAULT_MAX_ENTRIES } = {}) {
+    this.path = path
+    this.logger = logger
+    this.maxEntries = Math.max(10, maxEntries)
+    this.entries = []
+    this.dirty = false
+    this.flushTimer = null
+    this.load()
+  }
+
+  load() {
+    try {
+      if (!existsSync(this.path)) return
+      const parsed = safeJsonParse(readFileSync(this.path, 'utf8'), null)
+      if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) return
+      this.entries = parsed.entries
+        .filter((entry) => entry && typeof entry === 'object' && typeof entry.request_id === 'string')
+        .slice(-this.maxEntries)
+    } catch (error) {
+      this.logger?.warn?.('Request history load failed; starting fresh', { error: error?.message })
+      this.entries = []
+    }
+  }
+
+  /**
+   * 📖 Append one completed request record. Expected shape (all routing
+   * metadata, produced by the routeRequest finally-block):
+   * `{ request_id, at, set, protocol, model_requested, served_model, outcome,
+   *    attempts: [...], skipped: [...], wall_ms, tokens, stream,
+   *    last_resort_used }`
+   */
+  append(entry) {
+    if (!entry || typeof entry !== 'object') return
+    this.entries.push(entry)
+    while (this.entries.length > this.maxEntries) this.entries.shift()
+    this.dirty = true
+    this.scheduleFlush()
+  }
+
+  /** 📖 Newest-first slice for API responses and dashboards. */
+  recent(limit = 50) {
+    const n = Math.max(1, Math.min(limit, this.maxEntries))
+    return this.entries.slice(-n).reverse()
+  }
+
+  /** 📖 Aggregate counters over the retained window (newest `window` entries). */
+  stats({ window = this.maxEntries } = {}) {
+    const slice = this.entries.slice(-Math.max(1, window))
+    const total = slice.length
+    let served = 0
+    let failovers = 0
+    let lastResort = 0
+    let attemptsSum = 0
+    let wallSum = 0
+    let wallCount = 0
+    for (const entry of slice) {
+      if (entry.outcome === 'served') served += 1
+      if ((entry.attempts?.length || 0) > 1) failovers += 1
+      if (entry.last_resort_used === true) lastResort += 1
+      attemptsSum += entry.attempts?.length || 0
+      if (Number.isFinite(entry.wall_ms) && entry.wall_ms > 0) {
+        wallSum += entry.wall_ms
+        wallCount += 1
+      }
+    }
+    return {
+      retained: total,
+      served,
+      all_failed: total - served,
+      failover_rate: total > 0 ? Math.round((failovers / total) * 100) / 100 : 0,
+      success_rate: total > 0 ? Math.round((served / total) * 100) / 100 : null,
+      last_resort_used: lastResort,
+      avg_attempts: total > 0 ? Math.round((attemptsSum / total) * 100) / 100 : 0,
+      avg_wall_ms: wallCount > 0 ? Math.round(wallSum / wallCount) : null,
+    }
+  }
+
+  scheduleFlush() {
+    if (this.flushTimer) return
+    this.flushTimer = setTimeout(() => {
+      this.flushTimer = null
+      this.flush()
+    }, FLUSH_DEBOUNCE_MS)
+    if (typeof this.flushTimer.unref === 'function') this.flushTimer.unref()
+  }
+
+  flush() {
+    if (!this.dirty) return
+    try {
+      atomicWriteJson(this.path, { version: STATE_VERSION, entries: this.entries }, 0o600)
+      this.dirty = false
+    } catch (error) {
+      this.logger?.warn?.('Request history write failed', { error: error?.message })
+    }
+  }
+
+  clear() {
+    this.entries = []
+    this.dirty = true
+    this.flush()
+  }
+}

package/src/core/router-v2/response-gate.js

@@ -0,0 +1,175 @@
+/**
+ * @file response-gate.js
+ * @description Content-level response validation for Router v2.
+ *
+ * @details
+ *   📖 The single biggest v1 blind spot: free providers sometimes answer
+ *   HTTP 200 with an empty `choices` array, an embedded `error` object, or an
+ *   SSE stream that closes without ever producing useful content. v1 counted
+ *   all of those as SUCCESS, so the failover engine never fired and coding
+ *   agents silently received nothing.
+ *
+ *   📖 The gate lives in two layers:
+ *   - `validateChatCompletionPayload` checks a parsed non-streaming 200 body.
+ *   - `createStreamReadinessTracker` inspects SSE chunks as they arrive and
+ *     answers "has this stream produced anything useful yet?" so the daemon
+ *     can hold the first chunks briefly, fail over on an error payload before
+ *     anything reaches the client, and treat a content-less close as a real
+ *     failure.
+ *
+ *   📖 "Useful content" means: a non-empty text delta, tool_calls, a legacy
+ *   function_call, or reasoning tokens. A bare `{role: "assistant"}` first
+ *   chunk is normal framing, not content. Reasoning-only output DOES count
+ *   (some thinking models stream reasoning before text), so it is never
+ *   treated as a failure.
+ *
+ * @functions
+ *   → validateChatCompletionPayload(payload, opts) - Gate a parsed JSON body
+ *   → createStreamReadinessTracker() - Incremental SSE usefulness tracker
+ *   → estimateTokens(text) - Cheap completion-token estimate for streams
+ *
+ * @exports validateChatCompletionPayload, createStreamReadinessTracker, estimateTokens
+ */
+
+// 📖 A single SSE event larger than this cannot be a useful first frame of a
+// coding answer; treat it as unparsable garbage rather than buffering forever.
+const MAX_HOLD_BUFFER_BYTES = 256 * 1024
+
+/**
+ * 📖 Validate a parsed non-streaming chat-completion payload.
+ *
+ * @param {unknown} payload - the JSON.parse result of a 200 body
+ * @param {{ mode?: 'strict'|'basic' }} [opts]
+ *   - strict (default): also requires the first choice to carry actual
+ *     content (text, tool_calls, function_call or reasoning).
+ *   - basic: only rejects structural garbage (no choices, embedded error).
+ * @returns {{ ok: boolean, reason: string|null, detail: string|null }}
+ */
+export function validateChatCompletionPayload(payload, { mode = 'strict' } = {}) {
+  if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
+    return { ok: false, reason: 'invalid_json', detail: 'payload is not an object' }
+  }
+  if (payload.error !== undefined && payload.error !== null) {
+    const err = payload.error
+    const detail = typeof err === 'object' && err !== null
+      ? String(err.code || err.type || err.message || 'upstream error object').slice(0, 200)
+      : String(err).slice(0, 200)
+    return { ok: false, reason: 'error_payload', detail }
+  }
+  if (!Array.isArray(payload.choices) || payload.choices.length === 0) {
+    return { ok: false, reason: 'empty_choices', detail: 'choices missing or empty' }
+  }
+  if (mode === 'basic') return { ok: true, reason: null, detail: null }
+
+  const choice = payload.choices[0]
+  if (!choice || typeof choice !== 'object') {
+    return { ok: false, reason: 'empty_content', detail: 'first choice is not an object' }
+  }
+  const msg = choice.message
+  const content = typeof msg?.content === 'string' ? msg.content.trim() : ''
+  const hasToolCalls = Array.isArray(msg?.tool_calls) && msg.tool_calls.length > 0
+  const hasFunctionCall = msg?.function_call != null
+  const reasoning = typeof msg?.reasoning_content === 'string' ? msg.reasoning_content.trim() : ''
+  const reasoningAlt = typeof msg?.reasoning === 'string' ? msg.reasoning.trim() : ''
+  if (!content && !hasToolCalls && !hasFunctionCall && !reasoning && !reasoningAlt) {
+    return { ok: false, reason: 'empty_content', detail: 'no text, tool_calls or reasoning in first choice' }
+  }
+  return { ok: true, reason: null, detail: null }
+}
+
+/**
+ * 📖 Incremental tracker for SSE streams. Feed every decoded text chunk into
+ * `observe()`; the tracker extracts `data:` JSON payloads and reports:
+ *   - useful: seen real content (text delta, tool_calls, reasoning, ...)
+ *   - errorPayload: an SSE `data:` frame carried an `error` object
+ * The daemon combines those signals with its hold-buffer policy to decide
+ * between flush-to-client, fail-over-before-first-byte, and "content-less
+ * stream = failure".
+ *
+ * @returns {{
+ *   observe(text: string): void,
+ *   get useful(): boolean,
+ *   get errorPayload(): boolean,
+ *   get bytesSeen(): number,
+ *   get unparsableFrames(): number,
+ *   describe(): string,
+ * }}
+ */
+export function createStreamReadinessTracker() {
+  let useful = false
+  let errorPayload = false
+  let bytesSeen = 0
+  let unparsableFrames = 0
+  let lineRemainder = ''
+
+  const inspectDataPayload = (raw) => {
+    const trimmed = raw.trim()
+    if (!trimmed || trimmed === '[DONE]') return
+    let parsed
+    try {
+      parsed = JSON.parse(trimmed)
+    } catch {
+      unparsableFrames += 1
+      return
+    }
+    if (!parsed || typeof parsed !== 'object') return
+    if (parsed.error !== undefined && parsed.error !== null) {
+      errorPayload = true
+      return
+    }
+    const choices = parsed.choices
+    if (!Array.isArray(choices)) return
+    for (const choice of choices) {
+      const delta = choice?.delta ?? choice?.message
+      if (!delta || typeof delta !== 'object') continue
+      const content = typeof delta.content === 'string' ? delta.content : ''
+      if (content.length > 0) useful = true
+      if (Array.isArray(delta.tool_calls) && delta.tool_calls.length > 0) useful = true
+      if (delta.function_call != null) useful = true
+      const reasoning = typeof delta.reasoning_content === 'string' ? delta.reasoning_content : ''
+      if (reasoning.length > 0) useful = true
+      const reasoningAlt = typeof delta.reasoning === 'string' ? delta.reasoning : ''
+      if (reasoningAlt.length > 0) useful = true
+    }
+  }
+
+  return {
+    observe(text) {
+      if (typeof text !== 'string' || text.length === 0) return
+      bytesSeen += Buffer.byteLength(text)
+      const data = lineRemainder + text
+      const lines = data.split('\n')
+      // 📖 The last element is either an incomplete line (keep it for next
+      // chunk) or an empty trailing piece after a final newline.
+      lineRemainder = lines.pop() ?? ''
+      for (const line of lines) {
+        const trimmed = line.trim()
+        if (!trimmed || trimmed.startsWith(':')) continue
+        if (trimmed.startsWith('data:')) {
+          inspectDataPayload(trimmed.slice(5))
+          if (useful && errorPayload) return
+        }
+      }
+    },
+    get useful() { return useful },
+    get errorPayload() { return errorPayload },
+    get bytesSeen() { return bytesSeen },
+    get unparsableFrames() { return unparsableFrames },
+    get maxHoldBytes() { return MAX_HOLD_BUFFER_BYTES },
+    describe() {
+      return `useful=${useful} error=${errorPayload} bytes=${bytesSeen} unparsable=${unparsableFrames}`
+    },
+  }
+}
+
+/**
+ * 📖 Cheap completion-token estimate for streamed responses where the
+ * upstream never sends a usage block. Roughly 4 characters per token is the
+ * usual rule of thumb for English/code; good enough for usage dashboards.
+ * @param {string} text
+ * @returns {number}
+ */
+export function estimateTokens(text) {
+  if (typeof text !== 'string' || text.length === 0) return 0
+  return Math.max(1, Math.ceil(text.length / 4))
+}