@qadanm/opencode-smart-router 0.0.0-stage → 0.4.0

package/src/core/context/strategy.ts

@@ -0,0 +1,188 @@
+/**
+ * Context-management strategies.
+ *
+ * These are deliberately separate mechanisms, because they solve different
+ * problems and have very different costs:
+ *
+ *  - **Provider prompt caching** happens on the provider's side. The router
+ *    never controls it; it only avoids destroying it by needlessly switching
+ *    models. There is nothing to configure and it must never be disabled.
+ *  - **Context budgeting** is arithmetic done before a request: does the
+ *    transcript fit the model window the provider actually accepts?
+ *  - **Compaction** is destructive. It replaces the transcript with a summary
+ *    and permanently loses detail. On the old $10 plan it was the only way to
+ *    keep a long session on the cheapest provider, so the router compacted
+ *    eagerly.
+ *
+ * Only the third is a strategy. The three modes below differ almost entirely
+ * in *when*, and *whether*, compaction happens.
+ */
+
+import type { ContextManagementMode, ContextManagementSettings } from "../types.js"
+
+/** A margin added on top of the measured transcript before judging fit. */
+export const CONTEXT_ELIGIBILITY_MARGIN_TOKENS = 20_000
+
+/**
+ * The smallest share of a window that the reply margin may claim.
+ *
+ * A flat 20,000-token margin is right for a 1M window and impossible for an
+ * 8,192-token local one: the margin alone would exceed the window, so every
+ * request would be rejected before it started. The margin therefore shrinks
+ * with small windows and never grows beyond the flat value.
+ *
+ * The floor keeps a usable share of the window for the reply. At 8,192 tokens
+ * this reserves 1,024 and leaves 6,758 of transcript after the safety fraction,
+ * which fits a component, its styles and a focused instruction.
+ */
+const MINIMUM_REPLY_RESERVE_FRACTION = 0.125
+
+/**
+ * The reply margin that actually applies to a given window.
+ *
+ * Exported so the routing gate and the compaction strategy cannot drift apart:
+ * both call this with the same window and get the same answer.
+ */
+export const replyMarginFor = (windowTokens: number): number => {
+  if (!Number.isFinite(windowTokens) || windowTokens <= 0) return CONTEXT_ELIGIBILITY_MARGIN_TOKENS
+  return Math.min(CONTEXT_ELIGIBILITY_MARGIN_TOKENS, Math.floor(windowTokens * MINIMUM_REPLY_RESERVE_FRACTION))
+}
+
+/** The fraction of a model's window a request must stay under. */
+export const CONTEXT_SAFE_WINDOW_FRACTION = 0.95
+
+export interface CompactionDecision {
+  readonly compact: boolean
+  /** Why this decision was reached, in plain language for the owner. */
+  readonly reason: string
+  /** The transcript size that triggered the decision, when one did. */
+  readonly triggerTokens?: number
+  /** The size compaction is trying to get under, when relevant. */
+  readonly targetTokens?: number
+  /** True when the host's verified limit, not a router preference, drove it. */
+  readonly hardLimit: boolean
+}
+
+const no: (reason: string) => CompactionDecision = (reason) => ({ compact: false, reason, hardLimit: false })
+
+export interface CompactionContext {
+  readonly mode: ContextManagementMode
+  readonly settings: ContextManagementSettings
+  readonly contextTokens: number
+  /**
+   * The largest window the session can actually use on the provider it is on.
+   * This is the provider's accepted limit, never a theoretical maximum.
+   */
+  readonly effectiveWindow: number
+  /**
+   * The window of the cheapest qualified route. On the budget-optimized mode
+   * this is smaller than `effectiveWindow`, and staying inside it is what
+   * keeps a session on the cheapest provider.
+   */
+  readonly cheapestWindow?: number
+}
+
+/**
+ * Decide whether to compact.
+ *
+ * All three modes keep a real safety floor: a request that would exceed the
+ * provider's verified window must be handled before it is sent, because the
+ * alternative is a failed turn or silent truncation by the provider. They
+ * differ in whether *any* earlier compaction happens.
+ */
+export const decideCompaction = (input: CompactionContext): CompactionDecision => {
+  const { mode, settings, contextTokens, effectiveWindow } = input
+  if (!settings.enabled) {
+    return { compact: false, reason: "Context management is disabled; native context handling only.", hardLimit: false }
+  }
+  if (contextTokens <= 0) return no("No measured context size yet.")
+  if (effectiveWindow <= 0) return no("The model's accepted context window is unknown; no router-driven compaction.")
+
+  const required = contextTokens + replyMarginFor(effectiveWindow)
+  const safeLimit = effectiveWindow * CONTEXT_SAFE_WINDOW_FRACTION
+  const overflow = required > safeLimit
+
+  if (mode === "native") {
+    // Native mode still refuses to knowingly send an over-window request.
+    // The host's own compaction is the preferred recovery; the router only
+    // steps in when the request genuinely cannot fit.
+    if (!overflow) return no("Native mode: the transcript fits the verified window.")
+    if (settings.allowEmergencyCompaction === false) {
+      return {
+        compact: false,
+        reason: "Native mode: transcript is at the verified limit, but emergency compaction is disabled.",
+        triggerTokens: contextTokens,
+        targetTokens: Math.floor(safeLimit),
+        hardLimit: true,
+      }
+    }
+    return {
+      compact: true,
+      reason: "Native mode emergency compaction: the transcript would exceed the model's verified window.",
+      triggerTokens: contextTokens,
+      targetTokens: Math.floor(safeLimit),
+      hardLimit: true,
+    }
+  }
+
+  if (mode === "conservative") {
+    if (!overflow) return no("Conservative mode: the transcript fits the verified window; history preserved.")
+    return {
+      compact: true,
+      reason: "Conservative mode: compacting only because the transcript would exceed the verified window.",
+      triggerTokens: contextTokens,
+      targetTokens: Math.floor(safeLimit),
+      hardLimit: true,
+    }
+  }
+
+  // budget-optimized: the legacy behavior, preserved deliberately.
+  const target = input.cheapestWindow && input.cheapestWindow > 0 ? input.cheapestWindow : effectiveWindow
+  const targetLimit = target * CONTEXT_SAFE_WINDOW_FRACTION
+  const threshold = Math.min(
+    targetLimit,
+    settings.compactAtTokens > 0 ? settings.compactAtTokens : Number.POSITIVE_INFINITY,
+  )
+  if (required <= threshold) return no("Budget-optimized mode: inside the configured compact threshold.")
+
+  const hardLimit = required > safeLimit
+  return {
+    compact: true,
+    reason: hardLimit
+      ? "Budget-optimized mode: compacting to stay inside the verified window of the cheapest qualified route."
+      : "Budget-optimized mode: compacting early to keep this session on the cheapest qualified route.",
+    triggerTokens: contextTokens,
+    targetTokens: Math.floor(threshold),
+    hardLimit,
+  }
+}
+
+/**
+ * The practical window for a model.
+ *
+ * A model's theoretical maximum and the window a gateway will actually accept
+ * are different numbers. The adapter reports the accepted value into
+ * `ModelRecord.context`, and this helper exists so the distinction is explicit
+ * wherever a limit is used rather than being re-derived at each call site.
+ */
+export const practicalWindow = (model: { context: number; maxOutput: number }): number => {
+  if (model.context <= 0) return 0
+  // Output budget is part of the same window on providers that reserve it
+  // inside the context length. Subtracting it would double-count when the
+  // provider already reports a net input budget, so only a sanity floor is
+  // applied here; the authoritative figure is the reported window.
+  return model.context
+}
+
+export const describeMode = (mode: ContextManagementMode): string => {
+  switch (mode) {
+    case "native":
+      return "Native: no router-driven compaction unless the transcript would exceed the verified window."
+    case "conservative":
+      return "Conservative: history preserved; compaction only to prevent a real overflow."
+    case "budget-optimized":
+      return "Budget-optimized: compacts early to keep sessions on the cheapest qualified route."
+    default:
+      return "Unknown mode."
+  }
+}

package/src/core/context-health.ts

@@ -0,0 +1,39 @@
+import type { ContextHealth, ContextHealthSettings, SessionState } from "./types.js"
+import { pct } from "./util.js"
+
+export const computeContextHealth = (
+  session: SessionState | undefined,
+  settings: ContextHealthSettings,
+): ContextHealth => {
+  const tokens = session?.contextTokens ?? 0
+  const cacheRead = session?.cacheReadTokens ?? 0
+  const cacheSharePct = tokens > 0 ? pct(cacheRead, tokens) : 0
+  if (!settings.enabled) {
+    return { tokens, cacheRead, cacheSharePct, state: "healthy", message: "Context-health monitoring is disabled." }
+  }
+  if (tokens >= settings.recommendAtTokens) {
+    return {
+      tokens,
+      cacheRead,
+      cacheSharePct,
+      state: "recommend_new_session",
+      message: `This session has about ${tokens.toLocaleString()} context tokens. A fresh session with a short handoff will usually cost less per turn. The router will not compact or replace this conversation.`,
+    }
+  }
+  if (tokens >= settings.warnAtTokens) {
+    return {
+      tokens,
+      cacheRead,
+      cacheSharePct,
+      state: "watch",
+      message: `Context is about ${tokens.toLocaleString()} tokens. Cache reads are cheaper than fresh input but are not free. Consider a new session for unrelated work.`,
+    }
+  }
+  return {
+    tokens,
+    cacheRead,
+    cacheSharePct,
+    state: "healthy",
+    message: `${tokens.toLocaleString()} context tokens; ${cacheSharePct.toFixed(0)}% arrived as cache reads. Cache reads are cheaper than fresh input but are not free.`,
+  }
+}

package/src/core/failures.ts

@@ -0,0 +1,398 @@
+import type { FailureClass, SessionState } from "./types.js"
+
+export interface FailureInput {
+  readonly type?: string
+  readonly message: string
+  readonly status?: number
+}
+
+export interface FailureAssessment {
+  readonly class: FailureClass
+  readonly retryable: boolean
+  readonly capabilityFailure: boolean
+  readonly escalate: boolean
+  readonly cooldownMs: number
+  readonly reason: string
+  /**
+   * True when the failure is the provider's fault and should count against
+   * the model's reliability record. A user-initiated interrupt is not.
+   */
+  readonly countAgainstModel: boolean
+  /**
+   * True when the fault is in the provider's own pre-request path (billing
+   * scope, plan entitlement) rather than in the serving model. Such a fault
+   * belongs to the provider as a whole and must not be charged to a model.
+   */
+  readonly providerScoped: boolean
+  /**
+   * True when the provider's gateway refused the work because it is saturated.
+   *
+   * Kept separate from `providerScoped` because the two need different
+   * handling downstream. A billing-scope fault happens before a model is even
+   * chosen, so cooling that model would be arbitrary blame. A saturated
+   * gateway is a reason to step off the endpoint that just refused, so the
+   * serving model is held back too.
+   */
+  readonly saturation?: boolean
+}
+
+const includes = (value: string, terms: ReadonlyArray<string>) => terms.some((term) => value.includes(term))
+
+export const classifyFailure = (input: FailureInput, session?: SessionState): FailureAssessment => {
+  const message = input.message.toLowerCase()
+  const type = (input.type ?? "").toLowerCase()
+  const status = input.status
+
+  // A user or agent interrupt is not a provider fault and must never count
+  // against a model's reliability.
+  if (
+    type.includes("abort") ||
+    type.includes("interrupt") ||
+    includes(message, ["step interrupted", "request aborted", "operation canceled", "operation cancelled", "user aborted"])
+  ) {
+    return {
+      class: "interrupted",
+      retryable: false,
+      capabilityFailure: false,
+      escalate: false,
+      cooldownMs: 0,
+      reason: "The step was interrupted; this is not evidence of a provider fault.",
+      countAgainstModel: false,
+      providerScoped: false,
+    }
+  }
+
+
+  /**
+   * The account is out of allowance.
+   *
+   * This is the compensation for every provider whose quota cannot be read in
+   * advance: OpenDesign publishes no usage endpoint for an inference key, so the
+   * router's own figure is a lower bound and it will not notice a spent
+   * allowance until the provider says no. The first refusal is therefore the
+   * signal, and it has to be used properly — retrying cannot conjure credit, and
+   * hammering the same exhausted provider wastes a turn each time.
+   *
+   * Checked before the generic 4xx paths because gateways disagree on the status
+   * code: 402 is the honest one, but a 400 or 403 with "insufficient credits" is
+   * common, and treating that as `invalid_request` would blame the user's prompt
+   * for an empty wallet.
+   */
+  if (
+    // 402 is the unambiguous signal. A 5xx is never this: a server fault in the
+    // billing path is an outage, not an empty account, and OpenDesign returns
+    // exactly that as `workspace_billing_scope_unavailable`.
+    status === 402 ||
+    ((status === undefined || status < 500) &&
+      includes(message, [
+        "insufficient credit",
+        "insufficient_credit",
+        "insufficient balance",
+        "insufficient_quota",
+        "no credit left",
+        "out of credit",
+        "credit balance is",
+        "credit limit reached",
+        "quota exceeded",
+        "quota_exceeded",
+        "allowance exhausted",
+        "allowance is exhausted",
+        "exceeded your current quota",
+      ]))
+  ) {
+    return {
+      class: "quota_exhausted",
+      retryable: false,
+      capabilityFailure: false,
+      escalate: true,
+      cooldownMs: 0,
+      reason: "The provider reports no allowance left; retrying the same model cannot help.",
+      // The account is empty, not the model. Charging this to the model would
+      // make a perfectly good model look unreliable and quietly drop it.
+      countAgainstModel: false,
+      providerScoped: true,
+    }
+  }
+
+  if (
+    includes(message, [
+      "context length",
+      "context_length_exceeded",
+      "maximum context",
+      "too many tokens",
+      "prompt is too long",
+      "exceeds the context",
+      "context window",
+      "input is too long",
+    ])
+  ) {
+    return {
+      class: "context_overflow",
+      retryable: false,
+      capabilityFailure: false,
+      escalate: false,
+      cooldownMs: 0,
+      reason: "Context overflow is a session-size problem, not an intelligence failure. Compact or start a handoff session.",
+      countAgainstModel: false,
+      providerScoped: false,
+    }
+  }
+
+  if (status === 429 || includes(message, ["rate limit", "rate_limit", "too many requests"]) || type.includes("rate_limit") || type.includes("ratelimit")) {
+    return {
+      class: "rate_limit",
+      retryable: true,
+      capabilityFailure: false,
+      escalate: false,
+      cooldownMs: 2 * 60_000,
+      reason: "Provider rate limit; wait or use another provider. A stronger model would consume the same limit.",
+      countAgainstModel: true,
+      providerScoped: false,
+    }
+  }
+
+  if (status === 401 || status === 403 || includes(message, ["unauthorized", "invalid api key", "authentication", "forbidden", "permission denied"]) || type.includes("auth")) {
+    return {
+      class: "auth",
+      retryable: false,
+      capabilityFailure: false,
+      escalate: false,
+      cooldownMs: 15 * 60_000,
+      reason: "Authentication or authorization failed; escalating models cannot fix a credential problem.",
+      countAgainstModel: true,
+      providerScoped: false,
+    }
+  }
+
+  if (
+    includes(message, ["image", "media", "unsupported content", "invalid image", "vision", "content type"]) &&
+    includes(message, ["unsupported", "invalid", "not support", "reject", "format"])
+  ) {
+    return {
+      class: "image_rejected",
+      retryable: false,
+      capabilityFailure: false,
+      escalate: false,
+      cooldownMs: 10 * 60_000,
+      reason: "The endpoint rejected the image input; route visual work only to a verified vision model.",
+      countAgainstModel: true,
+      providerScoped: false,
+    }
+  }
+
+  // A deterministic request-shape rejection: the provider refused the prompt
+  // before generating anything (an HTTP 400-class failure carries no billing,
+  // rate or capacity meaning). Retry cannot change the outcome; the request
+  // itself is the problem, so the router repairs or fails over instead of
+  // retrying. The type-based arm requires a sub-500 status (or none, for a
+  // locally built rejection): a gateway that fails the same type with a 5xx
+  // is an outage, and stays with the outage rules below.
+  const roleDiscriminationFailure =
+    includes(message, [
+      "invalid_request_error",
+      "expected 'user' | 'assistant'",
+      'expected "user" | "assistant"',
+      "invalid discriminator value",
+    ]) || /messages\.\d+\.role/i.test(input.message)
+  const promptRejected =
+    roleDiscriminationFailure ||
+    ((type.includes("invalid-request") || type.includes("invalid_request")) &&
+      (input.status === undefined || input.status < 500))
+
+  if (promptRejected) {
+    return {
+      class: "prompt_rejected",
+      retryable: false,
+      capabilityFailure: false,
+      escalate: false,
+      cooldownMs: 5 * 60_000,
+      reason:
+        "The provider rejected the request shape before generating anything, so retrying cannot help. The router repairs the prompt or fails the turn over to the next best choice instead.",
+      // The model is healthy; the prompt shape was the problem. With prompt
+      // repair this class should not fire at all, and when it does the
+      // session's prompt shape, not the model, is what changed.
+      countAgainstModel: false,
+      providerScoped: false,
+    }
+  }
+
+  // A stream that starts and then dies reports HTTP 200 with a provider
+  // error inside the stream body, so the status code alone never identifies
+  // it. OpenCode surfaces these as a `provider.*` error type, most often
+  // `provider.internal` (an engine fault) or `provider.invalid-output` (a
+  // stream that ended without a finish reason).
+  const providerType = type.startsWith("provider.") || type.includes("invalidprovideroutput")
+  const streamFailure =
+    providerType ||
+    includes(message, [
+      "streaming response failed",
+      "enginecore encountered an issue",
+      "server_error",
+      "provider.internal",
+      "invalid-output",
+      "invalidprovideroutput",
+      "internal server error",
+      "upstream error",
+      "upstream failure",
+      "incomplete stream",
+      "stream closed",
+      "stream ended unexpectedly",
+      "stream ended without finish_reason",
+      "without finish_reason",
+      "connection closed",
+      "unexpected eof",
+    ]) ||
+    (/\[\s*5\d\d\s*\]/.test(message) && includes(message, ["stream", "response", "engine"]))
+
+  // A billing-scope resolution failure happens in the provider's own billing
+  // path before any model is selected. It would hit every model on that
+  // provider identically, so charging it to whichever model happened to be
+  // serving misattributes a provider-wide fault to one model and penalises
+  // a model that did nothing wrong.
+  const providerScopedBillingFault =
+    includes(message, [
+      "workspace_billing_scope_unavailable",
+      "billing_scope_unavailable",
+      "resolve billing scope",
+      "billing scope unavailable",
+      // OpenDesign evaluates the account's plan membership before it serves
+      // any request. Failing that is also a provider-side fault.
+      "membership entitlement",
+      "evaluate membership",
+    ]) || type.includes("billing_scope_unavailable")
+
+  // Capacity exhaustion resembles the billing-scope case above in one respect
+  // and differs in another, so it gets its own flag rather than reusing that
+  // one. Like the billing fault it happens at the gateway before a model does
+  // any work, so it is nothing to hold against the model's ability. Unlike the
+  // billing fault, the endpoint really did refuse this request, so it must be
+  // stepped away from at once.
+  //
+  // This matters because the alternative is worse than useless. When the
+  // gateway is saturated the retry hook can only wait and try again (the host
+  // exposes no way to change model mid-turn), so a fault that never reaches
+  // the provider means every retry and every following prompt lands on a
+  // gateway that is still refusing work.
+  //
+  // Deliberately narrow. Generic "overloaded", 5xx and stream failures stay
+  // model-scoped: those can be a single bad route, and holding an entire
+  // healthy provider back over one flaky model would be its own kind of error.
+  const upstreamSaturation =
+    includes(message, [
+      // The gateway's own advice names the provider as the problem.
+      "alternative llm api service providers",
+      "service is too busy",
+      // Capacity refusals raised by the admission layer.
+      "ttft_admission_refused",
+      "admission refused",
+      "at capacity",
+    ]) || (type.includes("upstream") && includes(message, ["busy", "capacity"]))
+
+  if (
+    status === 502 ||
+    status === 503 ||
+    status === 504 ||
+    streamFailure ||
+    includes(message, [
+      "overloaded",
+      "temporarily unavailable",
+      "upstream",
+      "service unavailable",
+      "provider outage",
+      "bad gateway",
+      "ttft_admission_refused",
+      "admission refused",
+      "at capacity",
+    ])
+  ) {
+    return {
+      class: "provider_outage",
+      retryable: true,
+      capabilityFailure: false,
+      escalate: false,
+      cooldownMs: 5 * 60_000,
+      reason: providerScopedBillingFault
+        ? "The provider's own pre-request path failed before a model was selected. This is provider-scoped, not a fault of the serving model, so it is not counted against the model's reliability."
+        : upstreamSaturation
+          ? "The provider's gateway is saturated. Every model behind it is affected, so the provider is counted as failing, and the endpoint that actually refused is held back too."
+          : "Provider capacity or outage; retry shortly or fail over. A stronger model would hit the same capacity limit, so no escalation is warranted.",
+      countAgainstModel: !providerScopedBillingFault,
+      providerScoped: providerScopedBillingFault,
+      ...(upstreamSaturation && !providerScopedBillingFault ? { saturation: true } : {}),
+    }
+  }
+
+  if (includes(message, ["timeout", "timed out", "econnreset", "econnrefused", "fetch failed", "socket", "network", "dns"])) {
+    return {
+      class: "network",
+      retryable: true,
+      capabilityFailure: false,
+      escalate: false,
+      cooldownMs: 60_000,
+      reason: "Network failure; retry with backoff. This is not evidence that a stronger model is needed.",
+      countAgainstModel: true,
+      providerScoped: false,
+    }
+  }
+
+  if (includes(message, ["tool", "shell command", "command failed", "exit code", "enoent", "permission denied"]) && type.includes("tool")) {
+    return {
+      class: "tool",
+      retryable: false,
+      capabilityFailure: false,
+      escalate: false,
+      cooldownMs: 0,
+      reason: "A tool or command failed; fix the tool, dependency, or permission issue rather than escalating the model.",
+      countAgainstModel: false,
+      providerScoped: false,
+    }
+  }
+
+  if (includes(message, ["invalid request", "bad request", "unsupported parameter", "malformed"])) {
+    return {
+      class: "invalid_request",
+      retryable: false,
+      capabilityFailure: false,
+      escalate: false,
+      cooldownMs: 5 * 60_000,
+      reason: "The request shape was invalid; fix the request or provider compatibility instead of escalating.",
+      countAgainstModel: true,
+      providerScoped: false,
+    }
+  }
+
+  const capabilityFailures = session?.failures.filter((failure) => failure.class === "capability").length ?? 0
+  if (
+    includes(message, ["cannot complete", "unable to complete", "gave up", "i don't know how", "not able to", "incomplete", "failed to implement"]) ||
+    (type.includes("model") && capabilityFailures >= 1)
+  ) {
+    return {
+      class: "capability",
+      retryable: false,
+      capabilityFailure: true,
+      escalate: capabilityFailures < 1,
+      cooldownMs: 0,
+      reason: "Possible capability failure; escalate one tier only after evidence, not after a single ambiguous error.",
+      countAgainstModel: true,
+      providerScoped: false,
+    }
+  }
+
+  return {
+    class: "unknown",
+    retryable: false,
+    capabilityFailure: false,
+    escalate: false,
+    cooldownMs: 30_000,
+    reason: "Unclassified failure; do not escalate automatically without evidence.",
+    countAgainstModel: false,
+    providerScoped: false,
+  }
+}
+
+export const failureCooldown = (assessment: FailureAssessment) => assessment.cooldownMs
+
+export const shouldAvoidProvider = (assessment: FailureAssessment) =>
+  assessment.class === "auth" || assessment.class === "rate_limit" || assessment.class === "provider_outage" || assessment.class === "network"
+
+export const shouldEscalate = (assessment: FailureAssessment) => assessment.escalate && assessment.capabilityFailure