@huynle/hub 0.11.0 → 0.11.1

package/src/central/health/classify.ts

@@ -0,0 +1,209 @@
+// (SS-1, plan v5ms8lgq) A session's health from its status board entry: why it is not making progress, next to the
+// derived state (which stays "working" during a retry). Pure. Typed errors (OpenCode's errorType/errorStatus) decide
+// first; message patterns cover the harnesses that send only text (Claude Code, Codex). An unknown failure is
+// "failed", never a false quota.
+import { providerFailureNeedsUser, type RetryInfo } from "../../status"
+
+export type HealthKind = "ok" | "retrying" | "stalled" | "quota" | "auth" | "model_unavailable" | "context_overflow" | "network" | "failed"
+export const HEALTH_KINDS: readonly HealthKind[] = ["ok", "retrying", "stalled", "quota", "auth", "model_unavailable", "context_overflow", "network", "failed"]
+
+/** (Decision 3kj1d70k) A transient retry counts as stalled after this many attempts or this long since its first retry. */
+export const STALL_ATTEMPTS = 3
+export const STALL_MS = 5 * 60_000
+export const HEALTH_DETAIL_MAX = 200
+/** Only the start of a provider message is examined: causes come first, and the patterns' work stays bounded. */
+export const CAUSE_TEXT_MAX = 2000
+
+export interface SessionHealth {
+  readonly kind: HealthKind
+  /** When this kind was first seen (set by the registry, which keeps the history). */
+  readonly since?: number
+  /** quota: when the provider says it resets (epoch ms), when the message says. */
+  readonly resetAt?: number
+  /** While retrying: the provider retry's attempt number. */
+  readonly attempt?: number
+  /** The provider's message, sanitised (≤ 200 characters, secrets redacted). */
+  readonly detail?: string
+}
+
+export interface HealthInput {
+  readonly state: string
+  /** state "error": the error's message (the status board's detail). */
+  readonly detail?: string
+  readonly retry?: RetryInfo
+  readonly errorType?: string
+  readonly errorStatus?: number
+}
+
+/** A cause that retrying will not fix: the session stays stuck until someone acts. */
+type Hard = "quota" | "auth" | "model_unavailable" | "context_overflow"
+type Cause = { readonly kind: Hard | "network"; readonly resetAt?: number }
+
+// SS-2: Codex's UsageLimitReached ("You’ve hit your usage limit", "out of credits", "spend cap") and Claude Code's
+// subscription limits ("You've hit your limit · resets 3pm", "5-hour limit reached") and billing_error.
+const QUOTA_WORDS = /\bquota\b|usage limit|insufficient[_ ]quota|billing|credit balance|out of credits|spend(?:ing)? (?:limit|cap)|budget (?:limit|exceeded)|hit your limit|\b(?:\d+-hour|weekly|daily|session) limit reached/i
+const RATE_LIMIT = /\b429\b|rate[ -]?limit|too many requests/i
+const AUTH = /\b40[13]\b|unauthori[sz]ed|forbidden|invalid (?:x-)?api[ -_]?key|authentication|not (?:logged|signed) in|(?:token|credentials?) (?:has )?expired|permission denied|oauth[_ ]org[_ ]not[_ ]allowed|account[_ ]on[_ ]hold|account (?:is )?(?:suspended|disabled)|cloud[_ ]credential/i
+const MODEL = /\bmodel\b[^.\n]{0,60}?\b(?:not found|does not exist|doesn't exist|is not available|unavailable|not supported|unsupported)\b|unknown model|no such model|model_not_found|invalid model|\b404\b[^\n]{0,160}\bmodel\b/i
+const CONTEXT = /context (?:length|window)|maximum context|too many tokens|prompt is too long|input is too long|context_length_exceeded|exceeds? the (?:model's )?(?:context|token) limit/i
+const NETWORK = /\bE(?:CONNRESET|CONNREFUSED|CONNABORTED|TIMEDOUT|NOTFOUND|AI_AGAIN|PIPE|HOSTUNREACH|NETUNREACH)\b|socket hang up|network error|fetch failed|connection (?:reset|refused|closed|failed|error)|error sending request|stream disconnected|timed out|\btimeout\b/i
+/** A response the provider stopped by its content policy is no connection problem (Codex reports it as a disconnect). */
+const CONTENT_POLICY = /content[_ ]filter|content[_ ]policy/i
+/**
+ * (Review rule 3) Without a provider-error marker only an errno code may say "network": "timeout" alone is a build
+ * or a command as often as a provider, and client-library wording ("fetch failed", "error sending request for url",
+ * "stream disconnected") is generic. SS-9's typed harness errors carry the real network failures.
+ */
+const CONNECTION = /\bE(?:CONNRESET|CONNREFUSED|CONNABORTED|TIMEDOUT|AI_AGAIN|HOSTUNREACH|NETUNREACH)\b/
+/** A reset hint further ahead than this is not believed (a year). */
+const RESET_MAX_MS = 365 * 24 * 3_600_000
+/**
+ * (SS-9) The typed errors Claude Code (StopFailure's error token) and Codex (codexErrorInfo, snake_case) report: a
+ * kind, or "words" for a marker whose message decides (a rate limit whose detail says "usage limit" is a quota). Any
+ * other claude.* / codex.* type is failed.
+ */
+const HARNESS_TYPES: Readonly<Record<string, Hard | "network" | "words">> = {
+  "claude.billing_error": "quota",
+  "claude.account_on_hold": "quota",
+  "claude.authentication_failed": "auth",
+  "claude.oauth_org_not_allowed": "auth",
+  "claude.cloud_credential_error": "auth",
+  "claude.model_not_found": "model_unavailable",
+  "claude.rate_limit": "words",
+  "claude.overloaded": "words",
+  "claude.invalid_request": "words",
+  "codex.usage_limit_exceeded": "quota",
+  "codex.unauthorized": "auth",
+  "codex.model_not_found": "model_unavailable",
+  "codex.context_window_exceeded": "context_overflow",
+  "codex.http_connection_failed": "network",
+  "codex.response_stream_connection_failed": "network",
+  "codex.response_stream_disconnected": "network",
+  "codex.rate_limit_exceeded": "words",
+  "codex.bad_request": "words",
+}
+const harnessType = (type: string | undefined): boolean => type !== undefined && (type.startsWith("claude.") || type.startsWith("codex."))
+/** A rate limit whose reset is at least this far ahead is a quota, not a transient limit. */
+const QUOTA_RESET_MS = 60 * 60_000
+
+const UNIT_MS: Readonly<Record<string, number>> = { h: 3_600_000, m: 60_000, s: 1000 }
+const RESET_HINT = /(?:reset(?:s|ting)?|try again|retry)\s+(?:after|in|at)\s+([^.;)\n]{1,60})/i
+const DURATION = /(\d+(?:\.\d+)?)\s*(h(?:ours?|rs?)?|m(?:in(?:ute)?s?)?|s(?:ec(?:ond)?s?)?)\b/gi
+
+/** When the provider says the limit resets: "reset after 118h", "resets in 2h 15m", "retry after 90 seconds", or an ISO time. */
+export function parseResetAt(message: string | undefined, now: number): number | undefined {
+  const hint = message?.match(RESET_HINT)?.[1]
+  if (!hint) return undefined
+  const iso = hint.match(/\d{4}-\d{2}-\d{2}T[\d:.]+(?:Z|[+-]\d{2}:?\d{2})?/)?.[0]
+  let at: number
+  if (iso) at = Date.parse(iso)
+  else {
+    let ms = 0
+    for (const [, amount, unit] of hint.matchAll(DURATION)) ms += Number(amount) * UNIT_MS[unit![0]!.toLowerCase()]!
+    if (!(ms > 0)) return undefined
+    at = now + Math.round(ms)
+  }
+  // A valid date, ahead, and within a year: anything else is not believed (and could not be shown as a date).
+  return Number.isFinite(at) && at > now && at - now <= RESET_MAX_MS && !Number.isNaN(new Date(at).getTime()) ? at : undefined
+}
+
+/** Secrets a provider message may echo back, each replaced by [redacted] (the label before it kept). */
+const SECRETS: readonly (readonly [RegExp, string])[] = [
+  [/-----BEGIN [A-Z ]+-----[\s\S]*?(?:-----END [A-Z ]+-----|$)/g, "[redacted]"],
+  [/\b(Authorization|Proxy-Authorization)\s*[:=]\s*(?:Basic|Bearer|Token|Digest)?\s*[^\s,;]+/gi, "$1: [redacted]"],
+  [/\b(Bearer|Basic)\s+[A-Za-z0-9._~+\/=-]{4,}/gi, "$1 [redacted]"],
+  [/\b([a-z][a-z0-9+.-]*:\/\/)[^\s\/@:]+:[^\s\/@]+@/gi, "$1[redacted]@"],
+  [/\b(Set-Cookie|Cookie)\s*:[^\r\n]*/gi, "$1: [redacted]"],
+  [/(hooks\.slack\.com\/(?:services|workflows|triggers)\/)[^\s"'<>)]+/gi, "$1[redacted]"],
+  [/\b(bot)?\d{8,10}:[A-Za-z0-9_-]{30,}/g, "$1[redacted]"],
+  // A key-like name's value: to its closing quote when quoted (spaces included), else to the next separator.
+  [/\b([\w-]*(?:api[_-]?key|key|token|secret|password|passwd|pwd|credential|signature|sig)[\w-]*)(["']?\s*[:=]\s*)(["'])(?:(?!\3)[^\r\n])*\3?/gi, "$1$2$3[redacted]$3"],
+  [/\b([\w-]*(?:api[_-]?key|key|token|secret|password|passwd|pwd|credential|signature|sig)[\w-]*)(["']?\s*[:=]\s*)(?!["'])[^\s"'&,;)]+/gi, "$1$2[redacted]"],
+  [/\beyJ[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}/g, "[redacted]"],
+  [/\b(?:AKIA|ASIA|AGPA|AIDA|AROA)[A-Z0-9]{16}\b/g, "[redacted]"],
+  [/\bgithub_pat_[A-Za-z0-9_]{20,}/g, "[redacted]"],
+  [/\b(?:sk|pk|rk|hst|hat|hcs|hca|hcr|ghp|gho|ghs|ghu|ghr|glpat|xox[abpr])[-_][A-Za-z0-9_-]{8,}/g, "[redacted]"],
+  [/\bAIza[A-Za-z0-9_-]{20,}/g, "[redacted]"],
+  [/\b(?:npm|hf)_[A-Za-z0-9]{20,}/g, "[redacted]"],
+  [/\b[0-9a-fA-F]{40,}\b/g, "[redacted]"],
+  // Any other long base64-like run (40+ characters): a token with no recognisable prefix.
+  [/[A-Za-z0-9+\/_-]{40,}={0,2}/g, "[redacted]"],
+]
+
+/**
+ * The first line only, secrets redacted, no control or format characters, ≤ 200 characters. Prefer showing less:
+ * a provider message is untrusted and may echo credentials.
+ */
+export function sanitizeHealthDetail(text: string | undefined): string | undefined {
+  if (!text) return undefined
+  let redacted = text.slice(0, CAUSE_TEXT_MAX)
+  for (const [pattern, replacement] of SECRETS) redacted = redacted.replace(pattern, replacement)
+  const firstLine = redacted.split(/\r?\n/).find((line) => line.trim() !== "") ?? ""
+  const clean = firstLine
+    .replace(/[\p{Cc}\p{Cf}]/gu, " ")
+    .replace(/\s+/g, " ")
+    .trim()
+  if (!clean) return undefined
+  return clean.length > HEALTH_DETAIL_MAX ? `${clean.slice(0, HEALTH_DETAIL_MAX - 1)}…` : clean
+}
+
+/**
+ * What a typed or worded error means, when it means something specific; undefined for a transient or unknown one.
+ * `marked`: the error carries a provider-error marker (a provider.* type, an HTTP status, or it is a provider retry).
+ * Without one (Claude Code and Codex send text only until SS-9's typed token), the words alone never give a
+ * provider kind: only an unmistakable connection error says network, everything else is failed.
+ */
+function cause(type: string | undefined, status: number | undefined, message: string | undefined, now: number, marked: boolean): Cause | undefined {
+  const text = (message ?? "").slice(0, CAUSE_TEXT_MAX)
+  const resetAt = parseResetAt(text, now)
+  const quota = (): Cause => ({ kind: "quota", ...(resetAt !== undefined ? { resetAt } : {}) })
+  // Typed first (OpenCode). providerFailureNeedsUser's vocabulary: quota, free-limit and budget-like types are quotas.
+  if (type === "provider.auth") return { kind: "auth" }
+  if (type === "provider.no-route") return { kind: "model_unavailable" }
+  if (type === "provider.transport" || type === "provider.timeout") return { kind: "network" }
+  if (type && type !== "provider.rate-limit" && type !== "provider.unknown" && providerFailureNeedsUser(type)) return quota()
+  if (harnessType(type)) {
+    const mapped = HARNESS_TYPES[type!]
+    if (mapped === undefined) return undefined
+    if (mapped !== "words") return mapped === "quota" ? quota() : { kind: mapped }
+  } else if (type && !type.startsWith("provider.")) return undefined // A tool's or harness's own error: never a provider class.
+  if (!marked) return CONNECTION.test(text) && !CONTENT_POLICY.test(text) ? { kind: "network" } : undefined
+  // Then the words (and the status) of the message.
+  if (QUOTA_WORDS.test(text)) return quota()
+  if ((type === "provider.rate-limit" || status === 429 || RATE_LIMIT.test(text)) && resetAt !== undefined && resetAt - now >= QUOTA_RESET_MS) return quota()
+  if (CONTEXT.test(text)) return { kind: "context_overflow" }
+  if (CONTENT_POLICY.test(text)) return undefined
+  if (MODEL.test(text) || (status === 404 && /\bmodel\b/i.test(text))) return { kind: "model_unavailable" }
+  if (status === 401 || status === 403 || AUTH.test(text)) return { kind: "auth" }
+  if (NETWORK.test(text)) return { kind: "network" }
+  return undefined
+}
+
+/**
+ * The session's health now. `retrySince`: when the current provider retry was first seen (the registry keeps it).
+ * A retry for a hard cause (quota, auth, model, context) is that cause at once: retrying will not fix it (decision
+ * 3kj1d70k: quota stalls on its first retry). Any other retry is retrying, then stalled after STALL_ATTEMPTS attempts
+ * or STALL_MS. A failed run is its cause, else failed.
+ */
+export function classifyHealth(input: HealthInput, context: { readonly now: number; readonly retrySince?: number }): SessionHealth {
+  const { now } = context
+  const retry = input.state === "working" ? input.retry : undefined
+  if (retry) {
+    const attempt = typeof retry.attempt === "number" && retry.attempt > 0 ? { attempt: retry.attempt } : {}
+    // A provider retry is itself the marker: only a provider request is retried this way.
+    const found = cause(retry.errorType, retry.errorStatus, retry.message, now, true)
+    const detail = found?.kind === "auth" ? undefined : sanitizeHealthDetail(retry.message)
+    const base = { ...attempt, ...(detail ? { detail } : {}) }
+    if (found && found.kind !== "network") return { kind: found.kind, ...(found.resetAt !== undefined ? { resetAt: found.resetAt } : {}), ...base }
+    const stalled = (retry.attempt ?? 0) >= STALL_ATTEMPTS || (context.retrySince !== undefined && now - context.retrySince >= STALL_MS)
+    return { kind: stalled ? "stalled" : "retrying", ...base }
+  }
+  if (input.state === "error") {
+    const marked = (input.errorType?.startsWith("provider.") ?? false) || harnessType(input.errorType) || input.errorStatus !== undefined
+    const found = cause(input.errorType, input.errorStatus, input.detail, now, marked)
+    // An auth failure keeps no text at all: the kind says enough, and its message is the likeliest to echo a key.
+    const detail = found?.kind === "auth" ? undefined : sanitizeHealthDetail(input.detail)
+    return { kind: found?.kind ?? "failed", ...(found?.resetAt !== undefined ? { resetAt: found.resetAt } : {}), ...(detail ? { detail } : {}) }
+  }
+  return { kind: "ok" }
+}