@huynle/hub 0.10.1 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +80 -0
- package/README.md +36 -8
- package/package.json +1 -1
- package/packages/claude-code/package.json +1 -1
- package/packages/claude-code/src/sessions.ts +4 -1
- package/packages/cli/dist/hub-cli.js +851 -466
- package/packages/cli/package.json +1 -1
- package/packages/cli/src/cli.ts +12 -7
- package/packages/cli/src/decisions.ts +52 -4
- package/packages/cli/src/format.ts +9 -0
- package/packages/cli/src/grants-mcp.ts +65 -0
- package/packages/cli/src/mcp-operations.ts +49 -0
- package/packages/cli/src/mcp.ts +26 -4
- package/packages/cli/src/subscriptions-mcp.ts +76 -0
- package/packages/cli/src/team-intake-mcp.ts +2 -1
- package/packages/cli/src/teams-mcp.ts +9 -3
- package/packages/codex/package.json +1 -1
- package/packages/codex/src/sessions.ts +14 -3
- package/packages/node/package.json +1 -1
- package/packages/preset/package.json +1 -1
- package/packages/protocol/package.json +1 -1
- package/packages/protocol/schema/api/v1/_root.json +119 -0
- package/packages/protocol/schema/api/v1/account.json +419 -0
- package/packages/protocol/schema/api/v1/decisions.json +635 -0
- package/packages/protocol/schema/api/v1/discovery.json +110 -0
- package/packages/protocol/schema/api/v1/events.json +679 -0
- package/packages/protocol/schema/api/v1/session-actions.json +660 -0
- package/packages/protocol/schema/api/v1/sessions.json +2055 -0
- package/packages/protocol/schema/api/v1/speech.json +492 -0
- package/packages/protocol/schema/api/v1/unspecified.json +282 -0
- package/packages/protocol/schema/app-delivery/v1.schema.json +67 -5
- package/packages/protocol/schema/app-front/v1.schema.json +7 -3
- package/packages/protocol/src/api-v1.ts +8 -0
- package/packages/protocol/src/index.ts +2 -0
- package/packages/protocol/src/teams.ts +16 -0
- package/packages/protocol/src/voice-recording.ts +39 -2
- package/packages/runtime-openshell/image/entry.sh +3 -1
- package/packages/runtime-openshell/image/start.sh +3 -1
- package/packages/runtime-openshell/package.json +1 -1
- package/packages/runtime-sdk/package.json +1 -1
- package/packages/sdk/package.json +1 -1
- package/packages/sdk/src/app-front.ts +2 -2
- package/packages/sdk/src/ask-fields.ts +76 -0
- package/packages/sdk/src/client.ts +24 -2
- package/packages/sdk/src/envelope.ts +3 -2
- package/packages/sdk/src/index.ts +1 -0
- package/packages/sdk/src/team-intake.ts +6 -0
- package/packages/sdk/src/teams.ts +4 -0
- package/packages/sdk/src/types.ts +36 -4
- package/packages/sdk/src/wake.ts +9 -4
- package/packages/spoke-sdk/package.json +1 -1
- package/src/central/agent/agent-limits.ts +185 -0
- package/src/central/agent/app-decisions.ts +6 -5
- package/src/central/agent/decisions.ts +136 -9
- package/src/central/agent/delivery.ts +164 -22
- package/src/central/agent/errors.ts +2 -1
- package/src/central/agent/events.ts +20 -2
- package/src/central/agent/grant-routes.ts +14 -1
- package/src/central/agent/grants.ts +23 -0
- package/src/central/agent/launch-engine.ts +9 -6
- package/src/central/agent/message-limits.ts +6 -0
- package/src/central/agent/message-routes.ts +33 -6
- package/src/central/agent/messages.ts +7 -0
- package/src/central/agent/placement.ts +4 -2
- package/src/central/agent/read-routes.ts +5 -0
- package/src/central/agent/routes.ts +13 -0
- package/src/central/agent/subscriptions.ts +15 -4
- package/src/central/api-v1.ts +115 -0
- package/src/central/artifacts/store.ts +4 -2
- package/src/central/concierge/board-results.ts +2 -1
- package/src/central/concierge/notifier.ts +4 -2
- package/src/central/config.ts +24 -3
- package/src/central/health/classify.ts +209 -0
- package/src/central/http.ts +380 -36
- package/src/central/recording/export.ts +57 -3
- package/src/central/recording/index.ts +236 -17
- package/src/central/recording/ingest.ts +11 -2
- package/src/central/recording/routes.ts +61 -7
- package/src/central/recording/schema.ts +68 -3
- package/src/central/recording/settings.ts +19 -5
- package/src/central/recording/store.ts +19 -0
- package/src/central/recording/taps.ts +45 -1
- package/src/central/registry.ts +34 -3
- package/src/central/speech/openai.ts +4 -4
- package/src/central/speech/openrouter.ts +4 -4
- package/src/central/speech/provider.ts +32 -1
- package/src/central/speech/service.ts +6 -5
- package/src/central/teams/budgets.ts +144 -0
- package/src/central/teams/hop-stalls.ts +80 -0
- package/src/central/teams/intake.ts +38 -6
- package/src/central/teams/involvement.ts +95 -0
- package/src/central/teams/item-decisions.ts +60 -0
- package/src/central/teams/item-eligibility.ts +35 -0
- package/src/central/teams/item-stalls.ts +87 -0
- package/src/central/teams/loops.ts +40 -0
- package/src/central/teams/policy.ts +10 -0
- package/src/central/teams/progress.ts +154 -0
- package/src/central/teams/routes.ts +2 -0
- package/src/central/teams/seats.ts +3 -3
- package/src/central/teams/staffing.ts +5 -0
- package/src/central/teams/store.ts +20 -1
- package/src/central/teams/validation.ts +16 -2
- package/src/central/workspaces/coordinator.ts +3 -1
- package/src/spoke/hub-tool.ts +26 -11
- package/web/dist/assets/{ArtifactPage-CsPjam8D.js → ArtifactPage-TbFJi59n.js} +1 -1
- package/web/dist/assets/{ArtifactPane-B9weO8L6.js → ArtifactPane-7D-M-Ich.js} +1 -1
- package/web/dist/assets/{ArtifactWorkbench-DzKhMs3l.js → ArtifactWorkbench-DlCLEjut.js} +1 -1
- package/web/dist/assets/{Diagram-CuRIffka.js → Diagram-PvOPetOI.js} +1 -1
- package/web/dist/assets/{Library-rv5DGtrm.js → Library-Cdi83pOV.js} +2 -2
- package/web/dist/assets/{ProfilesDestination-DUuQ9jmd.js → ProfilesDestination-cryC7P6j.js} +1 -1
- package/web/dist/assets/TeamsPage-B4xrlGhp.js +1 -0
- package/web/dist/assets/Watchtower-60B1JX_J.js +2 -0
- package/web/dist/assets/{artifacts-But87Jfo.js → artifacts-DlQEYmDl.js} +1 -1
- package/web/dist/assets/index-DA0gKIzv.js +33 -0
- package/web/dist/assets/{index-rW6LKQj1.css → index-Dyq-jBeE.css} +1 -1
- package/web/dist/assets/live-B2IfKm37.js +5 -0
- package/web/dist/assets/{live-BUofsQpz.css → live-De-kd3eW.css} +1 -1
- package/web/dist/assets/{pane-layout-BW0RdX5a.js → pane-layout-BAuu_pA3.js} +1 -1
- package/web/dist/assets/{renderer2d-DyANQ1WS.js → renderer2d-CsZnF3_o.js} +1 -1
- package/web/dist/assets/{renderer3d-vYYKFAJq.js → renderer3d-BIMA0wFn.js} +2 -2
- package/web/dist/assets/{safe-area-B2Bk__FK.js → safe-area-dtrnw9NF.js} +1 -1
- package/web/dist/assets/{silero-worker-e802lB5T.js → silero-worker-CC2etKS3.js} +1 -1
- package/web/dist/assets/{source-C-De8L3s.js → source-GYayxUxS.js} +1 -1
- package/web/dist/assets/{toast-CXTwTvDw.js → toast-q818mjUF.js} +1 -1
- package/web/dist/index.html +5 -5
- package/web/dist/assets/TeamsPage-D_17MkvN.js +0 -1
- package/web/dist/assets/Watchtower-DlrNOJ7X.js +0 -2
- package/web/dist/assets/index-bX8B6NLo.js +0 -33
- package/web/dist/assets/live-BGLA-peO.js +0 -5
|
@@ -67,7 +67,7 @@
|
|
|
67
67
|
// spent budget wait for the tick (no 0 ms timer loop).
|
|
68
68
|
import { CONCIERGE_PARTY, CONCIERGE_ROLE, type ConciergeSettings } from "../../../packages/protocol/src/concierge"
|
|
69
69
|
import { AgentError } from "../agent/errors"
|
|
70
|
-
import type
|
|
70
|
+
import { stateChanged, type AgentEvent, type AgentEvents, type EventRecord, type EventsSince } from "../agent/events"
|
|
71
71
|
import { updateVersioned } from "../agent/versioned"
|
|
72
72
|
import { realTimers } from "../backend/memory-bus"
|
|
73
73
|
import type { Clock, StateStore, Timers } from "../backend/types"
|
|
@@ -286,6 +286,8 @@ function classify(record: Pick<EventRecord, "event" | "family" | "audience">, ct
|
|
|
286
286
|
return line("routine", `${who} blocker_closed${blockerID ? ` ${blockerID}` : ""}`)
|
|
287
287
|
}
|
|
288
288
|
case "session.changed": {
|
|
289
|
+
// (SS-1) A change of health alone is no new state line (SS-4 adds the health line).
|
|
290
|
+
if (!stateChanged(d)) return undefined
|
|
289
291
|
const state = enumOf(d.state, STATES)
|
|
290
292
|
const k = `s:${id}`
|
|
291
293
|
if (state === "error") return line("priority", `${who} state:error`, { k, f: state, b: [`${root} state:error`] })
|
|
@@ -347,7 +349,7 @@ function overdueTurn(l: Line): Line {
|
|
|
347
349
|
const EARLY: ReadonlySet<string> = new Set(["blocker.opened", "message.changed", "launch.changed"])
|
|
348
350
|
|
|
349
351
|
/** Whether an append may be a priority line (an early pass): EARLY types, and a session going into error. */
|
|
350
|
-
const early = (event: AgentEvent): boolean => EARLY.has(event.type) || (event.type === "session.changed" && event.data?.state === "error")
|
|
352
|
+
const early = (event: AgentEvent): boolean => EARLY.has(event.type) || (event.type === "session.changed" && event.data?.state === "error" && stateChanged(event.data))
|
|
351
353
|
/** Whether an append may be a notable line (a turn done): a pass when its quiet window ends. */
|
|
352
354
|
const notable = (event: AgentEvent): boolean => event.type === "session.changed" && event.data?.state === "idle" && event.data?.prev === "working"
|
|
353
355
|
|
package/src/central/config.ts
CHANGED
|
@@ -8,13 +8,14 @@ import { hubEnv } from "../env"
|
|
|
8
8
|
import { isLoopbackHost } from "../loopback"
|
|
9
9
|
import { formatPattern, parsePattern } from "./agent/policy"
|
|
10
10
|
import type { SpeechQuotaOverrides } from "./agent/speech-routes"
|
|
11
|
-
import { SPEECH_RATE_LIMITS } from "../protocol"
|
|
11
|
+
import { HOP_CEILING_DEFAULT, SPEECH_RATE_LIMITS } from "../protocol"
|
|
12
12
|
import { APP_DIR, configBase, legacyPaths, migrateFirst, migrateStateDir, resolveHubStateDir, stateLegacyPaths } from "../migrate"
|
|
13
13
|
import { updateSpeech, type StoredSpeech } from "./speech/settings"
|
|
14
14
|
import { readContextCards, type ContextCardsConfig } from "./context-cards/config"
|
|
15
15
|
import { checkReleaseBase } from "./install/render"
|
|
16
16
|
import { cspMode, type CspMode } from "./csp"
|
|
17
17
|
import { parseDenyCidrs } from "./fetcher"
|
|
18
|
+
import { HOP_ABSOLUTE_MAX, HOP_CAP } from "./agent/message-limits"
|
|
18
19
|
|
|
19
20
|
export const GOOGLE_ISSUER = "https://accounts.google.com"
|
|
20
21
|
|
|
@@ -70,6 +71,14 @@ export interface CentralConfig {
|
|
|
70
71
|
readonly auth: AuthConfig
|
|
71
72
|
/** hub.json `agents.never`: permission patterns no agent may ever approve, for every user (design §5, §7). */
|
|
72
73
|
readonly agentsNever: readonly string[]
|
|
74
|
+
/** hub.json `agents.hopLimit`: global maximum agent-only hops before a wake is skipped. */
|
|
75
|
+
readonly hopLimit: number
|
|
76
|
+
/** (HC-1) hub.json `agents.hopCeiling` (default 100): the highest hop limit accepted. */
|
|
77
|
+
readonly hopCeiling: number
|
|
78
|
+
/** (HC-1) Whether hub.json set the ceiling (else the default). */
|
|
79
|
+
readonly hopCeilingSource?: "default" | "hub.json"
|
|
80
|
+
/** Which source supplied the hub-wide agent limits before console overrides. */
|
|
81
|
+
readonly agentLimitsSource?: "default" | "hub.json"
|
|
73
82
|
/** hub.json `agents.launches`: per-user launch caps, only those set (placement fills in its defaults). */
|
|
74
83
|
readonly launchCaps?: HubLaunches
|
|
75
84
|
/** hub.json `agents.webhooks.allowPrivate`: webhooks may target http, loopback, private and local hosts (admin opt-in). */
|
|
@@ -138,6 +147,9 @@ export interface HubLaunches {
|
|
|
138
147
|
export interface HubAgents {
|
|
139
148
|
/** Normalised `action[:glob]` patterns (see agent/policy.ts), duplicates dropped. */
|
|
140
149
|
readonly never?: readonly string[]
|
|
150
|
+
readonly hopLimit?: number
|
|
151
|
+
/** (HC-1) The highest hop limit the hub accepts (1–10,000, default 100); the console may override it. */
|
|
152
|
+
readonly hopCeiling?: number
|
|
141
153
|
readonly launches?: HubLaunches
|
|
142
154
|
readonly webhooks?: HubWebhooks
|
|
143
155
|
/** (AF2) Per-credential app speech quotas: any subset of SPEECH_RATE_LIMITS. */
|
|
@@ -169,7 +181,7 @@ type Env = Readonly<Record<string, string | undefined>>
|
|
|
169
181
|
|
|
170
182
|
const HUB_KEYS = new Set(["host", "port", "publicURL", "stateDir", "speech", "auth", "agents", "cluster", "contextCards"])
|
|
171
183
|
const CLUSTER_KEYS = ["database", "valkey"] as const
|
|
172
|
-
const AGENTS_KEYS = new Set(["never", "launches", "webhooks", "speechQuotas"])
|
|
184
|
+
const AGENTS_KEYS = new Set(["never", "launches", "webhooks", "speechQuotas", "hopLimit", "hopCeiling"])
|
|
173
185
|
/** The largest speech quota hub.json may set (counts and characters). */
|
|
174
186
|
const SPEECH_QUOTA_MAX = 1_000_000_000
|
|
175
187
|
const WEBHOOK_KEYS = new Set(["allowPrivate", "allowPrivateHosts"])
|
|
@@ -350,11 +362,16 @@ function readHubAgents(raw: unknown, file: string, warnings: string[], fail: (me
|
|
|
350
362
|
if (!isObj(raw)) fail("agents must be an object")
|
|
351
363
|
const input = raw as Record<string, unknown>
|
|
352
364
|
for (const key of Object.keys(input)) if (!AGENTS_KEYS.has(key)) warnings.push(`${file}: unknown key "agents.${key}" ignored`)
|
|
365
|
+
if (input.hopCeiling !== undefined && (!Number.isSafeInteger(input.hopCeiling) || (input.hopCeiling as number) < 1 || (input.hopCeiling as number) > HOP_ABSOLUTE_MAX))
|
|
366
|
+
fail(`agents.hopCeiling must be a whole number from 1 to ${HOP_ABSOLUTE_MAX}`)
|
|
367
|
+
const ceiling = (input.hopCeiling as number | undefined) ?? HOP_CEILING_DEFAULT
|
|
368
|
+
if (input.hopLimit !== undefined && (!Number.isSafeInteger(input.hopLimit) || (input.hopLimit as number) < 1 || (input.hopLimit as number) > ceiling))
|
|
369
|
+
fail(`agents.hopLimit must be a whole number from 1 to the hop ceiling (${ceiling}; agents.hopCeiling raises it)`)
|
|
353
370
|
const launches = input.launches === undefined ? undefined : readHubLaunches(input.launches, file, warnings, fail)
|
|
354
371
|
const never = input.never === undefined ? undefined : readNever(input.never, fail)
|
|
355
372
|
const webhooks = input.webhooks === undefined ? undefined : readHubWebhooks(input.webhooks, file, warnings, fail)
|
|
356
373
|
const speechQuotas = input.speechQuotas === undefined ? undefined : readSpeechQuotas(input.speechQuotas, file, warnings, fail)
|
|
357
|
-
return { ...(never ? { never } : {}), ...(launches ? { launches } : {}), ...(webhooks ? { webhooks } : {}), ...(speechQuotas ? { speechQuotas } : {}) }
|
|
374
|
+
return { ...(never ? { never } : {}), ...(input.hopLimit !== undefined ? { hopLimit: input.hopLimit as number } : {}), ...(input.hopCeiling !== undefined ? { hopCeiling: input.hopCeiling as number } : {}), ...(launches ? { launches } : {}), ...(webhooks ? { webhooks } : {}), ...(speechQuotas ? { speechQuotas } : {}) }
|
|
358
375
|
}
|
|
359
376
|
|
|
360
377
|
/** (AF2) `agents.speechQuotas`: {transcribe?: {perMinute?, perHour?}, speak?: {perMinute?, perHour?, charsPerHour?}}. */
|
|
@@ -601,6 +618,10 @@ export function centralConfig(rawEnv: Env = process.env, home: string = homedir(
|
|
|
601
618
|
...(file.speech ? { speech: file.speech } : {}),
|
|
602
619
|
auth,
|
|
603
620
|
agentsNever: [...(file.agents?.never ?? [])],
|
|
621
|
+
hopLimit: file.agents?.hopLimit ?? HOP_CAP,
|
|
622
|
+
hopCeiling: file.agents?.hopCeiling ?? HOP_CEILING_DEFAULT,
|
|
623
|
+
hopCeilingSource: file.agents?.hopCeiling !== undefined ? "hub.json" : "default",
|
|
624
|
+
agentLimitsSource: file.agents?.hopLimit !== undefined || file.agents?.launches !== undefined ? "hub.json" : "default",
|
|
604
625
|
...(file.agents?.launches ? { launchCaps: { ...file.agents.launches } } : {}),
|
|
605
626
|
...(file.agents?.webhooks?.allowPrivate === true ? { webhooksAllowPrivate: true as const } : {}),
|
|
606
627
|
...(file.agents?.webhooks?.allowPrivateHosts?.length ? { webhooksPrivateHosts: [...file.agents.webhooks.allowPrivateHosts] } : {}),
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
// (SS-1, plan v5ms8lgq) A session's health from its status board entry: why it is not making progress, next to the
|
|
2
|
+
// derived state (which stays "working" during a retry). Pure. Typed errors (OpenCode's errorType/errorStatus) decide
|
|
3
|
+
// first; message patterns cover the harnesses that send only text (Claude Code, Codex). An unknown failure is
|
|
4
|
+
// "failed", never a false quota.
|
|
5
|
+
import { providerFailureNeedsUser, type RetryInfo } from "../../status"
|
|
6
|
+
|
|
7
|
+
export type HealthKind = "ok" | "retrying" | "stalled" | "quota" | "auth" | "model_unavailable" | "context_overflow" | "network" | "failed"
|
|
8
|
+
export const HEALTH_KINDS: readonly HealthKind[] = ["ok", "retrying", "stalled", "quota", "auth", "model_unavailable", "context_overflow", "network", "failed"]
|
|
9
|
+
|
|
10
|
+
/** (Decision 3kj1d70k) A transient retry counts as stalled after this many attempts or this long since its first retry. */
|
|
11
|
+
export const STALL_ATTEMPTS = 3
|
|
12
|
+
export const STALL_MS = 5 * 60_000
|
|
13
|
+
export const HEALTH_DETAIL_MAX = 200
|
|
14
|
+
/** Only the start of a provider message is examined: causes come first, and the patterns' work stays bounded. */
|
|
15
|
+
export const CAUSE_TEXT_MAX = 2000
|
|
16
|
+
|
|
17
|
+
export interface SessionHealth {
|
|
18
|
+
readonly kind: HealthKind
|
|
19
|
+
/** When this kind was first seen (set by the registry, which keeps the history). */
|
|
20
|
+
readonly since?: number
|
|
21
|
+
/** quota: when the provider says it resets (epoch ms), when the message says. */
|
|
22
|
+
readonly resetAt?: number
|
|
23
|
+
/** While retrying: the provider retry's attempt number. */
|
|
24
|
+
readonly attempt?: number
|
|
25
|
+
/** The provider's message, sanitised (≤ 200 characters, secrets redacted). */
|
|
26
|
+
readonly detail?: string
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface HealthInput {
|
|
30
|
+
readonly state: string
|
|
31
|
+
/** state "error": the error's message (the status board's detail). */
|
|
32
|
+
readonly detail?: string
|
|
33
|
+
readonly retry?: RetryInfo
|
|
34
|
+
readonly errorType?: string
|
|
35
|
+
readonly errorStatus?: number
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** A cause that retrying will not fix: the session stays stuck until someone acts. */
|
|
39
|
+
type Hard = "quota" | "auth" | "model_unavailable" | "context_overflow"
|
|
40
|
+
type Cause = { readonly kind: Hard | "network"; readonly resetAt?: number }
|
|
41
|
+
|
|
42
|
+
// SS-2: Codex's UsageLimitReached ("You’ve hit your usage limit", "out of credits", "spend cap") and Claude Code's
|
|
43
|
+
// subscription limits ("You've hit your limit · resets 3pm", "5-hour limit reached") and billing_error.
|
|
44
|
+
const QUOTA_WORDS = /\bquota\b|usage limit|insufficient[_ ]quota|billing|credit balance|out of credits|spend(?:ing)? (?:limit|cap)|budget (?:limit|exceeded)|hit your limit|\b(?:\d+-hour|weekly|daily|session) limit reached/i
|
|
45
|
+
const RATE_LIMIT = /\b429\b|rate[ -]?limit|too many requests/i
|
|
46
|
+
const AUTH = /\b40[13]\b|unauthori[sz]ed|forbidden|invalid (?:x-)?api[ -_]?key|authentication|not (?:logged|signed) in|(?:token|credentials?) (?:has )?expired|permission denied|oauth[_ ]org[_ ]not[_ ]allowed|account[_ ]on[_ ]hold|account (?:is )?(?:suspended|disabled)|cloud[_ ]credential/i
|
|
47
|
+
const MODEL = /\bmodel\b[^.\n]{0,60}?\b(?:not found|does not exist|doesn't exist|is not available|unavailable|not supported|unsupported)\b|unknown model|no such model|model_not_found|invalid model|\b404\b[^\n]{0,160}\bmodel\b/i
|
|
48
|
+
const CONTEXT = /context (?:length|window)|maximum context|too many tokens|prompt is too long|input is too long|context_length_exceeded|exceeds? the (?:model's )?(?:context|token) limit/i
|
|
49
|
+
const NETWORK = /\bE(?:CONNRESET|CONNREFUSED|CONNABORTED|TIMEDOUT|NOTFOUND|AI_AGAIN|PIPE|HOSTUNREACH|NETUNREACH)\b|socket hang up|network error|fetch failed|connection (?:reset|refused|closed|failed|error)|error sending request|stream disconnected|timed out|\btimeout\b/i
|
|
50
|
+
/** A response the provider stopped by its content policy is no connection problem (Codex reports it as a disconnect). */
|
|
51
|
+
const CONTENT_POLICY = /content[_ ]filter|content[_ ]policy/i
|
|
52
|
+
/**
|
|
53
|
+
* (Review rule 3) Without a provider-error marker only an errno code may say "network": "timeout" alone is a build
|
|
54
|
+
* or a command as often as a provider, and client-library wording ("fetch failed", "error sending request for url",
|
|
55
|
+
* "stream disconnected") is generic. SS-9's typed harness errors carry the real network failures.
|
|
56
|
+
*/
|
|
57
|
+
const CONNECTION = /\bE(?:CONNRESET|CONNREFUSED|CONNABORTED|TIMEDOUT|AI_AGAIN|HOSTUNREACH|NETUNREACH)\b/
|
|
58
|
+
/** A reset hint further ahead than this is not believed (a year). */
|
|
59
|
+
const RESET_MAX_MS = 365 * 24 * 3_600_000
|
|
60
|
+
/**
|
|
61
|
+
* (SS-9) The typed errors Claude Code (StopFailure's error token) and Codex (codexErrorInfo, snake_case) report: a
|
|
62
|
+
* kind, or "words" for a marker whose message decides (a rate limit whose detail says "usage limit" is a quota). Any
|
|
63
|
+
* other claude.* / codex.* type is failed.
|
|
64
|
+
*/
|
|
65
|
+
const HARNESS_TYPES: Readonly<Record<string, Hard | "network" | "words">> = {
|
|
66
|
+
"claude.billing_error": "quota",
|
|
67
|
+
"claude.account_on_hold": "quota",
|
|
68
|
+
"claude.authentication_failed": "auth",
|
|
69
|
+
"claude.oauth_org_not_allowed": "auth",
|
|
70
|
+
"claude.cloud_credential_error": "auth",
|
|
71
|
+
"claude.model_not_found": "model_unavailable",
|
|
72
|
+
"claude.rate_limit": "words",
|
|
73
|
+
"claude.overloaded": "words",
|
|
74
|
+
"claude.invalid_request": "words",
|
|
75
|
+
"codex.usage_limit_exceeded": "quota",
|
|
76
|
+
"codex.unauthorized": "auth",
|
|
77
|
+
"codex.model_not_found": "model_unavailable",
|
|
78
|
+
"codex.context_window_exceeded": "context_overflow",
|
|
79
|
+
"codex.http_connection_failed": "network",
|
|
80
|
+
"codex.response_stream_connection_failed": "network",
|
|
81
|
+
"codex.response_stream_disconnected": "network",
|
|
82
|
+
"codex.rate_limit_exceeded": "words",
|
|
83
|
+
"codex.bad_request": "words",
|
|
84
|
+
}
|
|
85
|
+
const harnessType = (type: string | undefined): boolean => type !== undefined && (type.startsWith("claude.") || type.startsWith("codex."))
|
|
86
|
+
/** A rate limit whose reset is at least this far ahead is a quota, not a transient limit. */
|
|
87
|
+
const QUOTA_RESET_MS = 60 * 60_000
|
|
88
|
+
|
|
89
|
+
const UNIT_MS: Readonly<Record<string, number>> = { h: 3_600_000, m: 60_000, s: 1000 }
|
|
90
|
+
const RESET_HINT = /(?:reset(?:s|ting)?|try again|retry)\s+(?:after|in|at)\s+([^.;)\n]{1,60})/i
|
|
91
|
+
const DURATION = /(\d+(?:\.\d+)?)\s*(h(?:ours?|rs?)?|m(?:in(?:ute)?s?)?|s(?:ec(?:ond)?s?)?)\b/gi
|
|
92
|
+
|
|
93
|
+
/** When the provider says the limit resets: "reset after 118h", "resets in 2h 15m", "retry after 90 seconds", or an ISO time. */
|
|
94
|
+
export function parseResetAt(message: string | undefined, now: number): number | undefined {
|
|
95
|
+
const hint = message?.match(RESET_HINT)?.[1]
|
|
96
|
+
if (!hint) return undefined
|
|
97
|
+
const iso = hint.match(/\d{4}-\d{2}-\d{2}T[\d:.]+(?:Z|[+-]\d{2}:?\d{2})?/)?.[0]
|
|
98
|
+
let at: number
|
|
99
|
+
if (iso) at = Date.parse(iso)
|
|
100
|
+
else {
|
|
101
|
+
let ms = 0
|
|
102
|
+
for (const [, amount, unit] of hint.matchAll(DURATION)) ms += Number(amount) * UNIT_MS[unit![0]!.toLowerCase()]!
|
|
103
|
+
if (!(ms > 0)) return undefined
|
|
104
|
+
at = now + Math.round(ms)
|
|
105
|
+
}
|
|
106
|
+
// A valid date, ahead, and within a year: anything else is not believed (and could not be shown as a date).
|
|
107
|
+
return Number.isFinite(at) && at > now && at - now <= RESET_MAX_MS && !Number.isNaN(new Date(at).getTime()) ? at : undefined
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Secrets a provider message may echo back, each replaced by [redacted] (the label before it kept). */
|
|
111
|
+
const SECRETS: readonly (readonly [RegExp, string])[] = [
|
|
112
|
+
[/-----BEGIN [A-Z ]+-----[\s\S]*?(?:-----END [A-Z ]+-----|$)/g, "[redacted]"],
|
|
113
|
+
[/\b(Authorization|Proxy-Authorization)\s*[:=]\s*(?:Basic|Bearer|Token|Digest)?\s*[^\s,;]+/gi, "$1: [redacted]"],
|
|
114
|
+
[/\b(Bearer|Basic)\s+[A-Za-z0-9._~+\/=-]{4,}/gi, "$1 [redacted]"],
|
|
115
|
+
[/\b([a-z][a-z0-9+.-]*:\/\/)[^\s\/@:]+:[^\s\/@]+@/gi, "$1[redacted]@"],
|
|
116
|
+
[/\b(Set-Cookie|Cookie)\s*:[^\r\n]*/gi, "$1: [redacted]"],
|
|
117
|
+
[/(hooks\.slack\.com\/(?:services|workflows|triggers)\/)[^\s"'<>)]+/gi, "$1[redacted]"],
|
|
118
|
+
[/\b(bot)?\d{8,10}:[A-Za-z0-9_-]{30,}/g, "$1[redacted]"],
|
|
119
|
+
// A key-like name's value: to its closing quote when quoted (spaces included), else to the next separator.
|
|
120
|
+
[/\b([\w-]*(?:api[_-]?key|key|token|secret|password|passwd|pwd|credential|signature|sig)[\w-]*)(["']?\s*[:=]\s*)(["'])(?:(?!\3)[^\r\n])*\3?/gi, "$1$2$3[redacted]$3"],
|
|
121
|
+
[/\b([\w-]*(?:api[_-]?key|key|token|secret|password|passwd|pwd|credential|signature|sig)[\w-]*)(["']?\s*[:=]\s*)(?!["'])[^\s"'&,;)]+/gi, "$1$2[redacted]"],
|
|
122
|
+
[/\beyJ[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}/g, "[redacted]"],
|
|
123
|
+
[/\b(?:AKIA|ASIA|AGPA|AIDA|AROA)[A-Z0-9]{16}\b/g, "[redacted]"],
|
|
124
|
+
[/\bgithub_pat_[A-Za-z0-9_]{20,}/g, "[redacted]"],
|
|
125
|
+
[/\b(?:sk|pk|rk|hst|hat|hcs|hca|hcr|ghp|gho|ghs|ghu|ghr|glpat|xox[abpr])[-_][A-Za-z0-9_-]{8,}/g, "[redacted]"],
|
|
126
|
+
[/\bAIza[A-Za-z0-9_-]{20,}/g, "[redacted]"],
|
|
127
|
+
[/\b(?:npm|hf)_[A-Za-z0-9]{20,}/g, "[redacted]"],
|
|
128
|
+
[/\b[0-9a-fA-F]{40,}\b/g, "[redacted]"],
|
|
129
|
+
// Any other long base64-like run (40+ characters): a token with no recognisable prefix.
|
|
130
|
+
[/[A-Za-z0-9+\/_-]{40,}={0,2}/g, "[redacted]"],
|
|
131
|
+
]
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* The first line only, secrets redacted, no control or format characters, ≤ 200 characters. Prefer showing less:
|
|
135
|
+
* a provider message is untrusted and may echo credentials.
|
|
136
|
+
*/
|
|
137
|
+
export function sanitizeHealthDetail(text: string | undefined): string | undefined {
|
|
138
|
+
if (!text) return undefined
|
|
139
|
+
let redacted = text.slice(0, CAUSE_TEXT_MAX)
|
|
140
|
+
for (const [pattern, replacement] of SECRETS) redacted = redacted.replace(pattern, replacement)
|
|
141
|
+
const firstLine = redacted.split(/\r?\n/).find((line) => line.trim() !== "") ?? ""
|
|
142
|
+
const clean = firstLine
|
|
143
|
+
.replace(/[\p{Cc}\p{Cf}]/gu, " ")
|
|
144
|
+
.replace(/\s+/g, " ")
|
|
145
|
+
.trim()
|
|
146
|
+
if (!clean) return undefined
|
|
147
|
+
return clean.length > HEALTH_DETAIL_MAX ? `${clean.slice(0, HEALTH_DETAIL_MAX - 1)}…` : clean
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* What a typed or worded error means, when it means something specific; undefined for a transient or unknown one.
|
|
152
|
+
* `marked`: the error carries a provider-error marker (a provider.* type, an HTTP status, or it is a provider retry).
|
|
153
|
+
* Without one (Claude Code and Codex send text only until SS-9's typed token), the words alone never give a
|
|
154
|
+
* provider kind: only an unmistakable connection error says network, everything else is failed.
|
|
155
|
+
*/
|
|
156
|
+
function cause(type: string | undefined, status: number | undefined, message: string | undefined, now: number, marked: boolean): Cause | undefined {
|
|
157
|
+
const text = (message ?? "").slice(0, CAUSE_TEXT_MAX)
|
|
158
|
+
const resetAt = parseResetAt(text, now)
|
|
159
|
+
const quota = (): Cause => ({ kind: "quota", ...(resetAt !== undefined ? { resetAt } : {}) })
|
|
160
|
+
// Typed first (OpenCode). providerFailureNeedsUser's vocabulary: quota, free-limit and budget-like types are quotas.
|
|
161
|
+
if (type === "provider.auth") return { kind: "auth" }
|
|
162
|
+
if (type === "provider.no-route") return { kind: "model_unavailable" }
|
|
163
|
+
if (type === "provider.transport" || type === "provider.timeout") return { kind: "network" }
|
|
164
|
+
if (type && type !== "provider.rate-limit" && type !== "provider.unknown" && providerFailureNeedsUser(type)) return quota()
|
|
165
|
+
if (harnessType(type)) {
|
|
166
|
+
const mapped = HARNESS_TYPES[type!]
|
|
167
|
+
if (mapped === undefined) return undefined
|
|
168
|
+
if (mapped !== "words") return mapped === "quota" ? quota() : { kind: mapped }
|
|
169
|
+
} else if (type && !type.startsWith("provider.")) return undefined // A tool's or harness's own error: never a provider class.
|
|
170
|
+
if (!marked) return CONNECTION.test(text) && !CONTENT_POLICY.test(text) ? { kind: "network" } : undefined
|
|
171
|
+
// Then the words (and the status) of the message.
|
|
172
|
+
if (QUOTA_WORDS.test(text)) return quota()
|
|
173
|
+
if ((type === "provider.rate-limit" || status === 429 || RATE_LIMIT.test(text)) && resetAt !== undefined && resetAt - now >= QUOTA_RESET_MS) return quota()
|
|
174
|
+
if (CONTEXT.test(text)) return { kind: "context_overflow" }
|
|
175
|
+
if (CONTENT_POLICY.test(text)) return undefined
|
|
176
|
+
if (MODEL.test(text) || (status === 404 && /\bmodel\b/i.test(text))) return { kind: "model_unavailable" }
|
|
177
|
+
if (status === 401 || status === 403 || AUTH.test(text)) return { kind: "auth" }
|
|
178
|
+
if (NETWORK.test(text)) return { kind: "network" }
|
|
179
|
+
return undefined
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* The session's health now. `retrySince`: when the current provider retry was first seen (the registry keeps it).
|
|
184
|
+
* A retry for a hard cause (quota, auth, model, context) is that cause at once: retrying will not fix it (decision
|
|
185
|
+
* 3kj1d70k: quota stalls on its first retry). Any other retry is retrying, then stalled after STALL_ATTEMPTS attempts
|
|
186
|
+
* or STALL_MS. A failed run is its cause, else failed.
|
|
187
|
+
*/
|
|
188
|
+
export function classifyHealth(input: HealthInput, context: { readonly now: number; readonly retrySince?: number }): SessionHealth {
|
|
189
|
+
const { now } = context
|
|
190
|
+
const retry = input.state === "working" ? input.retry : undefined
|
|
191
|
+
if (retry) {
|
|
192
|
+
const attempt = typeof retry.attempt === "number" && retry.attempt > 0 ? { attempt: retry.attempt } : {}
|
|
193
|
+
// A provider retry is itself the marker: only a provider request is retried this way.
|
|
194
|
+
const found = cause(retry.errorType, retry.errorStatus, retry.message, now, true)
|
|
195
|
+
const detail = found?.kind === "auth" ? undefined : sanitizeHealthDetail(retry.message)
|
|
196
|
+
const base = { ...attempt, ...(detail ? { detail } : {}) }
|
|
197
|
+
if (found && found.kind !== "network") return { kind: found.kind, ...(found.resetAt !== undefined ? { resetAt: found.resetAt } : {}), ...base }
|
|
198
|
+
const stalled = (retry.attempt ?? 0) >= STALL_ATTEMPTS || (context.retrySince !== undefined && now - context.retrySince >= STALL_MS)
|
|
199
|
+
return { kind: stalled ? "stalled" : "retrying", ...base }
|
|
200
|
+
}
|
|
201
|
+
if (input.state === "error") {
|
|
202
|
+
const marked = (input.errorType?.startsWith("provider.") ?? false) || harnessType(input.errorType) || input.errorStatus !== undefined
|
|
203
|
+
const found = cause(input.errorType, input.errorStatus, input.detail, now, marked)
|
|
204
|
+
// An auth failure keeps no text at all: the kind says enough, and its message is the likeliest to echo a key.
|
|
205
|
+
const detail = found?.kind === "auth" ? undefined : sanitizeHealthDetail(input.detail)
|
|
206
|
+
return { kind: found?.kind ?? "failed", ...(found?.resetAt !== undefined ? { resetAt: found.resetAt } : {}), ...(detail ? { detail } : {}) }
|
|
207
|
+
}
|
|
208
|
+
return { kind: "ok" }
|
|
209
|
+
}
|