@crouton-kit/crouter 0.3.40 → 0.3.42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/dist/build-root.js +1 -0
  2. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/provider-rotation.test.ts +472 -9
  3. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +284 -76
  4. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/strip-skills-docs.ts +31 -24
  5. package/dist/builtin-pi-packages/pi-crtr-extensions/lib/subscription-state.ts +52 -15
  6. package/dist/clients/attach/attach-cmd.js +720 -747
  7. package/dist/clients/attach/chat-view.js +11 -0
  8. package/dist/clients/attach/view-socket.d.ts +8 -1
  9. package/dist/clients/attach/view-socket.js +15 -1
  10. package/dist/clients/web/server.js +5 -28
  11. package/dist/clients/web/web-cmd.js +1 -1
  12. package/dist/commands/__tests__/revive-now-gate.test.js +60 -0
  13. package/dist/commands/canvas-rebuild-index.js +5 -5
  14. package/dist/commands/memory/shared.d.ts +7 -3
  15. package/dist/commands/memory/shared.js +35 -5
  16. package/dist/commands/memory/write.js +5 -3
  17. package/dist/commands/node-snapshot.js +9 -1
  18. package/dist/commands/node.js +37 -17
  19. package/dist/commands/push.js +8 -0
  20. package/dist/commands/revive.d.ts +10 -0
  21. package/dist/commands/revive.js +28 -10
  22. package/dist/commands/sys/__tests__/setup-core.test.js +19 -0
  23. package/dist/commands/sys/doctor.js +1 -1
  24. package/dist/commands/sys/setup-core.js +3 -2
  25. package/dist/commands/sys/setup.js +1 -1
  26. package/dist/commands/worktree.d.ts +2 -0
  27. package/dist/commands/worktree.js +94 -0
  28. package/dist/core/__tests__/boot.test.js +4 -4
  29. package/dist/core/__tests__/canvas.test.js +19 -7
  30. package/dist/core/__tests__/child-followup.test.js +15 -5
  31. package/dist/core/__tests__/daemon-boot.test.js +6 -1
  32. package/dist/core/__tests__/daemon-wedge.test.js +18 -1
  33. package/dist/core/__tests__/fault-classifier.test.js +30 -0
  34. package/dist/core/__tests__/fixtures/fake-engine.d.ts +13 -0
  35. package/dist/core/__tests__/fixtures/fake-engine.js +20 -0
  36. package/dist/core/__tests__/full/spike-harness.test.js +8 -5
  37. package/dist/core/__tests__/grace-clock.test.js +18 -2
  38. package/dist/core/__tests__/host-teardown-process-group.test.js +465 -0
  39. package/dist/core/__tests__/prune-to-limit.test.js +14 -0
  40. package/dist/core/__tests__/review-model-floor.test.js +32 -0
  41. package/dist/core/__tests__/revive.test.js +18 -4
  42. package/dist/core/__tests__/session-cycles.test.js +77 -0
  43. package/dist/core/__tests__/worktree.test.js +85 -0
  44. package/dist/core/canvas/boot.js +12 -7
  45. package/dist/core/canvas/canvas.d.ts +27 -8
  46. package/dist/core/canvas/canvas.js +54 -26
  47. package/dist/core/canvas/db.js +14 -0
  48. package/dist/core/canvas/history.js +1 -0
  49. package/dist/core/canvas/paths.d.ts +10 -9
  50. package/dist/core/canvas/paths.js +10 -9
  51. package/dist/core/canvas/pid.d.ts +155 -1
  52. package/dist/core/canvas/pid.js +306 -1
  53. package/dist/core/canvas/status-glyph.d.ts +7 -0
  54. package/dist/core/canvas/status-glyph.js +10 -1
  55. package/dist/core/canvas/types.d.ts +34 -0
  56. package/dist/core/fault-classifier.js +5 -1
  57. package/dist/core/profiles/select.d.ts +4 -2
  58. package/dist/core/profiles/select.js +30 -4
  59. package/dist/core/runtime/bearings.js +4 -0
  60. package/dist/core/runtime/branded-host.d.ts +7 -0
  61. package/dist/core/runtime/branded-host.js +44 -17
  62. package/dist/core/runtime/broker-sdk.js +28 -68
  63. package/dist/core/runtime/broker.js +35 -4
  64. package/dist/core/runtime/host.d.ts +3 -3
  65. package/dist/core/runtime/host.js +148 -33
  66. package/dist/core/runtime/launch.d.ts +15 -15
  67. package/dist/core/runtime/launch.js +58 -4
  68. package/dist/core/runtime/naming.js +3 -2
  69. package/dist/core/runtime/nodes.d.ts +3 -1
  70. package/dist/core/runtime/nodes.js +1 -0
  71. package/dist/core/runtime/pi-cli.d.ts +2 -0
  72. package/dist/core/runtime/pi-cli.js +51 -0
  73. package/dist/core/runtime/placement.d.ts +1 -1
  74. package/dist/core/runtime/placement.js +5 -2
  75. package/dist/core/runtime/recap.js +2 -1
  76. package/dist/core/runtime/revive.d.ts +21 -15
  77. package/dist/core/runtime/revive.js +87 -52
  78. package/dist/core/runtime/session-cycles.d.ts +30 -0
  79. package/dist/core/runtime/session-cycles.js +77 -0
  80. package/dist/core/runtime/spawn.d.ts +4 -0
  81. package/dist/core/runtime/spawn.js +152 -100
  82. package/dist/core/runtime/tmux.d.ts +7 -10
  83. package/dist/core/runtime/tmux.js +9 -11
  84. package/dist/core/worktree.d.ts +35 -0
  85. package/dist/core/worktree.js +158 -0
  86. package/dist/daemon/crtrd.d.ts +17 -8
  87. package/dist/daemon/crtrd.js +191 -40
  88. package/dist/index.d.ts +3 -0
  89. package/dist/index.js +7 -0
  90. package/dist/pi-extensions/canvas-context-intro.d.ts +1 -0
  91. package/dist/pi-extensions/canvas-context-intro.js +34 -23
  92. package/dist/pi-extensions/canvas-stophook.js +11 -5
  93. package/dist/web-client/assets/{index-CbO8L0mN.js → index-B00YpRQ1.js} +20 -20
  94. package/dist/web-client/assets/index-DrkcvANq.css +2 -0
  95. package/dist/web-client/index.html +2 -2
  96. package/docs/compat/hearth-crtr-v1.md +191 -0
  97. package/docs/public-api.md +75 -0
  98. package/package.json +3 -4
  99. package/dist/core/__tests__/hearth-bootstrap.test.js +0 -136
  100. package/dist/core/hearth/__tests__/model-auth-guest.test.js +0 -151
  101. package/dist/core/hearth/config.d.ts +0 -3
  102. package/dist/core/hearth/config.js +0 -108
  103. package/dist/core/hearth/guest-env.d.ts +0 -9
  104. package/dist/core/hearth/guest-env.js +0 -27
  105. package/dist/core/hearth/index.d.ts +0 -4
  106. package/dist/core/hearth/index.js +0 -4
  107. package/dist/core/hearth/model-auth-guest.d.ts +0 -8
  108. package/dist/core/hearth/model-auth-guest.js +0 -430
  109. package/dist/core/hearth/provider.d.ts +0 -36
  110. package/dist/core/hearth/provider.js +0 -10
  111. package/dist/core/hearth/providers/__tests__/sweep-and-release.test.js +0 -362
  112. package/dist/core/hearth/providers/blaxel-bootstrap.d.ts +0 -12
  113. package/dist/core/hearth/providers/blaxel-bootstrap.js +0 -147
  114. package/dist/core/hearth/providers/blaxel-home.d.ts +0 -60
  115. package/dist/core/hearth/providers/blaxel-home.js +0 -405
  116. package/dist/core/hearth/providers/blaxel.d.ts +0 -36
  117. package/dist/core/hearth/providers/blaxel.js +0 -364
  118. package/dist/core/hearth/providers/types.d.ts +0 -93
  119. package/dist/core/hearth/types.d.ts +0 -155
  120. package/dist/hearth/control-plane/__tests__/error-serialization.test.js +0 -29
  121. package/dist/hearth/control-plane/__tests__/node-message.test.js +0 -60
  122. package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.d.ts +0 -1
  123. package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.js +0 -44
  124. package/dist/hearth/control-plane/__tests__/rate-limit-recurrence.test.d.ts +0 -1
  125. package/dist/hearth/control-plane/__tests__/rate-limit-recurrence.test.js +0 -49
  126. package/dist/hearth/control-plane/__tests__/relay-security.test.d.ts +0 -1
  127. package/dist/hearth/control-plane/__tests__/relay-security.test.js +0 -314
  128. package/dist/hearth/control-plane/__tests__/scheduler-scan-loop.test.d.ts +0 -1
  129. package/dist/hearth/control-plane/__tests__/scheduler-scan-loop.test.js +0 -133
  130. package/dist/hearth/control-plane/__tests__/trigger-delivery.test.d.ts +0 -1
  131. package/dist/hearth/control-plane/__tests__/trigger-delivery.test.js +0 -170
  132. package/dist/hearth/control-plane/__tests__/wake-roll.test.d.ts +0 -1
  133. package/dist/hearth/control-plane/__tests__/wake-roll.test.js +0 -230
  134. package/dist/hearth/control-plane/__tests__/webhook-ingress.test.d.ts +0 -1
  135. package/dist/hearth/control-plane/__tests__/webhook-ingress.test.js +0 -167
  136. package/dist/hearth/control-plane/config.d.ts +0 -21
  137. package/dist/hearth/control-plane/config.js +0 -77
  138. package/dist/hearth/control-plane/db.d.ts +0 -30
  139. package/dist/hearth/control-plane/db.js +0 -561
  140. package/dist/hearth/control-plane/hearth-target.d.ts +0 -23
  141. package/dist/hearth/control-plane/hearth-target.js +0 -68
  142. package/dist/hearth/control-plane/ingress/rate-limit.d.ts +0 -24
  143. package/dist/hearth/control-plane/ingress/rate-limit.js +0 -100
  144. package/dist/hearth/control-plane/ingress/route-store.d.ts +0 -31
  145. package/dist/hearth/control-plane/ingress/route-store.js +0 -61
  146. package/dist/hearth/control-plane/ingress/webhook-delivery-store.d.ts +0 -41
  147. package/dist/hearth/control-plane/ingress/webhook-delivery-store.js +0 -69
  148. package/dist/hearth/control-plane/ingress/webhook-route.d.ts +0 -55
  149. package/dist/hearth/control-plane/ingress/webhook-route.js +0 -285
  150. package/dist/hearth/control-plane/main.d.ts +0 -1
  151. package/dist/hearth/control-plane/main.js +0 -88
  152. package/dist/hearth/control-plane/node-message.d.ts +0 -31
  153. package/dist/hearth/control-plane/node-message.js +0 -98
  154. package/dist/hearth/control-plane/register.d.ts +0 -15
  155. package/dist/hearth/control-plane/register.js +0 -34
  156. package/dist/hearth/control-plane/registry.d.ts +0 -22
  157. package/dist/hearth/control-plane/registry.js +0 -168
  158. package/dist/hearth/control-plane/relay.d.ts +0 -44
  159. package/dist/hearth/control-plane/relay.js +0 -711
  160. package/dist/hearth/control-plane/scheduler/fire-store.d.ts +0 -36
  161. package/dist/hearth/control-plane/scheduler/fire-store.js +0 -73
  162. package/dist/hearth/control-plane/scheduler/recurrence.d.ts +0 -7
  163. package/dist/hearth/control-plane/scheduler/recurrence.js +0 -58
  164. package/dist/hearth/control-plane/scheduler/scan-loop.d.ts +0 -38
  165. package/dist/hearth/control-plane/scheduler/scan-loop.js +0 -138
  166. package/dist/hearth/control-plane/scheduler/schedule-store.d.ts +0 -32
  167. package/dist/hearth/control-plane/scheduler/schedule-store.js +0 -66
  168. package/dist/hearth/control-plane/secrets.d.ts +0 -31
  169. package/dist/hearth/control-plane/secrets.js +0 -134
  170. package/dist/hearth/control-plane/server.d.ts +0 -27
  171. package/dist/hearth/control-plane/server.js +0 -482
  172. package/dist/hearth/control-plane/serving.d.ts +0 -15
  173. package/dist/hearth/control-plane/serving.js +0 -106
  174. package/dist/hearth/control-plane/session.d.ts +0 -68
  175. package/dist/hearth/control-plane/session.js +0 -273
  176. package/dist/hearth/control-plane/triggers/acl.d.ts +0 -14
  177. package/dist/hearth/control-plane/triggers/acl.js +0 -52
  178. package/dist/hearth/control-plane/triggers/audit-store.d.ts +0 -38
  179. package/dist/hearth/control-plane/triggers/audit-store.js +0 -79
  180. package/dist/hearth/control-plane/triggers/deliver.d.ts +0 -43
  181. package/dist/hearth/control-plane/triggers/deliver.js +0 -76
  182. package/dist/hearth/control-plane/triggers/envelope.d.ts +0 -29
  183. package/dist/hearth/control-plane/triggers/envelope.js +0 -38
  184. package/dist/hearth/control-plane/types.d.ts +0 -86
  185. package/dist/hearth/control-plane/types.js +0 -1
  186. package/dist/hearth/control-plane/wake.d.ts +0 -86
  187. package/dist/hearth/control-plane/wake.js +0 -550
  188. package/dist/web-client/assets/index-DwO46Cs5.css +0 -2
  189. /package/dist/{core/__tests__/hearth-bootstrap.test.d.ts → commands/__tests__/revive-now-gate.test.d.ts} +0 -0
  190. /package/dist/{core/hearth/__tests__/model-auth-guest.test.d.ts → commands/sys/__tests__/setup-core.test.d.ts} +0 -0
  191. /package/dist/core/{hearth/providers/__tests__/sweep-and-release.test.d.ts → __tests__/fault-classifier.test.d.ts} +0 -0
  192. /package/dist/core/{hearth/providers/types.js → __tests__/host-teardown-process-group.test.d.ts} +0 -0
  193. /package/dist/core/{hearth/types.js → __tests__/review-model-floor.test.d.ts} +0 -0
  194. /package/dist/{hearth/control-plane/__tests__/error-serialization.test.d.ts → core/__tests__/session-cycles.test.d.ts} +0 -0
  195. /package/dist/{hearth/control-plane/__tests__/node-message.test.d.ts → core/__tests__/worktree.test.d.ts} +0 -0
@@ -40,30 +40,84 @@ import {
40
40
 
41
41
  const DEFAULT_RATE_LIMIT_BACKOFF_MS = 5 * 60 * 1000;
42
42
  const INVALID_REFRESH_TOKEN_BACKOFF_MS = 30 * 24 * 60 * 60 * 1000;
43
+ // A sole/authed-but-cooling-down provider is worth sleeping out rather than failing the
44
+ // turn (#123) -- but only up to a bound: INVALID_REFRESH_TOKEN_BACKOFF_MS (30 days) also
45
+ // shows up as a "cooldown" on a subscription entry, and that one genuinely needs reauth,
46
+ // not a wait. Cap at the same window as our own default rate-limit backoff.
47
+ const SOLE_PROVIDER_MAX_WAIT_MS = DEFAULT_RATE_LIMIT_BACKOFF_MS;
48
+ // Short in-place retry budget for a transient network/5xx blip on the SAME credential
49
+ // (#122): a couple of quick retries, no cooldown, no rotation to another credential or
50
+ // provider. Once this budget is exhausted the turn ends with an actionable error.
51
+ const TRANSIENT_RETRY_BACKOFFS_MS = [250, 750];
43
52
 
44
53
  let runtimeContext: ExtensionContext | undefined;
45
54
  let extensionAPI: ExtensionAPI | undefined;
46
55
 
47
56
  type AttemptResult = {
48
- result: "success" | "retry" | "fatal";
57
+ result: "success" | "rate_limited" | "fatal";
49
58
  retryAfterMs?: number;
50
59
  attemptAt: number;
60
+ classification?: ProviderErrorClassification;
51
61
  };
52
62
 
53
- function isFallbackLike(error: unknown): boolean {
63
+ // #122/#124: a genuine rate limit (HTTP 429, or 503 WITH a real retry-after header) must
64
+ // cool the credential down and rotate to the next one; a transient network/5xx blip
65
+ // (connection reset, socket errors, timeouts, plain 500/502/504, or 503 with NO
66
+ // retry-after) is not evidence the credential itself is bad -- it gets a short in-place
67
+ // retry on the same credential, never a cooldown. Everything else is fatal. Bare
68
+ // `usage`/`quota` substrings were dropped from the message match: they are false-positive
69
+ // magnets (e.g. "usage" appears in normal assistant text) that used to cool down healthy
70
+ // subscriptions for 5 minutes on any transient blip.
71
+ type ProviderErrorClassification = {
72
+ kind: "rate_limit" | "transient" | "fatal";
73
+ status?: number;
74
+ reason: string;
75
+ hadRetryAfterHeader: boolean;
76
+ };
77
+
78
+ const TRANSIENT_MESSAGE_PATTERN =
79
+ /connection|network|fetch failed|socket|econnreset|econnrefused|enotfound|etimedout|timed? out|timeout|overloaded|service.?unavailable|temporarily unavailable/i;
80
+ const RATE_LIMIT_MESSAGE_PATTERN = /\b429\b|rate.?limit|too many requests/i;
81
+ // Precise terminal usage-limit signals (as opposed to the bare `usage`/`quota` substrings
82
+ // dropped above): pi-ai's Codex adapter surfaces `usage_limit_reached` / `usage_not_included`
83
+ // as the friendly "You have hit your ChatGPT usage limit..." message, often on a non-429
84
+ // status, so the status-code check above misses it. These are specific enough not to false-
85
+ // positive on ordinary assistant text mentioning "usage".
86
+ const USAGE_LIMIT_MESSAGE_PATTERN = /usage[ _]limit|usage_not_included|usagelimiterror|insufficient_quota|quota exceeded/i;
87
+
88
+ function classifyProviderError(error: unknown, observed: { status?: number; retryAfterMs?: number }): ProviderErrorClassification {
54
89
  const value = error as { status?: unknown; errorMessage?: unknown; message?: unknown } | undefined;
55
- const status = typeof value?.status === "number" ? value.status : Number.NaN;
90
+ const errorStatus = typeof value?.status === "number" ? value.status : undefined;
91
+ const status = observed.status ?? errorStatus;
56
92
  const message = [value?.errorMessage, value?.message, error instanceof Error ? error.message : ""]
57
93
  .filter((part): part is string => typeof part === "string" && part.length > 0)
58
94
  .join(" ");
59
- return (
60
- status === 429 ||
61
- status === 500 ||
62
- status === 502 ||
63
- status === 503 ||
64
- status === 504 ||
65
- /\b429\b|rate.?limit|too many requests|overloaded|service.?unavailable|temporarily unavailable|upstream|quota|usage|connection|network|fetch failed|socket|econnreset|econnrefused|enotfound|etimedout|timed? out|timeout|request was aborted/i.test(message)
66
- );
95
+ const hadRetryAfterHeader = observed.retryAfterMs !== undefined;
96
+
97
+ if (status === 429) return { kind: "rate_limit", status, reason: "http 429", hadRetryAfterHeader };
98
+ if (status === 503 && hadRetryAfterHeader) return { kind: "rate_limit", status, reason: "http 503 with retry-after", hadRetryAfterHeader };
99
+ if (status === 500 || status === 502 || status === 503 || status === 504) {
100
+ return { kind: "transient", status, reason: `http ${status}`, hadRetryAfterHeader };
101
+ }
102
+ const transientMatch = message.match(TRANSIENT_MESSAGE_PATTERN);
103
+ if (transientMatch) return { kind: "transient", status, reason: transientMatch[0].toLowerCase(), hadRetryAfterHeader };
104
+ const rateLimitMatch = message.match(RATE_LIMIT_MESSAGE_PATTERN);
105
+ if (rateLimitMatch) return { kind: "rate_limit", status, reason: rateLimitMatch[0].toLowerCase(), hadRetryAfterHeader };
106
+ const usageLimitMatch = message.match(USAGE_LIMIT_MESSAGE_PATTERN);
107
+ if (usageLimitMatch) return { kind: "rate_limit", status, reason: usageLimitMatch[0].toLowerCase(), hadRetryAfterHeader };
108
+ return { kind: "fatal", status, reason: "unclassified", hadRetryAfterHeader };
109
+ }
110
+
111
+ async function defaultSleep(ms: number): Promise<void> {
112
+ await new Promise<void>((resolve) => setTimeout(resolve, ms));
113
+ }
114
+
115
+ // Indirection so tests can inject deterministic waits without real timers.
116
+ let activeSleep: typeof defaultSleep = defaultSleep;
117
+
118
+ /** Test-only seam: override the sleep implementation used for in-place/wait-out retries. Pass undefined to restore the default. */
119
+ export function __setSleepForTest(fn: typeof defaultSleep | undefined): void {
120
+ activeSleep = fn ?? defaultSleep;
67
121
  }
68
122
 
69
123
  function isInvalidRefreshTokenError(error: unknown): boolean {
@@ -75,6 +129,27 @@ function isInvalidRefreshTokenError(error: unknown): boolean {
75
129
  return status === 400 && /invalid_grant|refresh token .*invalid|refresh token .*not found/i.test(message);
76
130
  }
77
131
 
132
+ // A user/host abort (ESC, a critical-tier interrupt, a torn-down window) is NOT a provider
133
+ // failure — it must never cool down a credential or rotate providers. The upstream pi-ai
134
+ // providers set `reason: "aborted"` / `stopReason: "aborted"` (and throw "Request was aborted")
135
+ // ONLY when the request's AbortSignal fired, so those markers alone are a reliable abort
136
+ // signal; we do NOT require the signal to be threaded back through our own options (it may not
137
+ // be), which is what made an ESC drain both provider pools.
138
+ function isUserAbortEvent(event: Parameters<AssistantMessageEventStream["push"]>[0], signal: AbortSignal | undefined): boolean {
139
+ if (signal?.aborted) return true;
140
+ if (event.type !== "error") return false;
141
+ return event.reason === "aborted" || event.error.stopReason === "aborted" || /request was aborted/i.test(event.error.errorMessage ?? "");
142
+ }
143
+
144
+ function isUserAbortError(error: unknown, signal: AbortSignal | undefined): boolean {
145
+ if (signal?.aborted) return true;
146
+ const value = error as { name?: unknown; errorMessage?: unknown; message?: unknown } | undefined;
147
+ const message = [value?.errorMessage, value?.message, error instanceof Error ? error.message : String(error ?? "")]
148
+ .filter((part): part is string => typeof part === "string" && part.length > 0)
149
+ .join(" ");
150
+ return (error instanceof Error && /abort/i.test(error.name)) || /request was aborted|operation was aborted|the user aborted/i.test(message);
151
+ }
152
+
78
153
  function setStatus(text: string): void {
79
154
  runtimeContext?.ui.setStatus("provider-rotation", text);
80
155
  }
@@ -209,6 +284,10 @@ export function __setStreamForProviderForTest(fn: typeof defaultStreamForProvide
209
284
  activeStreamForProvider = fn ?? defaultStreamForProvider;
210
285
  }
211
286
 
287
+ // Drives ONE credential, retrying transient (non-rate-limit) failures in place on the
288
+ // SAME credential per #122, and reports back a classified result for #124's logging.
289
+ // A genuine rate limit ends the attempt immediately (the caller cools the credential down
290
+ // and rotates); a transient blip gets a short in-place retry budget before giving up.
212
291
  async function streamProviderAttempt(
213
292
  providerId: ManagedProviderId,
214
293
  model: Model<any>,
@@ -217,84 +296,162 @@ async function streamProviderAttempt(
217
296
  credential: SubscriptionCredential,
218
297
  outer: AssistantMessageEventStream,
219
298
  ): Promise<AttemptResult> {
220
- const attemptAt = Date.now();
221
- markSubscriptionAttempt(providerId, credential.label, attemptAt);
222
- let requestedRetryAfterMs: number | undefined;
223
- const inner = streamForProvider(model, context, {
224
- ...options,
225
- apiKey: credential.access,
226
- onResponse: async (response, responseModel) => {
227
- if (response.status === 429 || response.status === 503) {
228
- requestedRetryAfterMs = retryAfterMs(response.headers);
229
- }
230
- await options?.onResponse?.(response, responseModel);
231
- },
232
- });
299
+ for (let attempt = 1; ; attempt++) {
300
+ const attemptAt = Date.now();
301
+ markSubscriptionAttempt(providerId, credential.label, attemptAt);
302
+ let observedStatus: number | undefined;
303
+ let requestedRetryAfterMs: number | undefined;
304
+ const inner = streamForProvider(model, context, {
305
+ ...options,
306
+ apiKey: credential.access,
307
+ onResponse: async (response, responseModel) => {
308
+ observedStatus = response.status;
309
+ if (response.status === 429 || response.status === 503) {
310
+ requestedRetryAfterMs = retryAfterMs(response.headers);
311
+ }
312
+ await options?.onResponse?.(response, responseModel);
313
+ },
314
+ });
233
315
 
234
- const bufferedEvents: Parameters<AssistantMessageEventStream["push"]>[0][] = [];
235
- let sawContent = false;
316
+ const bufferedEvents: Parameters<AssistantMessageEventStream["push"]>[0][] = [];
317
+ let sawContent = false;
318
+ let classification: ProviderErrorClassification | undefined;
236
319
 
237
- const flush = (): void => {
238
- for (const event of bufferedEvents) outer.push(event);
239
- bufferedEvents.length = 0;
240
- };
320
+ const flush = (): void => {
321
+ for (const event of bufferedEvents) outer.push(event);
322
+ bufferedEvents.length = 0;
323
+ };
241
324
 
242
- try {
243
- for await (const event of inner) {
244
- if (event.type === "error") {
245
- if (isFallbackLike(event.error)) {
246
- return { result: sawContent ? "fatal" : "retry", retryAfterMs: requestedRetryAfterMs, attemptAt };
325
+ try {
326
+ for await (const event of inner) {
327
+ if (event.type === "error") {
328
+ if (isUserAbortEvent(event, options?.signal)) {
329
+ bufferedEvents.push(event);
330
+ flush();
331
+ return { result: "fatal", attemptAt };
332
+ }
333
+ classification = classifyProviderError(event.error, { status: observedStatus, retryAfterMs: requestedRetryAfterMs });
334
+ if (classification.kind === "fatal" || sawContent) {
335
+ bufferedEvents.push(event);
336
+ flush();
337
+ return { result: "fatal", attemptAt, classification };
338
+ }
339
+ break;
247
340
  }
341
+ if (event.type !== "start" && event.type !== "done") sawContent = true;
248
342
  bufferedEvents.push(event);
343
+ }
344
+ } catch (error) {
345
+ if (isUserAbortError(error, options?.signal)) {
346
+ bufferedEvents.push({
347
+ type: "error",
348
+ reason: "aborted",
349
+ error: { ...toAssistantError(model, error instanceof Error ? error.message : String(error)), stopReason: "aborted" },
350
+ });
249
351
  flush();
250
352
  return { result: "fatal", attemptAt };
251
353
  }
252
- if (event.type !== "start" && event.type !== "done") sawContent = true;
253
- bufferedEvents.push(event);
354
+ classification = classifyProviderError(error, { status: observedStatus, retryAfterMs: requestedRetryAfterMs });
355
+ if (classification.kind === "fatal" || sawContent) {
356
+ bufferedEvents.push({
357
+ type: "error",
358
+ reason: "error",
359
+ error: toAssistantError(model, error instanceof Error ? error.message : String(error)),
360
+ });
361
+ flush();
362
+ return { result: "fatal", attemptAt, classification };
363
+ }
254
364
  }
255
- } catch (error) {
256
- if (isFallbackLike(error)) {
257
- return { result: sawContent ? "fatal" : "retry", retryAfterMs: requestedRetryAfterMs, attemptAt };
365
+
366
+ if (!classification && requestedRetryAfterMs !== undefined && !sawContent) {
367
+ // Some providers signal a 429/503 purely via the response header without ever raising
368
+ // a stream "error" event; treat that the same as an explicit rate-limit classification.
369
+ classification = { kind: "rate_limit", status: observedStatus, reason: "retry-after header on success path", hadRetryAfterHeader: true };
370
+ }
371
+
372
+ if (!classification) {
373
+ flush();
374
+ return { result: "success", attemptAt };
258
375
  }
259
- bufferedEvents.push({
260
- type: "error",
261
- reason: "error",
262
- error: toAssistantError(model, error instanceof Error ? error.message : String(error)),
263
- });
264
- flush();
265
- return { result: "fatal", attemptAt };
266
- }
267
376
 
268
- if (requestedRetryAfterMs !== undefined && !sawContent) {
269
- return { result: "retry", retryAfterMs: requestedRetryAfterMs, attemptAt };
377
+ if (classification.kind === "rate_limit") {
378
+ return { result: "rate_limited", retryAfterMs: requestedRetryAfterMs, attemptAt, classification };
379
+ }
380
+
381
+ // Transient: retry the SAME credential/provider in place -- no cooldown, no rotation.
382
+ const backoffMs = TRANSIENT_RETRY_BACKOFFS_MS[attempt - 1];
383
+ if (backoffMs === undefined) {
384
+ // Exhausted the short in-place retry budget: surface a clear terminal error rather
385
+ // than spinning forever or silently rotating a healthy credential away.
386
+ bufferedEvents.push({
387
+ type: "error",
388
+ reason: "error",
389
+ error: toAssistantError(
390
+ model,
391
+ `${getProviderLabel(providerId)} "${credential.label}" hit a transient error after ${attempt} attempts (${classification.reason})`,
392
+ ),
393
+ });
394
+ flush();
395
+ return { result: "fatal", attemptAt, classification };
396
+ }
397
+ logRotation(
398
+ `${getProviderLabel(providerId)} subscription "${credential.label}" hit a transient error (${classification.reason}, status=${classification.status ?? "n/a"}); retrying same credential in ${backoffMs}ms (attempt ${attempt + 1})`,
399
+ );
400
+ await activeSleep(backoffMs);
270
401
  }
402
+ }
271
403
 
272
- flush();
273
- return { result: "success", attemptAt };
404
+ function emitTerminalError(stream: AssistantMessageEventStream, model: Model<any>, message: string): void {
405
+ stream.push({ type: "error", reason: "error", error: toAssistantError(model, message) });
406
+ stream.end();
407
+ }
408
+
409
+ // #123: gates cross-provider fallback on the target actually being usable right now --
410
+ // authenticated (its subscription pool has at least one credential, even if currently
411
+ // cooling down) AND registered in the model registry -- rather than blindly switching the
412
+ // session model to a provider nobody logged into. Returns undefined when there is nothing
413
+ // viable to switch to (config has no ladder rung, the target was already tried this turn,
414
+ // the target has zero credentials, or its resolved model isn't in the registry).
415
+ function resolveViableFallback(
416
+ providerId: ManagedProviderId,
417
+ modelId: string | undefined,
418
+ attempted: Set<ManagedProviderId>,
419
+ ): FallbackTarget | undefined {
420
+ const target = resolveFallbackTarget(providerId, modelId, readRotationConfig(), extensionAPI?.getThinkingLevel?.());
421
+ if (!target || attempted.has(target.providerId)) return undefined;
422
+ if (readSubscriptionPool(target.providerId).length === 0) return undefined;
423
+ if (!runtimeContext?.modelRegistry.find(target.providerId, target.modelId)) return undefined;
424
+ return target;
274
425
  }
275
426
 
276
- async function switchToFallbackModel(originalModel: { providerId: ManagedProviderId; modelId: string }): Promise<FallbackTarget> {
427
+ // Switches the session model to an already-viability-checked fallback target. On the rare
428
+ // race where the switch itself still fails (e.g. `setModel` rejects it), emits a graceful,
429
+ // actionable terminal error on `stream` instead of throwing a generic "failed to switch".
430
+ async function switchToFallbackModel(
431
+ stream: AssistantMessageEventStream,
432
+ model: Model<any>,
433
+ originalModel: { providerId: ManagedProviderId; modelId: string },
434
+ ): Promise<FallbackTarget | undefined> {
277
435
  const ctx = runtimeContext;
278
436
  if (!extensionAPI || !ctx) {
279
- throw new Error(`Provider fallback unavailable: runtime not initialized for ${originalModel.providerId}/${originalModel.modelId}`);
437
+ emitTerminalError(stream, model, `Provider fallback unavailable: runtime not initialized for ${originalModel.providerId}/${originalModel.modelId}`);
438
+ return undefined;
280
439
  }
281
440
  const target = await switchToFallbackIfPossible(extensionAPI, ctx, originalModel);
282
441
  if (!target) {
283
- const resolved = resolveFallbackTarget(originalModel.providerId, originalModel.modelId, readRotationConfig(), extensionAPI.getThinkingLevel?.());
284
- throw new Error(`Provider fallback unavailable: failed to switch to ${resolved.label}`);
442
+ emitTerminalError(stream, model, `Provider fallback unavailable: failed to switch models for ${getProviderLabel(originalModel.providerId)}`);
443
+ return undefined;
285
444
  }
286
445
  return target;
287
446
  }
288
447
 
289
- function emitTerminalError(stream: AssistantMessageEventStream, model: Model<any>, message: string): void {
290
- stream.push({ type: "error", reason: "error", error: toAssistantError(model, message) });
291
- stream.end();
292
- }
293
-
294
448
  // Drives one managed provider's pool into `stream`; on exhaustion it switches the
295
449
  // session model to the cross-provider fallback and CONTINUES the same turn on it,
296
450
  // rather than failing the turn with an error. `attempted` guards anthropic<->codex
297
- // ping-pong: once both pools are dry we emit a single terminal error.
451
+ // ping-pong: once both pools are dry (or unauthenticated) we emit a single terminal error.
452
+ // If neither a fallback nor more credentials are available but the sole/current provider
453
+ // is merely cooling down, we sleep out the shortest cooldown and retry it in place (#123)
454
+ // instead of failing the turn.
298
455
  async function runManagedProvider(
299
456
  model: Model<any>,
300
457
  context: Context,
@@ -304,7 +461,8 @@ async function runManagedProvider(
304
461
  ): Promise<void> {
305
462
  const providerId = model.provider as ManagedProviderId;
306
463
  attempted.add(providerId);
307
- const available = readSubscriptionPool(providerId).filter((entry) => !entry.rateLimitedUntil || entry.rateLimitedUntil <= Date.now());
464
+ const rawPool = readSubscriptionPool(providerId);
465
+ const available = rawPool.filter((entry) => !entry.rateLimitedUntil || entry.rateLimitedUntil <= Date.now());
308
466
 
309
467
  for (const credential of available) {
310
468
  let freshCredential: SubscriptionCredential;
@@ -344,39 +502,89 @@ async function runManagedProvider(
344
502
  return;
345
503
  }
346
504
 
347
- if (result.result === "retry") {
505
+ if (result.result === "rate_limited") {
348
506
  const cooldownMs = result.retryAfterMs ?? DEFAULT_RATE_LIMIT_BACKOFF_MS;
349
507
  markSubscriptionRateLimited(providerId, freshCredential.label, cooldownMs, result.attemptAt);
350
- logRotation(`${getProviderLabel(providerId)} subscription "${freshCredential.label}" cooled down for ${Math.ceil(cooldownMs / 1000)}s; trying next credential`);
508
+ // #124: make a misclassification obvious at a glance -- status, matched reason, and
509
+ // whether a real retry-after header drove the cooldown vs. our default backoff.
510
+ const retryAfterPart = result.retryAfterMs !== undefined ? `retry-after=${Math.ceil(result.retryAfterMs / 1000)}s` : "no retry-after header (default backoff)";
511
+ logRotation(
512
+ `${getProviderLabel(providerId)} subscription "${freshCredential.label}" rate-limited (status=${result.classification?.status ?? "n/a"}, reason=${result.classification?.reason ?? "unclassified"}, ${retryAfterPart}); cooling down for ${Math.ceil(cooldownMs / 1000)}s, trying next credential`,
513
+ );
351
514
  continue;
352
515
  }
353
516
 
354
517
  // Fatal (or partial content already streamed): cannot safely switch providers mid-turn.
518
+ if (result.classification) {
519
+ logRotation(
520
+ `${getProviderLabel(providerId)} subscription "${freshCredential.label}" failed fatally (status=${result.classification.status ?? "n/a"}, reason=${result.classification.reason})`,
521
+ );
522
+ }
355
523
  stream.end();
356
524
  return;
357
525
  }
358
526
 
359
- // Pool exhausted switch the session model to the fallback and continue this turn on it.
527
+ // Pool exhausted (or this provider was never authenticated at all) figure out what to
528
+ // do next: switch to a viable cross-provider fallback, wait out a sole/authed provider's
529
+ // cooldown, or surface an actionable terminal error. Re-read the pool here rather than
530
+ // reusing the pre-loop `rawPool` snapshot: the credential loop above mutates cooldowns
531
+ // in place via `markSubscriptionRateLimited` (e.g. a sole credential that just got a
532
+ // fresh 429), and the stale snapshot would still show it as available, making the
533
+ // wait-out math below think there's nothing to wait for and wrongly declare exhaustion.
534
+ const currentPool = readSubscriptionPool(providerId);
360
535
  const originalModel = getCurrentModelRef(model);
361
536
  if (!originalModel) throw new Error(`Provider fallback unavailable: missing current model`);
362
537
 
363
- const plannedTarget = resolveFallbackTarget(originalModel.providerId, originalModel.modelId, readRotationConfig(), extensionAPI?.getThinkingLevel?.());
364
- if (attempted.has(plannedTarget.providerId)) {
365
- emitTerminalError(stream, model, `All managed provider pools exhausted (tried ${[...attempted].join(", ")})`);
538
+ const viableTarget = resolveViableFallback(originalModel.providerId, originalModel.modelId, attempted);
539
+ if (viableTarget) {
540
+ const target = await switchToFallbackModel(stream, model, originalModel);
541
+ if (!target) return; // terminal error already emitted
542
+ logRotation(`${getProviderLabel(providerId)} pool exhausted -> ${target.label}`);
543
+ setStatus(`${getProviderLabel(providerId)} -> ${target.label}`);
544
+
545
+ const fallbackModel = runtimeContext?.modelRegistry.find(target.providerId, target.modelId);
546
+ if (!fallbackModel) {
547
+ emitTerminalError(stream, model, `Provider fallback unavailable: ${target.label} not in model registry`);
548
+ return;
549
+ }
550
+
551
+ await runManagedProvider(fallbackModel, context, options, stream, attempted);
366
552
  return;
367
553
  }
368
554
 
369
- const target = await switchToFallbackModel(originalModel);
370
- logRotation(`${getProviderLabel(providerId)} pool exhausted -> ${target.label}`);
371
- setStatus(`${getProviderLabel(providerId)} -> ${target.label}`);
555
+ const label = getProviderLabel(providerId);
556
+ const triedList = [...attempted].join(", ");
372
557
 
373
- const fallbackModel = runtimeContext?.modelRegistry.find(target.providerId, target.modelId);
374
- if (!fallbackModel) {
375
- emitTerminalError(stream, model, `Provider fallback unavailable: ${target.label} not in model registry`);
558
+ // #123: this provider was never authenticated at all (no subscription pool entries ever
559
+ // existed for it) -- there's nothing to wait out, so this is a config/setup problem, not
560
+ // a transient one.
561
+ if (currentPool.length === 0) {
562
+ emitTerminalError(stream, model, `${label} not authenticated — run \`crtr sys setup\` (or /provider-sub ${providerId} add) (tried ${triedList})`);
376
563
  return;
377
564
  }
378
565
 
379
- await runManagedProvider(fallbackModel, context, options, stream, attempted);
566
+ // Authenticated but every credential is cooling down, and no authenticated fallback
567
+ // exists right now: sleep out the SHORTEST cooldown and retry the same provider instead
568
+ // of failing the turn -- this is the common "single subscription, briefly rate-limited"
569
+ // case, not a permanent outage.
570
+ const soonestRateLimitedUntil = Math.min(...currentPool.map((entry) => entry.rateLimitedUntil || 0));
571
+ const waitMs = soonestRateLimitedUntil - Date.now();
572
+
573
+ if (waitMs > 0 && waitMs <= SOLE_PROVIDER_MAX_WAIT_MS) {
574
+ logRotation(`${label} cooling down for ${Math.ceil(waitMs / 1000)}s with no authenticated fallback; waiting to retry the same provider`);
575
+ setStatus(`${label} cooling down for ${Math.ceil(waitMs / 1000)}s, retrying...`);
576
+ await activeSleep(waitMs);
577
+ await runManagedProvider(model, context, options, stream, attempted);
578
+ return;
579
+ }
580
+
581
+ emitTerminalError(
582
+ stream,
583
+ model,
584
+ waitMs > SOLE_PROVIDER_MAX_WAIT_MS
585
+ ? `${label} cooling down for ${Math.ceil(waitMs / 1000)}s — too long to wait automatically; authenticate a fallback provider with /provider-sub, or try again later (tried ${triedList})`
586
+ : `All managed provider pools exhausted (tried ${triedList})`,
587
+ );
380
588
  }
381
589
 
382
590
  function streamManagedProvider(model: Model<any>, context: Context, options?: SimpleStreamOptions): AssistantMessageEventStream {
@@ -1,39 +1,46 @@
1
1
  /**
2
- * Strip pi-skill-system references from the system prompt
2
+ * Strip pi's "Pi documentation" block from the system prompt.
3
3
  *
4
- * Pi core hardcodes a "Pi documentation" block in its system prompt that maps
5
- * topics to doc files, including `skills (docs/skills.md)`, and lists "skills"
6
- * among the topics it covers. That mapping makes the agent extrapolate a
7
- * topic -> docs/<topic>.md pattern (e.g. a nonexistent docs/rules.md) and
8
- * surfaces pi's internal skill system when we don't want it referenced.
4
+ * Pi core hardcodes a "Pi documentation (read only when the user asks about pi
5
+ * itself, its SDK, extensions, themes, skills, or TUI):" block a marker line
6
+ * plus a bullet list pointing at pi's README / docs / examples under
7
+ * node_modules. It causes two problems we fix by removing the whole block:
9
8
  *
10
- * This extension removes those references at runtime in `before_agent_start`,
11
- * so it survives `pi update` / `brew upgrade` (which would overwrite any edit
12
- * to the installed dist). It only touches the two skill references; everything
13
- * else in the block is left intact, and it degrades to a no-op if pi changes
14
- * the wording.
9
+ * 1. It maps topics to doc files (`skills (docs/skills.md)`, etc.), which makes
10
+ * the agent extrapolate a topic -> docs/<topic>.md pattern and surface pi's
11
+ * internal skill system when we don't want it referenced.
12
+ * 2. It is a THIRD-PARTY TELL. When authenticating a Claude subscription over
13
+ * OAuth, Anthropic inspects the request and — if it doesn't look like genuine
14
+ * Claude Code — routes it to "extra usage" instead of the plan, returning
15
+ * HTTP 400 ("Third-party apps now draw from your extra usage..."). The
16
+ * companion `pi-claude-oauth-adapter` normally strips this block for exactly
17
+ * that reason, but it matches the marker string INCLUDING the word "skills,"
18
+ * — and the earlier version of THIS extension mangled that marker (it removed
19
+ * just the "skills" word), so the adapter's match failed and the block leaked
20
+ * to the wire → every subscription turn 400'd. Removing the entire block here
21
+ * is self-contained: it fixes the leak regardless of the adapter's marker,
22
+ * load order, or presence, and subsumes the old skill-reference removal.
15
23
  *
16
- * Install: drop in ~/.pi/agent/extensions/ (global). Takes effect next session.
24
+ * Runs in `before_agent_start` so it survives `pi update` / `brew upgrade`
25
+ * (which would overwrite any edit to the installed dist), and degrades to a
26
+ * no-op if pi changes the wording.
17
27
  */
18
28
 
19
29
  import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
20
30
 
21
- function stripSkillRefs(prompt: string): string {
22
- if (!prompt.includes("docs/skills.md")) return prompt; // nothing to do / wording changed
23
- let p = prompt;
24
- // 1. The topic -> doc mapping entry, with surrounding comma cleanup so the
25
- // list reads cleanly whether the entry is mid-list or trailing.
26
- p = p.replace(/,[ \t]*skills \(docs\/skills\.md\)/g, "");
27
- p = p.replace(/skills \(docs\/skills\.md\)[ \t]*,[ \t]*/g, "");
28
- // 2. The word "skills" in the "...asks about pi itself, its SDK, ... skills,
29
- // or TUI" enumeration.
30
- p = p.replace(/,[ \t]*skills(?=[ \t]*,[ \t]*or TUI)/g, "");
31
- return p;
31
+ // Matches the whole block: the "Pi documentation (read only ...):" marker line
32
+ // (with or without the "skills," enumeration) plus its trailing "- ..." bullets,
33
+ // and any leading blank lines so the surrounding prompt closes up cleanly.
34
+ const PI_DOCS_BLOCK = /\n*^Pi documentation \(read only[^\n]*:(?:\n[ \t]*-[^\n]*)*/m;
35
+
36
+ function stripPiDocsBlock(prompt: string): string {
37
+ if (!prompt.includes("Pi documentation (read only")) return prompt; // no-op / wording changed
38
+ return prompt.replace(PI_DOCS_BLOCK, "").trimEnd() + "\n";
32
39
  }
33
40
 
34
41
  export default function stripSkillsDocs(pi: ExtensionAPI) {
35
42
  pi.on("before_agent_start", async (event) => {
36
- const stripped = stripSkillRefs(event.systemPrompt);
43
+ const stripped = stripPiDocsBlock(event.systemPrompt);
37
44
  if (stripped === event.systemPrompt) return;
38
45
  return { systemPrompt: stripped };
39
46
  });