@crouton-kit/crouter 0.3.40 → 0.3.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-root.js +1 -0
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/provider-rotation.test.ts +472 -9
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +284 -76
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/strip-skills-docs.ts +31 -24
- package/dist/builtin-pi-packages/pi-crtr-extensions/lib/subscription-state.ts +52 -15
- package/dist/clients/attach/attach-cmd.js +720 -747
- package/dist/clients/attach/chat-view.js +11 -0
- package/dist/clients/attach/view-socket.d.ts +8 -1
- package/dist/clients/attach/view-socket.js +15 -1
- package/dist/clients/web/server.js +5 -28
- package/dist/clients/web/web-cmd.js +1 -1
- package/dist/commands/__tests__/revive-now-gate.test.js +60 -0
- package/dist/commands/canvas-rebuild-index.js +5 -5
- package/dist/commands/memory/shared.d.ts +7 -3
- package/dist/commands/memory/shared.js +35 -5
- package/dist/commands/memory/write.js +5 -3
- package/dist/commands/node-snapshot.js +9 -1
- package/dist/commands/node.js +37 -17
- package/dist/commands/push.js +8 -0
- package/dist/commands/revive.d.ts +10 -0
- package/dist/commands/revive.js +28 -10
- package/dist/commands/sys/__tests__/setup-core.test.js +19 -0
- package/dist/commands/sys/doctor.js +1 -1
- package/dist/commands/sys/setup-core.js +3 -2
- package/dist/commands/sys/setup.js +1 -1
- package/dist/commands/worktree.d.ts +2 -0
- package/dist/commands/worktree.js +94 -0
- package/dist/core/__tests__/boot.test.js +4 -4
- package/dist/core/__tests__/canvas.test.js +19 -7
- package/dist/core/__tests__/child-followup.test.js +15 -5
- package/dist/core/__tests__/daemon-boot.test.js +6 -1
- package/dist/core/__tests__/daemon-wedge.test.js +18 -1
- package/dist/core/__tests__/fault-classifier.test.js +30 -0
- package/dist/core/__tests__/fixtures/fake-engine.d.ts +13 -0
- package/dist/core/__tests__/fixtures/fake-engine.js +20 -0
- package/dist/core/__tests__/full/spike-harness.test.js +8 -5
- package/dist/core/__tests__/grace-clock.test.js +18 -2
- package/dist/core/__tests__/host-teardown-process-group.test.js +465 -0
- package/dist/core/__tests__/prune-to-limit.test.js +14 -0
- package/dist/core/__tests__/review-model-floor.test.js +32 -0
- package/dist/core/__tests__/revive.test.js +18 -4
- package/dist/core/__tests__/session-cycles.test.js +77 -0
- package/dist/core/__tests__/worktree.test.js +85 -0
- package/dist/core/canvas/boot.js +12 -7
- package/dist/core/canvas/canvas.d.ts +27 -8
- package/dist/core/canvas/canvas.js +54 -26
- package/dist/core/canvas/db.js +14 -0
- package/dist/core/canvas/history.js +1 -0
- package/dist/core/canvas/paths.d.ts +10 -9
- package/dist/core/canvas/paths.js +10 -9
- package/dist/core/canvas/pid.d.ts +155 -1
- package/dist/core/canvas/pid.js +306 -1
- package/dist/core/canvas/status-glyph.d.ts +7 -0
- package/dist/core/canvas/status-glyph.js +10 -1
- package/dist/core/canvas/types.d.ts +34 -0
- package/dist/core/fault-classifier.js +5 -1
- package/dist/core/profiles/select.d.ts +4 -2
- package/dist/core/profiles/select.js +30 -4
- package/dist/core/runtime/bearings.js +4 -0
- package/dist/core/runtime/branded-host.d.ts +7 -0
- package/dist/core/runtime/branded-host.js +44 -17
- package/dist/core/runtime/broker-sdk.js +28 -68
- package/dist/core/runtime/broker.js +35 -4
- package/dist/core/runtime/host.d.ts +3 -3
- package/dist/core/runtime/host.js +148 -33
- package/dist/core/runtime/launch.d.ts +15 -15
- package/dist/core/runtime/launch.js +58 -4
- package/dist/core/runtime/naming.js +3 -2
- package/dist/core/runtime/nodes.d.ts +3 -1
- package/dist/core/runtime/nodes.js +1 -0
- package/dist/core/runtime/pi-cli.d.ts +2 -0
- package/dist/core/runtime/pi-cli.js +51 -0
- package/dist/core/runtime/placement.d.ts +1 -1
- package/dist/core/runtime/placement.js +5 -2
- package/dist/core/runtime/recap.js +2 -1
- package/dist/core/runtime/revive.d.ts +21 -15
- package/dist/core/runtime/revive.js +87 -52
- package/dist/core/runtime/session-cycles.d.ts +30 -0
- package/dist/core/runtime/session-cycles.js +77 -0
- package/dist/core/runtime/spawn.d.ts +4 -0
- package/dist/core/runtime/spawn.js +152 -100
- package/dist/core/runtime/tmux.d.ts +7 -10
- package/dist/core/runtime/tmux.js +9 -11
- package/dist/core/worktree.d.ts +35 -0
- package/dist/core/worktree.js +158 -0
- package/dist/daemon/crtrd.d.ts +17 -8
- package/dist/daemon/crtrd.js +191 -40
- package/dist/index.d.ts +3 -0
- package/dist/index.js +7 -0
- package/dist/pi-extensions/canvas-context-intro.d.ts +1 -0
- package/dist/pi-extensions/canvas-context-intro.js +34 -23
- package/dist/pi-extensions/canvas-stophook.js +11 -5
- package/dist/web-client/assets/{index-CbO8L0mN.js → index-B00YpRQ1.js} +20 -20
- package/dist/web-client/assets/index-DrkcvANq.css +2 -0
- package/dist/web-client/index.html +2 -2
- package/docs/compat/hearth-crtr-v1.md +191 -0
- package/docs/public-api.md +75 -0
- package/package.json +3 -4
- package/dist/core/__tests__/hearth-bootstrap.test.js +0 -136
- package/dist/core/hearth/__tests__/model-auth-guest.test.js +0 -151
- package/dist/core/hearth/config.d.ts +0 -3
- package/dist/core/hearth/config.js +0 -108
- package/dist/core/hearth/guest-env.d.ts +0 -9
- package/dist/core/hearth/guest-env.js +0 -27
- package/dist/core/hearth/index.d.ts +0 -4
- package/dist/core/hearth/index.js +0 -4
- package/dist/core/hearth/model-auth-guest.d.ts +0 -8
- package/dist/core/hearth/model-auth-guest.js +0 -430
- package/dist/core/hearth/provider.d.ts +0 -36
- package/dist/core/hearth/provider.js +0 -10
- package/dist/core/hearth/providers/__tests__/sweep-and-release.test.js +0 -362
- package/dist/core/hearth/providers/blaxel-bootstrap.d.ts +0 -12
- package/dist/core/hearth/providers/blaxel-bootstrap.js +0 -147
- package/dist/core/hearth/providers/blaxel-home.d.ts +0 -60
- package/dist/core/hearth/providers/blaxel-home.js +0 -405
- package/dist/core/hearth/providers/blaxel.d.ts +0 -36
- package/dist/core/hearth/providers/blaxel.js +0 -364
- package/dist/core/hearth/providers/types.d.ts +0 -93
- package/dist/core/hearth/types.d.ts +0 -155
- package/dist/hearth/control-plane/__tests__/error-serialization.test.js +0 -29
- package/dist/hearth/control-plane/__tests__/node-message.test.js +0 -60
- package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.js +0 -44
- package/dist/hearth/control-plane/__tests__/rate-limit-recurrence.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/rate-limit-recurrence.test.js +0 -49
- package/dist/hearth/control-plane/__tests__/relay-security.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/relay-security.test.js +0 -314
- package/dist/hearth/control-plane/__tests__/scheduler-scan-loop.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/scheduler-scan-loop.test.js +0 -133
- package/dist/hearth/control-plane/__tests__/trigger-delivery.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/trigger-delivery.test.js +0 -170
- package/dist/hearth/control-plane/__tests__/wake-roll.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/wake-roll.test.js +0 -230
- package/dist/hearth/control-plane/__tests__/webhook-ingress.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/webhook-ingress.test.js +0 -167
- package/dist/hearth/control-plane/config.d.ts +0 -21
- package/dist/hearth/control-plane/config.js +0 -77
- package/dist/hearth/control-plane/db.d.ts +0 -30
- package/dist/hearth/control-plane/db.js +0 -561
- package/dist/hearth/control-plane/hearth-target.d.ts +0 -23
- package/dist/hearth/control-plane/hearth-target.js +0 -68
- package/dist/hearth/control-plane/ingress/rate-limit.d.ts +0 -24
- package/dist/hearth/control-plane/ingress/rate-limit.js +0 -100
- package/dist/hearth/control-plane/ingress/route-store.d.ts +0 -31
- package/dist/hearth/control-plane/ingress/route-store.js +0 -61
- package/dist/hearth/control-plane/ingress/webhook-delivery-store.d.ts +0 -41
- package/dist/hearth/control-plane/ingress/webhook-delivery-store.js +0 -69
- package/dist/hearth/control-plane/ingress/webhook-route.d.ts +0 -55
- package/dist/hearth/control-plane/ingress/webhook-route.js +0 -285
- package/dist/hearth/control-plane/main.d.ts +0 -1
- package/dist/hearth/control-plane/main.js +0 -88
- package/dist/hearth/control-plane/node-message.d.ts +0 -31
- package/dist/hearth/control-plane/node-message.js +0 -98
- package/dist/hearth/control-plane/register.d.ts +0 -15
- package/dist/hearth/control-plane/register.js +0 -34
- package/dist/hearth/control-plane/registry.d.ts +0 -22
- package/dist/hearth/control-plane/registry.js +0 -168
- package/dist/hearth/control-plane/relay.d.ts +0 -44
- package/dist/hearth/control-plane/relay.js +0 -711
- package/dist/hearth/control-plane/scheduler/fire-store.d.ts +0 -36
- package/dist/hearth/control-plane/scheduler/fire-store.js +0 -73
- package/dist/hearth/control-plane/scheduler/recurrence.d.ts +0 -7
- package/dist/hearth/control-plane/scheduler/recurrence.js +0 -58
- package/dist/hearth/control-plane/scheduler/scan-loop.d.ts +0 -38
- package/dist/hearth/control-plane/scheduler/scan-loop.js +0 -138
- package/dist/hearth/control-plane/scheduler/schedule-store.d.ts +0 -32
- package/dist/hearth/control-plane/scheduler/schedule-store.js +0 -66
- package/dist/hearth/control-plane/secrets.d.ts +0 -31
- package/dist/hearth/control-plane/secrets.js +0 -134
- package/dist/hearth/control-plane/server.d.ts +0 -27
- package/dist/hearth/control-plane/server.js +0 -482
- package/dist/hearth/control-plane/serving.d.ts +0 -15
- package/dist/hearth/control-plane/serving.js +0 -106
- package/dist/hearth/control-plane/session.d.ts +0 -68
- package/dist/hearth/control-plane/session.js +0 -273
- package/dist/hearth/control-plane/triggers/acl.d.ts +0 -14
- package/dist/hearth/control-plane/triggers/acl.js +0 -52
- package/dist/hearth/control-plane/triggers/audit-store.d.ts +0 -38
- package/dist/hearth/control-plane/triggers/audit-store.js +0 -79
- package/dist/hearth/control-plane/triggers/deliver.d.ts +0 -43
- package/dist/hearth/control-plane/triggers/deliver.js +0 -76
- package/dist/hearth/control-plane/triggers/envelope.d.ts +0 -29
- package/dist/hearth/control-plane/triggers/envelope.js +0 -38
- package/dist/hearth/control-plane/types.d.ts +0 -86
- package/dist/hearth/control-plane/types.js +0 -1
- package/dist/hearth/control-plane/wake.d.ts +0 -86
- package/dist/hearth/control-plane/wake.js +0 -550
- package/dist/web-client/assets/index-DwO46Cs5.css +0 -2
- /package/dist/{core/__tests__/hearth-bootstrap.test.d.ts → commands/__tests__/revive-now-gate.test.d.ts} +0 -0
- /package/dist/{core/hearth/__tests__/model-auth-guest.test.d.ts → commands/sys/__tests__/setup-core.test.d.ts} +0 -0
- /package/dist/core/{hearth/providers/__tests__/sweep-and-release.test.d.ts → __tests__/fault-classifier.test.d.ts} +0 -0
- /package/dist/core/{hearth/providers/types.js → __tests__/host-teardown-process-group.test.d.ts} +0 -0
- /package/dist/core/{hearth/types.js → __tests__/review-model-floor.test.d.ts} +0 -0
- /package/dist/{hearth/control-plane/__tests__/error-serialization.test.d.ts → core/__tests__/session-cycles.test.d.ts} +0 -0
- /package/dist/{hearth/control-plane/__tests__/node-message.test.d.ts → core/__tests__/worktree.test.d.ts} +0 -0
|
@@ -40,30 +40,84 @@ import {
|
|
|
40
40
|
|
|
41
41
|
const DEFAULT_RATE_LIMIT_BACKOFF_MS = 5 * 60 * 1000;
|
|
42
42
|
const INVALID_REFRESH_TOKEN_BACKOFF_MS = 30 * 24 * 60 * 60 * 1000;
|
|
43
|
+
// A sole/authed-but-cooling-down provider is worth sleeping out rather than failing the
|
|
44
|
+
// turn (#123) -- but only up to a bound: INVALID_REFRESH_TOKEN_BACKOFF_MS (30 days) also
|
|
45
|
+
// shows up as a "cooldown" on a subscription entry, and that one genuinely needs reauth,
|
|
46
|
+
// not a wait. Cap at the same window as our own default rate-limit backoff.
|
|
47
|
+
const SOLE_PROVIDER_MAX_WAIT_MS = DEFAULT_RATE_LIMIT_BACKOFF_MS;
|
|
48
|
+
// Short in-place retry budget for a transient network/5xx blip on the SAME credential
|
|
49
|
+
// (#122): a couple of quick retries, no cooldown, no rotation to another credential or
|
|
50
|
+
// provider. Once this budget is exhausted the turn ends with an actionable error.
|
|
51
|
+
const TRANSIENT_RETRY_BACKOFFS_MS = [250, 750];
|
|
43
52
|
|
|
44
53
|
let runtimeContext: ExtensionContext | undefined;
|
|
45
54
|
let extensionAPI: ExtensionAPI | undefined;
|
|
46
55
|
|
|
47
56
|
type AttemptResult = {
|
|
48
|
-
result: "success" | "
|
|
57
|
+
result: "success" | "rate_limited" | "fatal";
|
|
49
58
|
retryAfterMs?: number;
|
|
50
59
|
attemptAt: number;
|
|
60
|
+
classification?: ProviderErrorClassification;
|
|
51
61
|
};
|
|
52
62
|
|
|
53
|
-
|
|
63
|
+
// #122/#124: a genuine rate limit (HTTP 429, or 503 WITH a real retry-after header) must
|
|
64
|
+
// cool the credential down and rotate to the next one; a transient network/5xx blip
|
|
65
|
+
// (connection reset, socket errors, timeouts, plain 500/502/504, or 503 with NO
|
|
66
|
+
// retry-after) is not evidence the credential itself is bad -- it gets a short in-place
|
|
67
|
+
// retry on the same credential, never a cooldown. Everything else is fatal. Bare
|
|
68
|
+
// `usage`/`quota` substrings were dropped from the message match: they are false-positive
|
|
69
|
+
// magnets (e.g. "usage" appears in normal assistant text) that used to cool down healthy
|
|
70
|
+
// subscriptions for 5 minutes on any transient blip.
|
|
71
|
+
type ProviderErrorClassification = {
|
|
72
|
+
kind: "rate_limit" | "transient" | "fatal";
|
|
73
|
+
status?: number;
|
|
74
|
+
reason: string;
|
|
75
|
+
hadRetryAfterHeader: boolean;
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const TRANSIENT_MESSAGE_PATTERN =
|
|
79
|
+
/connection|network|fetch failed|socket|econnreset|econnrefused|enotfound|etimedout|timed? out|timeout|overloaded|service.?unavailable|temporarily unavailable/i;
|
|
80
|
+
const RATE_LIMIT_MESSAGE_PATTERN = /\b429\b|rate.?limit|too many requests/i;
|
|
81
|
+
// Precise terminal usage-limit signals (as opposed to the bare `usage`/`quota` substrings
|
|
82
|
+
// dropped above): pi-ai's Codex adapter surfaces `usage_limit_reached` / `usage_not_included`
|
|
83
|
+
// as the friendly "You have hit your ChatGPT usage limit..." message, often on a non-429
|
|
84
|
+
// status, so the status-code check above misses it. These are specific enough not to false-
|
|
85
|
+
// positive on ordinary assistant text mentioning "usage".
|
|
86
|
+
const USAGE_LIMIT_MESSAGE_PATTERN = /usage[ _]limit|usage_not_included|usagelimiterror|insufficient_quota|quota exceeded/i;
|
|
87
|
+
|
|
88
|
+
function classifyProviderError(error: unknown, observed: { status?: number; retryAfterMs?: number }): ProviderErrorClassification {
|
|
54
89
|
const value = error as { status?: unknown; errorMessage?: unknown; message?: unknown } | undefined;
|
|
55
|
-
const
|
|
90
|
+
const errorStatus = typeof value?.status === "number" ? value.status : undefined;
|
|
91
|
+
const status = observed.status ?? errorStatus;
|
|
56
92
|
const message = [value?.errorMessage, value?.message, error instanceof Error ? error.message : ""]
|
|
57
93
|
.filter((part): part is string => typeof part === "string" && part.length > 0)
|
|
58
94
|
.join(" ");
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
status
|
|
65
|
-
|
|
66
|
-
);
|
|
95
|
+
const hadRetryAfterHeader = observed.retryAfterMs !== undefined;
|
|
96
|
+
|
|
97
|
+
if (status === 429) return { kind: "rate_limit", status, reason: "http 429", hadRetryAfterHeader };
|
|
98
|
+
if (status === 503 && hadRetryAfterHeader) return { kind: "rate_limit", status, reason: "http 503 with retry-after", hadRetryAfterHeader };
|
|
99
|
+
if (status === 500 || status === 502 || status === 503 || status === 504) {
|
|
100
|
+
return { kind: "transient", status, reason: `http ${status}`, hadRetryAfterHeader };
|
|
101
|
+
}
|
|
102
|
+
const transientMatch = message.match(TRANSIENT_MESSAGE_PATTERN);
|
|
103
|
+
if (transientMatch) return { kind: "transient", status, reason: transientMatch[0].toLowerCase(), hadRetryAfterHeader };
|
|
104
|
+
const rateLimitMatch = message.match(RATE_LIMIT_MESSAGE_PATTERN);
|
|
105
|
+
if (rateLimitMatch) return { kind: "rate_limit", status, reason: rateLimitMatch[0].toLowerCase(), hadRetryAfterHeader };
|
|
106
|
+
const usageLimitMatch = message.match(USAGE_LIMIT_MESSAGE_PATTERN);
|
|
107
|
+
if (usageLimitMatch) return { kind: "rate_limit", status, reason: usageLimitMatch[0].toLowerCase(), hadRetryAfterHeader };
|
|
108
|
+
return { kind: "fatal", status, reason: "unclassified", hadRetryAfterHeader };
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async function defaultSleep(ms: number): Promise<void> {
|
|
112
|
+
await new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Indirection so tests can inject deterministic waits without real timers.
|
|
116
|
+
let activeSleep: typeof defaultSleep = defaultSleep;
|
|
117
|
+
|
|
118
|
+
/** Test-only seam: override the sleep implementation used for in-place/wait-out retries. Pass undefined to restore the default. */
|
|
119
|
+
export function __setSleepForTest(fn: typeof defaultSleep | undefined): void {
|
|
120
|
+
activeSleep = fn ?? defaultSleep;
|
|
67
121
|
}
|
|
68
122
|
|
|
69
123
|
function isInvalidRefreshTokenError(error: unknown): boolean {
|
|
@@ -75,6 +129,27 @@ function isInvalidRefreshTokenError(error: unknown): boolean {
|
|
|
75
129
|
return status === 400 && /invalid_grant|refresh token .*invalid|refresh token .*not found/i.test(message);
|
|
76
130
|
}
|
|
77
131
|
|
|
132
|
+
// A user/host abort (ESC, a critical-tier interrupt, a torn-down window) is NOT a provider
|
|
133
|
+
// failure — it must never cool down a credential or rotate providers. The upstream pi-ai
|
|
134
|
+
// providers set `reason: "aborted"` / `stopReason: "aborted"` (and throw "Request was aborted")
|
|
135
|
+
// ONLY when the request's AbortSignal fired, so those markers alone are a reliable abort
|
|
136
|
+
// signal; we do NOT require the signal to be threaded back through our own options (it may not
|
|
137
|
+
// be), which is what made an ESC drain both provider pools.
|
|
138
|
+
function isUserAbortEvent(event: Parameters<AssistantMessageEventStream["push"]>[0], signal: AbortSignal | undefined): boolean {
|
|
139
|
+
if (signal?.aborted) return true;
|
|
140
|
+
if (event.type !== "error") return false;
|
|
141
|
+
return event.reason === "aborted" || event.error.stopReason === "aborted" || /request was aborted/i.test(event.error.errorMessage ?? "");
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function isUserAbortError(error: unknown, signal: AbortSignal | undefined): boolean {
|
|
145
|
+
if (signal?.aborted) return true;
|
|
146
|
+
const value = error as { name?: unknown; errorMessage?: unknown; message?: unknown } | undefined;
|
|
147
|
+
const message = [value?.errorMessage, value?.message, error instanceof Error ? error.message : String(error ?? "")]
|
|
148
|
+
.filter((part): part is string => typeof part === "string" && part.length > 0)
|
|
149
|
+
.join(" ");
|
|
150
|
+
return (error instanceof Error && /abort/i.test(error.name)) || /request was aborted|operation was aborted|the user aborted/i.test(message);
|
|
151
|
+
}
|
|
152
|
+
|
|
78
153
|
function setStatus(text: string): void {
|
|
79
154
|
runtimeContext?.ui.setStatus("provider-rotation", text);
|
|
80
155
|
}
|
|
@@ -209,6 +284,10 @@ export function __setStreamForProviderForTest(fn: typeof defaultStreamForProvide
|
|
|
209
284
|
activeStreamForProvider = fn ?? defaultStreamForProvider;
|
|
210
285
|
}
|
|
211
286
|
|
|
287
|
+
// Drives ONE credential, retrying transient (non-rate-limit) failures in place on the
|
|
288
|
+
// SAME credential per #122, and reports back a classified result for #124's logging.
|
|
289
|
+
// A genuine rate limit ends the attempt immediately (the caller cools the credential down
|
|
290
|
+
// and rotates); a transient blip gets a short in-place retry budget before giving up.
|
|
212
291
|
async function streamProviderAttempt(
|
|
213
292
|
providerId: ManagedProviderId,
|
|
214
293
|
model: Model<any>,
|
|
@@ -217,84 +296,162 @@ async function streamProviderAttempt(
|
|
|
217
296
|
credential: SubscriptionCredential,
|
|
218
297
|
outer: AssistantMessageEventStream,
|
|
219
298
|
): Promise<AttemptResult> {
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
299
|
+
for (let attempt = 1; ; attempt++) {
|
|
300
|
+
const attemptAt = Date.now();
|
|
301
|
+
markSubscriptionAttempt(providerId, credential.label, attemptAt);
|
|
302
|
+
let observedStatus: number | undefined;
|
|
303
|
+
let requestedRetryAfterMs: number | undefined;
|
|
304
|
+
const inner = streamForProvider(model, context, {
|
|
305
|
+
...options,
|
|
306
|
+
apiKey: credential.access,
|
|
307
|
+
onResponse: async (response, responseModel) => {
|
|
308
|
+
observedStatus = response.status;
|
|
309
|
+
if (response.status === 429 || response.status === 503) {
|
|
310
|
+
requestedRetryAfterMs = retryAfterMs(response.headers);
|
|
311
|
+
}
|
|
312
|
+
await options?.onResponse?.(response, responseModel);
|
|
313
|
+
},
|
|
314
|
+
});
|
|
233
315
|
|
|
234
|
-
|
|
235
|
-
|
|
316
|
+
const bufferedEvents: Parameters<AssistantMessageEventStream["push"]>[0][] = [];
|
|
317
|
+
let sawContent = false;
|
|
318
|
+
let classification: ProviderErrorClassification | undefined;
|
|
236
319
|
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
320
|
+
const flush = (): void => {
|
|
321
|
+
for (const event of bufferedEvents) outer.push(event);
|
|
322
|
+
bufferedEvents.length = 0;
|
|
323
|
+
};
|
|
241
324
|
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
325
|
+
try {
|
|
326
|
+
for await (const event of inner) {
|
|
327
|
+
if (event.type === "error") {
|
|
328
|
+
if (isUserAbortEvent(event, options?.signal)) {
|
|
329
|
+
bufferedEvents.push(event);
|
|
330
|
+
flush();
|
|
331
|
+
return { result: "fatal", attemptAt };
|
|
332
|
+
}
|
|
333
|
+
classification = classifyProviderError(event.error, { status: observedStatus, retryAfterMs: requestedRetryAfterMs });
|
|
334
|
+
if (classification.kind === "fatal" || sawContent) {
|
|
335
|
+
bufferedEvents.push(event);
|
|
336
|
+
flush();
|
|
337
|
+
return { result: "fatal", attemptAt, classification };
|
|
338
|
+
}
|
|
339
|
+
break;
|
|
247
340
|
}
|
|
341
|
+
if (event.type !== "start" && event.type !== "done") sawContent = true;
|
|
248
342
|
bufferedEvents.push(event);
|
|
343
|
+
}
|
|
344
|
+
} catch (error) {
|
|
345
|
+
if (isUserAbortError(error, options?.signal)) {
|
|
346
|
+
bufferedEvents.push({
|
|
347
|
+
type: "error",
|
|
348
|
+
reason: "aborted",
|
|
349
|
+
error: { ...toAssistantError(model, error instanceof Error ? error.message : String(error)), stopReason: "aborted" },
|
|
350
|
+
});
|
|
249
351
|
flush();
|
|
250
352
|
return { result: "fatal", attemptAt };
|
|
251
353
|
}
|
|
252
|
-
|
|
253
|
-
|
|
354
|
+
classification = classifyProviderError(error, { status: observedStatus, retryAfterMs: requestedRetryAfterMs });
|
|
355
|
+
if (classification.kind === "fatal" || sawContent) {
|
|
356
|
+
bufferedEvents.push({
|
|
357
|
+
type: "error",
|
|
358
|
+
reason: "error",
|
|
359
|
+
error: toAssistantError(model, error instanceof Error ? error.message : String(error)),
|
|
360
|
+
});
|
|
361
|
+
flush();
|
|
362
|
+
return { result: "fatal", attemptAt, classification };
|
|
363
|
+
}
|
|
254
364
|
}
|
|
255
|
-
|
|
256
|
-
if (
|
|
257
|
-
|
|
365
|
+
|
|
366
|
+
if (!classification && requestedRetryAfterMs !== undefined && !sawContent) {
|
|
367
|
+
// Some providers signal a 429/503 purely via the response header without ever raising
|
|
368
|
+
// a stream "error" event; treat that the same as an explicit rate-limit classification.
|
|
369
|
+
classification = { kind: "rate_limit", status: observedStatus, reason: "retry-after header on success path", hadRetryAfterHeader: true };
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
if (!classification) {
|
|
373
|
+
flush();
|
|
374
|
+
return { result: "success", attemptAt };
|
|
258
375
|
}
|
|
259
|
-
bufferedEvents.push({
|
|
260
|
-
type: "error",
|
|
261
|
-
reason: "error",
|
|
262
|
-
error: toAssistantError(model, error instanceof Error ? error.message : String(error)),
|
|
263
|
-
});
|
|
264
|
-
flush();
|
|
265
|
-
return { result: "fatal", attemptAt };
|
|
266
|
-
}
|
|
267
376
|
|
|
268
|
-
|
|
269
|
-
|
|
377
|
+
if (classification.kind === "rate_limit") {
|
|
378
|
+
return { result: "rate_limited", retryAfterMs: requestedRetryAfterMs, attemptAt, classification };
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
// Transient: retry the SAME credential/provider in place -- no cooldown, no rotation.
|
|
382
|
+
const backoffMs = TRANSIENT_RETRY_BACKOFFS_MS[attempt - 1];
|
|
383
|
+
if (backoffMs === undefined) {
|
|
384
|
+
// Exhausted the short in-place retry budget: surface a clear terminal error rather
|
|
385
|
+
// than spinning forever or silently rotating a healthy credential away.
|
|
386
|
+
bufferedEvents.push({
|
|
387
|
+
type: "error",
|
|
388
|
+
reason: "error",
|
|
389
|
+
error: toAssistantError(
|
|
390
|
+
model,
|
|
391
|
+
`${getProviderLabel(providerId)} "${credential.label}" hit a transient error after ${attempt} attempts (${classification.reason})`,
|
|
392
|
+
),
|
|
393
|
+
});
|
|
394
|
+
flush();
|
|
395
|
+
return { result: "fatal", attemptAt, classification };
|
|
396
|
+
}
|
|
397
|
+
logRotation(
|
|
398
|
+
`${getProviderLabel(providerId)} subscription "${credential.label}" hit a transient error (${classification.reason}, status=${classification.status ?? "n/a"}); retrying same credential in ${backoffMs}ms (attempt ${attempt + 1})`,
|
|
399
|
+
);
|
|
400
|
+
await activeSleep(backoffMs);
|
|
270
401
|
}
|
|
402
|
+
}
|
|
271
403
|
|
|
272
|
-
|
|
273
|
-
|
|
404
|
+
function emitTerminalError(stream: AssistantMessageEventStream, model: Model<any>, message: string): void {
|
|
405
|
+
stream.push({ type: "error", reason: "error", error: toAssistantError(model, message) });
|
|
406
|
+
stream.end();
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
// #123: gates cross-provider fallback on the target actually being usable right now --
|
|
410
|
+
// authenticated (its subscription pool has at least one credential, even if currently
|
|
411
|
+
// cooling down) AND registered in the model registry -- rather than blindly switching the
|
|
412
|
+
// session model to a provider nobody logged into. Returns undefined when there is nothing
|
|
413
|
+
// viable to switch to (config has no ladder rung, the target was already tried this turn,
|
|
414
|
+
// the target has zero credentials, or its resolved model isn't in the registry).
|
|
415
|
+
function resolveViableFallback(
|
|
416
|
+
providerId: ManagedProviderId,
|
|
417
|
+
modelId: string | undefined,
|
|
418
|
+
attempted: Set<ManagedProviderId>,
|
|
419
|
+
): FallbackTarget | undefined {
|
|
420
|
+
const target = resolveFallbackTarget(providerId, modelId, readRotationConfig(), extensionAPI?.getThinkingLevel?.());
|
|
421
|
+
if (!target || attempted.has(target.providerId)) return undefined;
|
|
422
|
+
if (readSubscriptionPool(target.providerId).length === 0) return undefined;
|
|
423
|
+
if (!runtimeContext?.modelRegistry.find(target.providerId, target.modelId)) return undefined;
|
|
424
|
+
return target;
|
|
274
425
|
}
|
|
275
426
|
|
|
276
|
-
|
|
427
|
+
// Switches the session model to an already-viability-checked fallback target. On the rare
|
|
428
|
+
// race where the switch itself still fails (e.g. `setModel` rejects it), emits a graceful,
|
|
429
|
+
// actionable terminal error on `stream` instead of throwing a generic "failed to switch".
|
|
430
|
+
async function switchToFallbackModel(
|
|
431
|
+
stream: AssistantMessageEventStream,
|
|
432
|
+
model: Model<any>,
|
|
433
|
+
originalModel: { providerId: ManagedProviderId; modelId: string },
|
|
434
|
+
): Promise<FallbackTarget | undefined> {
|
|
277
435
|
const ctx = runtimeContext;
|
|
278
436
|
if (!extensionAPI || !ctx) {
|
|
279
|
-
|
|
437
|
+
emitTerminalError(stream, model, `Provider fallback unavailable: runtime not initialized for ${originalModel.providerId}/${originalModel.modelId}`);
|
|
438
|
+
return undefined;
|
|
280
439
|
}
|
|
281
440
|
const target = await switchToFallbackIfPossible(extensionAPI, ctx, originalModel);
|
|
282
441
|
if (!target) {
|
|
283
|
-
|
|
284
|
-
|
|
442
|
+
emitTerminalError(stream, model, `Provider fallback unavailable: failed to switch models for ${getProviderLabel(originalModel.providerId)}`);
|
|
443
|
+
return undefined;
|
|
285
444
|
}
|
|
286
445
|
return target;
|
|
287
446
|
}
|
|
288
447
|
|
|
289
|
-
function emitTerminalError(stream: AssistantMessageEventStream, model: Model<any>, message: string): void {
|
|
290
|
-
stream.push({ type: "error", reason: "error", error: toAssistantError(model, message) });
|
|
291
|
-
stream.end();
|
|
292
|
-
}
|
|
293
|
-
|
|
294
448
|
// Drives one managed provider's pool into `stream`; on exhaustion it switches the
|
|
295
449
|
// session model to the cross-provider fallback and CONTINUES the same turn on it,
|
|
296
450
|
// rather than failing the turn with an error. `attempted` guards anthropic<->codex
|
|
297
|
-
// ping-pong: once both pools are dry we emit a single terminal error.
|
|
451
|
+
// ping-pong: once both pools are dry (or unauthenticated) we emit a single terminal error.
|
|
452
|
+
// If neither a fallback nor more credentials are available but the sole/current provider
|
|
453
|
+
// is merely cooling down, we sleep out the shortest cooldown and retry it in place (#123)
|
|
454
|
+
// instead of failing the turn.
|
|
298
455
|
async function runManagedProvider(
|
|
299
456
|
model: Model<any>,
|
|
300
457
|
context: Context,
|
|
@@ -304,7 +461,8 @@ async function runManagedProvider(
|
|
|
304
461
|
): Promise<void> {
|
|
305
462
|
const providerId = model.provider as ManagedProviderId;
|
|
306
463
|
attempted.add(providerId);
|
|
307
|
-
const
|
|
464
|
+
const rawPool = readSubscriptionPool(providerId);
|
|
465
|
+
const available = rawPool.filter((entry) => !entry.rateLimitedUntil || entry.rateLimitedUntil <= Date.now());
|
|
308
466
|
|
|
309
467
|
for (const credential of available) {
|
|
310
468
|
let freshCredential: SubscriptionCredential;
|
|
@@ -344,39 +502,89 @@ async function runManagedProvider(
|
|
|
344
502
|
return;
|
|
345
503
|
}
|
|
346
504
|
|
|
347
|
-
if (result.result === "
|
|
505
|
+
if (result.result === "rate_limited") {
|
|
348
506
|
const cooldownMs = result.retryAfterMs ?? DEFAULT_RATE_LIMIT_BACKOFF_MS;
|
|
349
507
|
markSubscriptionRateLimited(providerId, freshCredential.label, cooldownMs, result.attemptAt);
|
|
350
|
-
|
|
508
|
+
// #124: make a misclassification obvious at a glance -- status, matched reason, and
|
|
509
|
+
// whether a real retry-after header drove the cooldown vs. our default backoff.
|
|
510
|
+
const retryAfterPart = result.retryAfterMs !== undefined ? `retry-after=${Math.ceil(result.retryAfterMs / 1000)}s` : "no retry-after header (default backoff)";
|
|
511
|
+
logRotation(
|
|
512
|
+
`${getProviderLabel(providerId)} subscription "${freshCredential.label}" rate-limited (status=${result.classification?.status ?? "n/a"}, reason=${result.classification?.reason ?? "unclassified"}, ${retryAfterPart}); cooling down for ${Math.ceil(cooldownMs / 1000)}s, trying next credential`,
|
|
513
|
+
);
|
|
351
514
|
continue;
|
|
352
515
|
}
|
|
353
516
|
|
|
354
517
|
// Fatal (or partial content already streamed): cannot safely switch providers mid-turn.
|
|
518
|
+
if (result.classification) {
|
|
519
|
+
logRotation(
|
|
520
|
+
`${getProviderLabel(providerId)} subscription "${freshCredential.label}" failed fatally (status=${result.classification.status ?? "n/a"}, reason=${result.classification.reason})`,
|
|
521
|
+
);
|
|
522
|
+
}
|
|
355
523
|
stream.end();
|
|
356
524
|
return;
|
|
357
525
|
}
|
|
358
526
|
|
|
359
|
-
// Pool exhausted
|
|
527
|
+
// Pool exhausted (or this provider was never authenticated at all) — figure out what to
|
|
528
|
+
// do next: switch to a viable cross-provider fallback, wait out a sole/authed provider's
|
|
529
|
+
// cooldown, or surface an actionable terminal error. Re-read the pool here rather than
|
|
530
|
+
// reusing the pre-loop `rawPool` snapshot: the credential loop above mutates cooldowns
|
|
531
|
+
// in place via `markSubscriptionRateLimited` (e.g. a sole credential that just got a
|
|
532
|
+
// fresh 429), and the stale snapshot would still show it as available, making the
|
|
533
|
+
// wait-out math below think there's nothing to wait for and wrongly declare exhaustion.
|
|
534
|
+
const currentPool = readSubscriptionPool(providerId);
|
|
360
535
|
const originalModel = getCurrentModelRef(model);
|
|
361
536
|
if (!originalModel) throw new Error(`Provider fallback unavailable: missing current model`);
|
|
362
537
|
|
|
363
|
-
const
|
|
364
|
-
if (
|
|
365
|
-
|
|
538
|
+
const viableTarget = resolveViableFallback(originalModel.providerId, originalModel.modelId, attempted);
|
|
539
|
+
if (viableTarget) {
|
|
540
|
+
const target = await switchToFallbackModel(stream, model, originalModel);
|
|
541
|
+
if (!target) return; // terminal error already emitted
|
|
542
|
+
logRotation(`${getProviderLabel(providerId)} pool exhausted -> ${target.label}`);
|
|
543
|
+
setStatus(`${getProviderLabel(providerId)} -> ${target.label}`);
|
|
544
|
+
|
|
545
|
+
const fallbackModel = runtimeContext?.modelRegistry.find(target.providerId, target.modelId);
|
|
546
|
+
if (!fallbackModel) {
|
|
547
|
+
emitTerminalError(stream, model, `Provider fallback unavailable: ${target.label} not in model registry`);
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
await runManagedProvider(fallbackModel, context, options, stream, attempted);
|
|
366
552
|
return;
|
|
367
553
|
}
|
|
368
554
|
|
|
369
|
-
const
|
|
370
|
-
|
|
371
|
-
setStatus(`${getProviderLabel(providerId)} -> ${target.label}`);
|
|
555
|
+
const label = getProviderLabel(providerId);
|
|
556
|
+
const triedList = [...attempted].join(", ");
|
|
372
557
|
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
558
|
+
// #123: this provider was never authenticated at all (no subscription pool entries ever
|
|
559
|
+
// existed for it) -- there's nothing to wait out, so this is a config/setup problem, not
|
|
560
|
+
// a transient one.
|
|
561
|
+
if (currentPool.length === 0) {
|
|
562
|
+
emitTerminalError(stream, model, `${label} not authenticated — run \`crtr sys setup\` (or /provider-sub ${providerId} add) (tried ${triedList})`);
|
|
376
563
|
return;
|
|
377
564
|
}
|
|
378
565
|
|
|
379
|
-
|
|
566
|
+
// Authenticated but every credential is cooling down, and no authenticated fallback
|
|
567
|
+
// exists right now: sleep out the SHORTEST cooldown and retry the same provider instead
|
|
568
|
+
// of failing the turn -- this is the common "single subscription, briefly rate-limited"
|
|
569
|
+
// case, not a permanent outage.
|
|
570
|
+
const soonestRateLimitedUntil = Math.min(...currentPool.map((entry) => entry.rateLimitedUntil || 0));
|
|
571
|
+
const waitMs = soonestRateLimitedUntil - Date.now();
|
|
572
|
+
|
|
573
|
+
if (waitMs > 0 && waitMs <= SOLE_PROVIDER_MAX_WAIT_MS) {
|
|
574
|
+
logRotation(`${label} cooling down for ${Math.ceil(waitMs / 1000)}s with no authenticated fallback; waiting to retry the same provider`);
|
|
575
|
+
setStatus(`${label} cooling down for ${Math.ceil(waitMs / 1000)}s, retrying...`);
|
|
576
|
+
await activeSleep(waitMs);
|
|
577
|
+
await runManagedProvider(model, context, options, stream, attempted);
|
|
578
|
+
return;
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
emitTerminalError(
|
|
582
|
+
stream,
|
|
583
|
+
model,
|
|
584
|
+
waitMs > SOLE_PROVIDER_MAX_WAIT_MS
|
|
585
|
+
? `${label} cooling down for ${Math.ceil(waitMs / 1000)}s — too long to wait automatically; authenticate a fallback provider with /provider-sub, or try again later (tried ${triedList})`
|
|
586
|
+
: `All managed provider pools exhausted (tried ${triedList})`,
|
|
587
|
+
);
|
|
380
588
|
}
|
|
381
589
|
|
|
382
590
|
function streamManagedProvider(model: Model<any>, context: Context, options?: SimpleStreamOptions): AssistantMessageEventStream {
|
|
@@ -1,39 +1,46 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Strip pi
|
|
2
|
+
* Strip pi's "Pi documentation" block from the system prompt.
|
|
3
3
|
*
|
|
4
|
-
* Pi core hardcodes a "Pi documentation
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* surfaces pi's internal skill system when we don't want it referenced.
|
|
4
|
+
* Pi core hardcodes a "Pi documentation (read only when the user asks about pi
|
|
5
|
+
* itself, its SDK, extensions, themes, skills, or TUI):" block — a marker line
|
|
6
|
+
* plus a bullet list pointing at pi's README / docs / examples under
|
|
7
|
+
* node_modules. It causes two problems we fix by removing the whole block:
|
|
9
8
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* the
|
|
9
|
+
* 1. It maps topics to doc files (`skills (docs/skills.md)`, etc.), which makes
|
|
10
|
+
* the agent extrapolate a topic -> docs/<topic>.md pattern and surface pi's
|
|
11
|
+
* internal skill system when we don't want it referenced.
|
|
12
|
+
* 2. It is a THIRD-PARTY TELL. When authenticating a Claude subscription over
|
|
13
|
+
* OAuth, Anthropic inspects the request and — if it doesn't look like genuine
|
|
14
|
+
* Claude Code — routes it to "extra usage" instead of the plan, returning
|
|
15
|
+
* HTTP 400 ("Third-party apps now draw from your extra usage..."). The
|
|
16
|
+
* companion `pi-claude-oauth-adapter` normally strips this block for exactly
|
|
17
|
+
* that reason, but it matches the marker string INCLUDING the word "skills,"
|
|
18
|
+
* — and the earlier version of THIS extension mangled that marker (it removed
|
|
19
|
+
* just the "skills" word), so the adapter's match failed and the block leaked
|
|
20
|
+
* to the wire → every subscription turn 400'd. Removing the entire block here
|
|
21
|
+
* is self-contained: it fixes the leak regardless of the adapter's marker,
|
|
22
|
+
* load order, or presence, and subsumes the old skill-reference removal.
|
|
15
23
|
*
|
|
16
|
-
*
|
|
24
|
+
* Runs in `before_agent_start` so it survives `pi update` / `brew upgrade`
|
|
25
|
+
* (which would overwrite any edit to the installed dist), and degrades to a
|
|
26
|
+
* no-op if pi changes the wording.
|
|
17
27
|
*/
|
|
18
28
|
|
|
19
29
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
20
30
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
// or TUI" enumeration.
|
|
30
|
-
p = p.replace(/,[ \t]*skills(?=[ \t]*,[ \t]*or TUI)/g, "");
|
|
31
|
-
return p;
|
|
31
|
+
// Matches the whole block: the "Pi documentation (read only ...):" marker line
|
|
32
|
+
// (with or without the "skills," enumeration) plus its trailing "- ..." bullets,
|
|
33
|
+
// and any leading blank lines so the surrounding prompt closes up cleanly.
|
|
34
|
+
const PI_DOCS_BLOCK = /\n*^Pi documentation \(read only[^\n]*:(?:\n[ \t]*-[^\n]*)*/m;
|
|
35
|
+
|
|
36
|
+
function stripPiDocsBlock(prompt: string): string {
|
|
37
|
+
if (!prompt.includes("Pi documentation (read only")) return prompt; // no-op / wording changed
|
|
38
|
+
return prompt.replace(PI_DOCS_BLOCK, "").trimEnd() + "\n";
|
|
32
39
|
}
|
|
33
40
|
|
|
34
41
|
export default function stripSkillsDocs(pi: ExtensionAPI) {
|
|
35
42
|
pi.on("before_agent_start", async (event) => {
|
|
36
|
-
const stripped =
|
|
43
|
+
const stripped = stripPiDocsBlock(event.systemPrompt);
|
|
37
44
|
if (stripped === event.systemPrompt) return;
|
|
38
45
|
return { systemPrompt: stripped };
|
|
39
46
|
});
|