koneck 2.74.0 → 2.75.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine.d.ts +31 -0
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +225 -16
- package/dist/engine.js.map +1 -1
- package/dist/failover.d.ts +13 -1
- package/dist/failover.d.ts.map +1 -1
- package/dist/failover.js +18 -2
- package/dist/failover.js.map +1 -1
- package/dist/types.d.ts +13 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/web/sessions.d.ts +2 -0
- package/dist/web/sessions.d.ts.map +1 -1
- package/dist/web/sessions.js +27 -0
- package/dist/web/sessions.js.map +1 -1
- package/package.json +1 -1
package/dist/engine.d.ts
CHANGED
|
@@ -71,6 +71,37 @@ export declare function isRetryableProviderError(err: unknown): boolean;
|
|
|
71
71
|
* Jitter matters as much as the delay. Parallel agents fail together, so without it they retry in
|
|
72
72
|
* lockstep and collide again on exactly the request that rate-limited them.
|
|
73
73
|
*/
|
|
74
|
+
/**
|
|
75
|
+
* Whether a refusal is about what the account may use rather than about the key.
|
|
76
|
+
*
|
|
77
|
+
* A 401 and a 403 look the same from outside and mean quite different things underneath: a key that
|
|
78
|
+
* is wrong, and a key that is right for an account which has not paid for this model. Only the
|
|
79
|
+
* first is fixed by looking at the key, and telling somebody to check a working key while their
|
|
80
|
+
* balance is the problem costs them the afternoon.
|
|
81
|
+
*
|
|
82
|
+
* Read from the provider's own words, in the languages providers answer in. 充值 is top up, 余额 is
|
|
83
|
+
* balance, 权限 is permission.
|
|
84
|
+
*/
|
|
85
|
+
export declare function looksLikeEntitlement(err: unknown): boolean;
|
|
86
|
+
/** The provider's own sentence about it, for saying what it said rather than paraphrasing. */
|
|
87
|
+
export declare function entitlementReason(err: unknown): string;
|
|
88
|
+
/**
|
|
89
|
+
* Whether nothing could be reached at all, as opposed to something answering unhappily.
|
|
90
|
+
*
|
|
91
|
+
* The distinction decides how long to keep trying. A 429 or a 500 is a provider that is up and
|
|
92
|
+
* telling you something; a connection that cannot be made is usually the link, and the link usually
|
|
93
|
+
* comes back. Carries no status by definition — an HTTP status means an HTTP response happened.
|
|
94
|
+
*/
|
|
95
|
+
/**
|
|
96
|
+
* Waits, unless the turn is stopped first.
|
|
97
|
+
*
|
|
98
|
+
* A plain setTimeout is fine for a second and wrong for thirty: while waiting out an outage the
|
|
99
|
+
* stop button and ctrl-c have to work, and a timer that cannot be interrupted turns "stop" into
|
|
100
|
+
* "stop in half a minute". The listener is removed either way, so a long run does not accumulate
|
|
101
|
+
* one per attempt.
|
|
102
|
+
*/
|
|
103
|
+
export declare function sleepUnlessAborted(ms: number, signal?: AbortSignal): Promise<void>;
|
|
104
|
+
export declare function isConnectivityFailure(err: unknown): boolean;
|
|
74
105
|
export declare function retryDelayMs(err: unknown, attempt: number, random?: () => number): number;
|
|
75
106
|
/**
|
|
76
107
|
* Whether a provider is saying the model cannot do tool calling.
|
package/dist/engine.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../src/engine.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAc5B,OAAO,EACE,KAAK,IAAI,EAAE,MAAM,WAAW,CAAC;AAoBtC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAMvD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAWtC,OAAO,EAA2B,KAAK,WAAW,EAAE,MAAM,cAAc,CAAC;AACzE,OAAO,EAIL,KAAK,cAAc,EACpB,MAAM,UAAU,CAAC;AAWlB,OAAO,EAAE,kBAAkB,EAAuD,MAAM,iBAAiB,CAAC;AAI1G,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,KAAK,EACV,OAAO,EACP,WAAW,EAEX,UAAU,EAKX,MAAM,YAAY,CAAC;AAIpB;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CACnC,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;AAC7C;;;;GAIG;AACH,KAAK,GAAE;IAAE,QAAQ,CAAC,EAAE,OAAO,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GACtD,MAAM,
|
|
1
|
+
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../src/engine.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAc5B,OAAO,EACE,KAAK,IAAI,EAAE,MAAM,WAAW,CAAC;AAoBtC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAMvD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAWtC,OAAO,EAA2B,KAAK,WAAW,EAAE,MAAM,cAAc,CAAC;AACzE,OAAO,EAIL,KAAK,cAAc,EACpB,MAAM,UAAU,CAAC;AAWlB,OAAO,EAAE,kBAAkB,EAAuD,MAAM,iBAAiB,CAAC;AAI1G,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,KAAK,EACV,OAAO,EACP,WAAW,EAEX,UAAU,EAKX,MAAM,YAAY,CAAC;AAIpB;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CACnC,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;AAC7C;;;;GAIG;AACH,KAAK,GAAE;IAAE,QAAQ,CAAC,EAAE,OAAO,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GACtD,MAAM,CAmGR;AAED,qGAAqG;AACrG,MAAM,WAAW,aAAa;IAAG,MAAM,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE;AAEpF;;;;;;;;;;;GAWG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,UAAU,EACjB,QAAQ,EAAE,aAAa,EACvB,KAAK,EAAE;IAAE,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GACxG,IAAI,CAQN;AAyCD;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAkD9D;AAED;;;;;;;;;;GAUG;AACH;;;;;;;;;;GAUG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAe1D;AAED,8FAA8F;AAC9F,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAQtD;AAED;;;;;;GAMG;AACH;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAWlF;AAED,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAO3D;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAc,GAAG,MAAM,CAuBxF;AAED;;;;;;;GAOG;AACH;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAGzD;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAI5D;AAED,wBAAgB,yBAAyB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAG/D;AAsED;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAkB7D;AAMD;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,OAAO,GAAG,OAAO,CAExF;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAcvD;AAID,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,MAAM,CAQvD;AAID;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,SAAK,GAAG,MAAM,CAOrE;AAiDD,kEAAkE;AAClE,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,SAAM,GAAG,MAAM,CAG/D;AAED,uEAAuE;AACvE,eAAO,MAAM,gBAAgB,OAAQ,CAAC;AAEtC,6FAA6F;AAC7F,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,WAAW,CAAC,aAAa,CAAC,GAAG,MAAM,CAStF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,SAAmB,GAAG,MAAM,CAK/E;AA2ID;;;;;;;GAOG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yDAAyD;IACzD,GAAG,EAAE,OAAO,CAAC;IACb,gDAAgD;IAChD,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,SAAS,EAAE,gBAAgB,GAC1B,MAAM,EAAE,CAMV;AAED,wBAAgB,eAAe,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,OAAO,CAE7E;AA+DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAG7E;AAQD;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAAE,CAAC,EACrC,KAAK,EAAE,SAAS,CAAC,EAAE,EACnB,KAAK,EAAE,MAAM,EACb,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GACzC,OAAO,CAAC,CAAC,EAAE,CAAC,CAad;AAkJD;;;;;GAKG;AACH;;;;;;;GAOG;AACH,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,WAAW,GAAG,MAAM,GAAG,SAAS,CAM1E;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,SAAS,OAAO,EAAE,GAAG,OAAO,CAGlE;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,CAevD;AAGD;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CASpE;AAGD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,OAAO,EACpB,QAAQ,CAAC,EAAE,MAAM,GAChB,MAAM,CAiDR;AAED,wBAAsB,YAAY,CAChC,QAAQ,EAAE,OAAO,EAAE,EACnB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,WAAW,EACnB,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,UAAU,EACjB,UAAU,EAAE,cAAc,EAAE,EAC5B,OAAO,EAAE,WAAW,EAAE,EAEtB,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAC7C,UAAU,SAAI,EACd,WAAW,CAAC,EAAE,WAAW,EACzB,UAAU,CAAC,EAAE,kBAAkB;AAC/B;;;;;;GAMG;AACH,OAAO,GAAE;IAAE,OAAO,EAAE,IAAI,CAAA;CAA4B;AACpD,2FAA2F;AAC3F,UAAU,CAAC,EAAE,UAAU;AACvB,4FAA4F;AAC5F,eAAe,GAAE;IAAE,OAAO,EAAE,MAAM,CAAA;CAAmB,GACpD,OAAO,CAAC,OAAO,CAAC,CAgvDlB;AAID,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,kEAAkE;IAClE,QAAQ,CAAC,UAAU,EAAE,SAAS,eAAe,EAAE,CAAC;IAChD,8FAA8F;IAC9F,QAAQ,CAAC,UAAU,EAAE,SAAS;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,SAAS;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,WAAW,CAAC,EAAE,MAAM,CAAA;SAAE,EAAE,CAAA;KAAE,EAAE,CAAC;IAC3G,6FAA6F;IAC7F,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,EAAE,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/F;;;;;;OAMG;IACH,OAAO,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;CAC3B;AAGD;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAarE;AAED,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,WAAW,EACnB,eAAe,CAAC,EAAE,OAAO,EAAE;AAC3B;;;;;;;GAOG;AACH,YAAY,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC;AAClC,iGAAiG;AACjG,WAAW,CAAC,EAAE,IAAI,GACjB,OAAO,CAAC,YAAY,CAAC,CAwRvB;AAwBD;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,uFAAuF;IACvF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAGD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAExD;AAED,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,WAAW,EACnB,eAAe,CAAC,EAAE,OAAO,EAAE,EAC3B,MAAM,CAAC,EAAE,YAAY,GACpB,OAAO,CAAC,YAAY,CAAC,CAwKvB"}
|
package/dist/engine.js
CHANGED
|
@@ -7,7 +7,7 @@ import { execa } from 'execa';
|
|
|
7
7
|
import { TOOL_DEFINITIONS, dispatchTool, describeShell, unknownToolError, resolveToolAlias, aliasNotice, isToolFailure, } from './tools.js';
|
|
8
8
|
import { tui } from './tui.js';
|
|
9
9
|
import { hasWorkspaceOwnedGitRepo, gitScope, createWorktree, mergeWorktree, preservedWorktreeNotice } from './worktree.js';
|
|
10
|
-
import { resolveProvider, getApiKey, keyProvenance, servesLocally, servesOwnWeights, isPrivateHost } from './providers.js';
|
|
10
|
+
import { resolveProvider, getApiKey, keyProvenance, servesLocally, servesOwnWeights, needsNoCredential, isPrivateHost } from './providers.js';
|
|
11
11
|
import { wireEffort, looksLikeEffortUnsupported, acceptedEfforts, nearestAccepted } from './pace.js';
|
|
12
12
|
import { triggerFor, pickNext, describeSwitch, askSwitch, keyOf as failoverKey } from './failover.js';
|
|
13
13
|
import { findChurn, describeChurn, WINDOW as CHURN_WINDOW } from './cycle.js';
|
|
@@ -92,9 +92,25 @@ about = {}) {
|
|
|
92
92
|
const hint = status === 429
|
|
93
93
|
? `Rate limited or out of quota on ${provider}. Check the plan and billing for that account.`
|
|
94
94
|
: status === 401 || status === 403
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
95
|
+
/*
|
|
96
|
+
* A 403 is not always about the key, and saying it is sends somebody to check a key that is
|
|
97
|
+
* fine.
|
|
98
|
+
*
|
|
99
|
+
* Verbatim from a provider: "Access restricted. Deposit required to unlock premium models.
|
|
100
|
+
* (access_denied)". KONECK answered that with "rejected the credential. Check the API key,
|
|
101
|
+
* and that it is enabled for this endpoint" — advice about the one thing that was not the
|
|
102
|
+
* matter. The key was accepted; the account is not entitled to that model.
|
|
103
|
+
*/
|
|
104
|
+
? looksLikeEntitlement(err)
|
|
105
|
+
? `${provider} accepted the key but will not serve "${model}" on this account`
|
|
106
|
+
+ `${entitlementReason(err) ? ' — ' + entitlementReason(err) : '.'}`
|
|
107
|
+
+ `\nThis is the account rather than the key: nothing about the key needs changing. `
|
|
108
|
+
+ `Either add credit or a plan for that model, or use /models to pick one the account `
|
|
109
|
+
+ `already covers.`
|
|
110
|
+
+ (about.credential ? `\nThe key used: ${about.credential}` : '')
|
|
111
|
+
: `${provider} rejected the credential. Check the API key, and that it is enabled for this `
|
|
112
|
+
+ `endpoint.`
|
|
113
|
+
+ (about.credential ? `\nThe key used: ${about.credential}` : '')
|
|
98
114
|
: status === 404
|
|
99
115
|
? `${provider} does not recognise model "${model}". Use /models to pick one it offers.`
|
|
100
116
|
: status === 400
|
|
@@ -176,6 +192,23 @@ const MAX_CONTINUATION_NUDGES = 4;
|
|
|
176
192
|
const STREAM_IDLE_MS = 120_000;
|
|
177
193
|
/** How many times a turn is retried after a transient provider failure. */
|
|
178
194
|
const MAX_TURN_RETRIES = 3;
|
|
195
|
+
/**
|
|
196
|
+
* How long to keep trying when the network is simply not there.
|
|
197
|
+
*
|
|
198
|
+
* A provider that answers with a 500 or a 429 is up and saying something, and three attempts is a
|
|
199
|
+
* fair test of whether it will settle. A connection that cannot be made at all is a different
|
|
200
|
+
* situation: the wifi dropped, the tunnel restarted, the laptop moved between networks. Those last
|
|
201
|
+
* minutes rather than seconds, and ending an hour of work because of two is the wrong answer —
|
|
202
|
+
* "keep trying and carry on when it comes back" is what a person would do, and what was asked for.
|
|
203
|
+
*
|
|
204
|
+
* Bounded rather than infinite, because a run nobody is watching should not hold a machine open for
|
|
205
|
+
* ever. An hour is long enough to cover a reboot of anything on the path and short enough to be a
|
|
206
|
+
* limit rather than a leak. Stopping sooner is always available: the turn's abort signal is honoured
|
|
207
|
+
* between attempts, so ctrl-c and the stop button both work throughout.
|
|
208
|
+
*/
|
|
209
|
+
const OFFLINE_RETRY_WINDOW_MS = 60 * 60 * 1000;
|
|
210
|
+
/** The longest wait between attempts while offline. Long enough to be patient, short enough to notice recovery. */
|
|
211
|
+
const OFFLINE_MAX_BACKOFF_MS = 30_000;
|
|
179
212
|
/** Consecutive empty responses before the provider is declared unresponsive. */
|
|
180
213
|
const MAX_EMPTY_TURNS = 3;
|
|
181
214
|
/**
|
|
@@ -207,6 +240,16 @@ export function isRetryableProviderError(err) {
|
|
|
207
240
|
return true; // rate limited: back off and retry
|
|
208
241
|
if (status !== undefined && status >= 500)
|
|
209
242
|
return true; // server-side, usually transient
|
|
243
|
+
/*
|
|
244
|
+
* An account that may not use this model will not be able to use it on the third attempt either.
|
|
245
|
+
*
|
|
246
|
+
* "Access restricted. Deposit required to unlock premium models" contains the word "model", so
|
|
247
|
+
* the router rule below read it as a routing problem and retried three times over several
|
|
248
|
+
* seconds to arrive at the same refusal. It is a durable state of the account, and the way out is
|
|
249
|
+
* a different model or a payment, not another request.
|
|
250
|
+
*/
|
|
251
|
+
if ((status === 401 || status === 403) && looksLikeEntitlement(err))
|
|
252
|
+
return false;
|
|
210
253
|
// A 401/403/404 naming a model or route is the router, not the key.
|
|
211
254
|
if (status === 401 || status === 403 || status === 404) {
|
|
212
255
|
return /model|route|not supported|unavailable|no such/.test(text);
|
|
@@ -241,6 +284,82 @@ export function isRetryableProviderError(err) {
|
|
|
241
284
|
* Jitter matters as much as the delay. Parallel agents fail together, so without it they retry in
|
|
242
285
|
* lockstep and collide again on exactly the request that rate-limited them.
|
|
243
286
|
*/
|
|
287
|
+
/**
|
|
288
|
+
* Whether a refusal is about what the account may use rather than about the key.
|
|
289
|
+
*
|
|
290
|
+
* A 401 and a 403 look the same from outside and mean quite different things underneath: a key that
|
|
291
|
+
* is wrong, and a key that is right for an account which has not paid for this model. Only the
|
|
292
|
+
* first is fixed by looking at the key, and telling somebody to check a working key while their
|
|
293
|
+
* balance is the problem costs them the afternoon.
|
|
294
|
+
*
|
|
295
|
+
* Read from the provider's own words, in the languages providers answer in. 充值 is top up, 余额 is
|
|
296
|
+
* balance, 权限 is permission.
|
|
297
|
+
*/
|
|
298
|
+
export function looksLikeEntitlement(err) {
|
|
299
|
+
const holder = err;
|
|
300
|
+
const text = [
|
|
301
|
+
typeof holder?.message === 'string' ? holder.message : '',
|
|
302
|
+
typeof holder?.error?.message === 'string' ? holder.error.message : '',
|
|
303
|
+
typeof holder?.error?.code === 'string' ? holder.error.code : '',
|
|
304
|
+
].join(' ').toLowerCase();
|
|
305
|
+
if (!text)
|
|
306
|
+
return false;
|
|
307
|
+
// An explicit "the key is wrong" wins, however much billing language surrounds it: a provider
|
|
308
|
+
// that names the key is not talking about the balance.
|
|
309
|
+
if (/invalid api key|incorrect api key|api key not valid|no api key|missing api key|invalid[_ ]api[_ ]key|authentication_error/.test(text)) {
|
|
310
|
+
return false;
|
|
311
|
+
}
|
|
312
|
+
return /deposit|top ?up|insufficient|balance|credit|quota|subscription|not entitled|no access to|upgrade your|billing|payment required|premium model|plan does not|充值|余额|权限不足/
|
|
313
|
+
.test(text);
|
|
314
|
+
}
|
|
315
|
+
/** The provider's own sentence about it, for saying what it said rather than paraphrasing. */
|
|
316
|
+
export function entitlementReason(err) {
|
|
317
|
+
const holder = err;
|
|
318
|
+
const raw = [
|
|
319
|
+
typeof holder?.error?.message === 'string' ? holder.error.message : '',
|
|
320
|
+
typeof holder?.message === 'string' ? holder.message : '',
|
|
321
|
+
].find(t => t && t.trim() !== '') ?? '';
|
|
322
|
+
// The request id is for a support ticket, not for a person reading an error, and it is long.
|
|
323
|
+
return raw.replace(/\(request id:[^)]*\)/i, '').replace(/\s+/g, ' ').trim().slice(0, 200);
|
|
324
|
+
}
|
|
325
|
+
/**
|
|
326
|
+
* Whether nothing could be reached at all, as opposed to something answering unhappily.
|
|
327
|
+
*
|
|
328
|
+
* The distinction decides how long to keep trying. A 429 or a 500 is a provider that is up and
|
|
329
|
+
* telling you something; a connection that cannot be made is usually the link, and the link usually
|
|
330
|
+
* comes back. Carries no status by definition — an HTTP status means an HTTP response happened.
|
|
331
|
+
*/
|
|
332
|
+
/**
|
|
333
|
+
* Waits, unless the turn is stopped first.
|
|
334
|
+
*
|
|
335
|
+
* A plain setTimeout is fine for a second and wrong for thirty: while waiting out an outage the
|
|
336
|
+
* stop button and ctrl-c have to work, and a timer that cannot be interrupted turns "stop" into
|
|
337
|
+
* "stop in half a minute". The listener is removed either way, so a long run does not accumulate
|
|
338
|
+
* one per attempt.
|
|
339
|
+
*/
|
|
340
|
+
export function sleepUnlessAborted(ms, signal) {
|
|
341
|
+
if (signal?.aborted)
|
|
342
|
+
return Promise.resolve();
|
|
343
|
+
return new Promise(resolve => {
|
|
344
|
+
const done = () => {
|
|
345
|
+
clearTimeout(timer);
|
|
346
|
+
signal?.removeEventListener('abort', done);
|
|
347
|
+
resolve();
|
|
348
|
+
};
|
|
349
|
+
const timer = setTimeout(done, ms);
|
|
350
|
+
signal?.addEventListener('abort', done, { once: true });
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
export function isConnectivityFailure(err) {
|
|
354
|
+
const e = err;
|
|
355
|
+
if (typeof e?.status === 'number')
|
|
356
|
+
return false;
|
|
357
|
+
const text = `${e?.message ?? ''} ${e?.code ?? ''}`.toLowerCase();
|
|
358
|
+
if (!text.trim())
|
|
359
|
+
return false;
|
|
360
|
+
return /connection error|econnrefused|econnreset|econnaborted|etimedout|enotfound|eai_again|ehostunreach|enetunreach|enetdown|epipe|socket hang up|fetch failed|network|dns|tunnel|offline/
|
|
361
|
+
.test(text);
|
|
362
|
+
}
|
|
244
363
|
export function retryDelayMs(err, attempt, random = Math.random) {
|
|
245
364
|
const text = String(err?.message ?? '');
|
|
246
365
|
const headers = err?.headers;
|
|
@@ -1053,6 +1172,14 @@ detectedContext = { current: 0 }) {
|
|
|
1053
1172
|
let duplicateCalls = 0;
|
|
1054
1173
|
let usedMutatingTool = false;
|
|
1055
1174
|
let turnRetries = 0;
|
|
1175
|
+
/**
|
|
1176
|
+
* When the connection first went away, or 0 while it is up.
|
|
1177
|
+
*
|
|
1178
|
+
* Kept so the offline budget is measured in wall-clock time rather than in attempts: with a
|
|
1179
|
+
* backoff, "thirty attempts" is a different amount of patience early in an outage than late in
|
|
1180
|
+
* one, and what somebody means by "wait for the network" is an amount of time.
|
|
1181
|
+
*/
|
|
1182
|
+
let offlineSince = 0;
|
|
1056
1183
|
let emptyTurns = 0;
|
|
1057
1184
|
/** Retries spent on an empty turn from a model already known to use tools. */
|
|
1058
1185
|
let emptyRetries = 0;
|
|
@@ -1138,6 +1265,31 @@ detectedContext = { current: 0 }) {
|
|
|
1138
1265
|
const file = filePathFrom(args);
|
|
1139
1266
|
return file ? { kind: 'file', target: file } : null;
|
|
1140
1267
|
}
|
|
1268
|
+
/**
|
|
1269
|
+
* Whether KONECK could actually authenticate as this candidate.
|
|
1270
|
+
*
|
|
1271
|
+
* Asked before failing over, because moving to a provider with no key is not a recovery — it is
|
|
1272
|
+
* the same failure again with an extra step, and everything the run spawns afterwards inherits
|
|
1273
|
+
* it. A run that failed over twice and then lost two sub-agents to "a provider credential error"
|
|
1274
|
+
* is what this is for.
|
|
1275
|
+
*
|
|
1276
|
+
* A provider serving locally needs nothing, and is usable by definition. Anything that throws
|
|
1277
|
+
* while being resolved — an unknown name, an endpoint that cannot be worked out — is not usable,
|
|
1278
|
+
* which is the right answer for a candidate KONECK cannot even describe.
|
|
1279
|
+
*/
|
|
1280
|
+
function canAuthenticate(candidate) {
|
|
1281
|
+
const name = candidate.provider ?? config.provider;
|
|
1282
|
+
try {
|
|
1283
|
+
const resolved = resolveProvider(name);
|
|
1284
|
+
if (needsNoCredential(resolved))
|
|
1285
|
+
return true;
|
|
1286
|
+
getApiKey(resolved);
|
|
1287
|
+
return true;
|
|
1288
|
+
}
|
|
1289
|
+
catch {
|
|
1290
|
+
return false;
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1141
1293
|
async function tryFailover(err) {
|
|
1142
1294
|
if (!failoverPolicy?.enabled)
|
|
1143
1295
|
return false;
|
|
@@ -1160,7 +1312,7 @@ detectedContext = { current: 0 }) {
|
|
|
1160
1312
|
}
|
|
1161
1313
|
}
|
|
1162
1314
|
const from = { provider: config.provider, model: config.model };
|
|
1163
|
-
const to = pickNext(failoverPolicy, from, switchedTo, discovered);
|
|
1315
|
+
const to = pickNext(failoverPolicy, from, switchedTo, discovered, Math.random, canAuthenticate);
|
|
1164
1316
|
if (!to)
|
|
1165
1317
|
return false;
|
|
1166
1318
|
if (failoverPolicy.decide === 'ask') {
|
|
@@ -1500,6 +1652,21 @@ detectedContext = { current: 0 }) {
|
|
|
1500
1652
|
// keeps its streaming return type; dropped for the run if this model refuses it.
|
|
1501
1653
|
...paceField,
|
|
1502
1654
|
}, { signal: combinedSignal });
|
|
1655
|
+
/*
|
|
1656
|
+
* The request went through, so the trouble is over.
|
|
1657
|
+
*
|
|
1658
|
+
* These used to be set once for the life of the run and never cleared, which made the retry
|
|
1659
|
+
* budget a lifetime allowance: a run that survived three hiccups in its first ten minutes had
|
|
1660
|
+
* nothing left for hour two, and the fourth blip on turn ninety ended it. Reset here, where
|
|
1661
|
+
* the provider has demonstrably answered, so the counts mean "trouble now".
|
|
1662
|
+
*/
|
|
1663
|
+
if (offlineSince !== 0) {
|
|
1664
|
+
offlineSince = 0;
|
|
1665
|
+
config.onOffline?.(null);
|
|
1666
|
+
if (!silent)
|
|
1667
|
+
tui.printWarning(`Reconnected to ${config.provider} — carrying on.`);
|
|
1668
|
+
}
|
|
1669
|
+
turnRetries = 0;
|
|
1503
1670
|
// A per-turn AbortSignal guards the request, but once the SDK has the response and is
|
|
1504
1671
|
// reading the body an abort is not reliably observed, so a silent connection can hang the
|
|
1505
1672
|
// whole run — a turn sat idle for eight minutes under a five minute timeout. This watchdog
|
|
@@ -1869,18 +2036,60 @@ detectedContext = { current: 0 }) {
|
|
|
1869
2036
|
turn--; // do this turn again
|
|
1870
2037
|
continue;
|
|
1871
2038
|
}
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
2039
|
+
/*
|
|
2040
|
+
* The network being gone is waited out; a provider being unhappy is retried a few times.
|
|
2041
|
+
*
|
|
2042
|
+
* These were the same path with the same budget of three, which meant a wifi drop or a
|
|
2043
|
+
* tunnel restart — minutes, ordinarily — ended a run that had been going for an hour. What is
|
|
2044
|
+
* wanted there is what a person would do: keep trying, and carry on when it comes back.
|
|
2045
|
+
*
|
|
2046
|
+
* The budget also used to be spent for the life of the run rather than per turn. A run that
|
|
2047
|
+
* survived three hiccups in its first ten minutes had nothing left for hour two, and the
|
|
2048
|
+
* fourth blip on turn ninety ended it. It is reset on every turn that gets through, so the
|
|
2049
|
+
* count means "trouble now" rather than "trouble ever".
|
|
2050
|
+
*/
|
|
2051
|
+
if (isRetryableProviderError(err)) {
|
|
2052
|
+
const offline = isConnectivityFailure(err);
|
|
2053
|
+
if (offline && offlineSince === 0)
|
|
2054
|
+
offlineSince = Date.now();
|
|
2055
|
+
const waitedFor = offline ? Date.now() - offlineSince : 0;
|
|
2056
|
+
const mayRetry = offline
|
|
2057
|
+
? waitedFor < OFFLINE_RETRY_WINDOW_MS
|
|
2058
|
+
: turnRetries < MAX_TURN_RETRIES;
|
|
2059
|
+
if (mayRetry) {
|
|
2060
|
+
turnRetries++;
|
|
2061
|
+
const delay = offline
|
|
2062
|
+
? Math.min(retryDelayMs(err, Math.min(turnRetries, 6)), OFFLINE_MAX_BACKOFF_MS)
|
|
2063
|
+
: retryDelayMs(err, turnRetries);
|
|
2064
|
+
if (spinner) {
|
|
2065
|
+
spinner.fail(offline
|
|
2066
|
+
? `No connection to ${config.provider} — waiting for it to come back (attempt ${turnRetries})`
|
|
2067
|
+
: `Provider error on turn ${stats.turns}, retry ${turnRetries}/${MAX_TURN_RETRIES}`);
|
|
2068
|
+
}
|
|
2069
|
+
if (!silent) {
|
|
2070
|
+
tui.printWarning(offline
|
|
2071
|
+
// Said in terms of what is happening and what will happen, since this is the message
|
|
2072
|
+
// somebody reads while wondering whether to give up and start again.
|
|
2073
|
+
? `Cannot reach ${config.provider} — the connection looks down. Attempt `
|
|
2074
|
+
+ `${turnRetries}, trying again in ${Math.round(delay / 1000)}s. The run picks up `
|
|
2075
|
+
+ `where it left off once the network is back; stop it if you would rather not wait.`
|
|
2076
|
+
: `${describeProviderError(err, config.provider, config.model, { hadImage: carriesImage(messages) }).split('\n')[0]} — retrying (${turnRetries}/${MAX_TURN_RETRIES}).`);
|
|
2077
|
+
}
|
|
2078
|
+
config.onOffline?.(offline
|
|
2079
|
+
? { provider: config.provider, attempt: turnRetries, nextInMs: delay, downForMs: waitedFor }
|
|
2080
|
+
: null);
|
|
2081
|
+
// Interruptible: a wait measured in minutes must not swallow a stop.
|
|
2082
|
+
await sleepUnlessAborted(delay, combinedSignal);
|
|
2083
|
+
if (combinedSignal.aborted)
|
|
2084
|
+
break;
|
|
2085
|
+
turn--; // do this turn again
|
|
2086
|
+
continue;
|
|
2087
|
+
}
|
|
2088
|
+
if (offline && !silent) {
|
|
2089
|
+
tui.printWarning(`Still cannot reach ${config.provider} after ${Math.round(waitedFor / 60000)} minutes `
|
|
2090
|
+
+ `and ${turnRetries} attempts. Giving up on this turn — the conversation is kept, so `
|
|
2091
|
+
+ `saying "continue" once the network is back carries on from here.`);
|
|
1880
2092
|
}
|
|
1881
|
-
await new Promise(r => setTimeout(r, retryDelayMs(err, turnRetries)));
|
|
1882
|
-
turn--; // do this turn again
|
|
1883
|
-
continue;
|
|
1884
2093
|
}
|
|
1885
2094
|
if (spinner)
|
|
1886
2095
|
spinner.fail('API error on turn ' + stats.turns);
|