instar 1.3.251 → 1.3.252
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/providers/adapters/gemini-cli/observability/geminiCapacityPolicy.d.ts +10 -0
- package/dist/providers/adapters/gemini-cli/observability/geminiCapacityPolicy.d.ts.map +1 -1
- package/dist/providers/adapters/gemini-cli/observability/geminiCapacityPolicy.js +58 -2
- package/dist/providers/adapters/gemini-cli/observability/geminiCapacityPolicy.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +2 -2
- package/upgrades/1.3.252.md +56 -0
- package/upgrades/side-effects/gemini-model-fallback-on-exhaustion.md +81 -0
|
@@ -44,11 +44,21 @@ export interface GeminiCapacityGate {
|
|
|
44
44
|
export declare function isGeminiCapacityError(message: string | null | undefined): boolean;
|
|
45
45
|
export declare function parseGeminiRetryAfterMs(message: string | null | undefined): number | undefined;
|
|
46
46
|
export declare function resolveKnownGeminiFallback(requestedModel: string, config: GeminiCapacityPolicyConfig | undefined): string;
|
|
47
|
+
/**
|
|
48
|
+
* Pick a known Gemini model to switch to when `exhaustedModel` hit its capacity
|
|
49
|
+
* limit. A candidate qualifies when it is (a) a known Gemini model, (b) not the
|
|
50
|
+
* just-exhausted model, and (c) not itself inside a recorded exhaustion window
|
|
51
|
+
* at `now`. Prefers an operator-configured fallback when it qualifies, otherwise
|
|
52
|
+
* the first known model with headroom. Returns undefined when every known model
|
|
53
|
+
* is exhausted — the genuine account-wide block case.
|
|
54
|
+
*/
|
|
55
|
+
export declare function pickGeminiFallbackModel(exhaustedModel: string, config: GeminiCapacityPolicyConfig | undefined, now: number): string | undefined;
|
|
47
56
|
export declare function decideGeminiCapacityPolicy(params: {
|
|
48
57
|
errorMessage: string;
|
|
49
58
|
attempt: number;
|
|
50
59
|
model: string;
|
|
51
60
|
config?: GeminiCapacityPolicyConfig;
|
|
61
|
+
now?: number;
|
|
52
62
|
}): GeminiCapacityDecision;
|
|
53
63
|
export declare function getGeminiCapacityGate(now?: number): GeminiCapacityGate;
|
|
54
64
|
export declare function recordGeminiCapacityDeferral(params: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"geminiCapacityPolicy.d.ts","sourceRoot":"","sources":["../../../../../src/providers/adapters/gemini-cli/observability/geminiCapacityPolicy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAOH,MAAM,WAAW,0BAA0B;IACzC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,yEAAyE;IACzE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kFAAkF;IAClF,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,oBAAoB,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC;AAE9D,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,oBAAoB,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,OAAO,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;
|
|
1
|
+
{"version":3,"file":"geminiCapacityPolicy.d.ts","sourceRoot":"","sources":["../../../../../src/providers/adapters/gemini-cli/observability/geminiCapacityPolicy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAOH,MAAM,WAAW,0BAA0B;IACzC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,yEAAyE;IACzE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kFAAkF;IAClF,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,oBAAoB,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC;AAE9D,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,oBAAoB,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,OAAO,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAsBD,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAajF;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAoB9F;AAOD,wBAAgB,0BAA0B,CACxC,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,0BAA0B,GAAG,SAAS,GAC7C,MAAM,CAIR;AAED;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CACrC,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,0BAA0B,GAAG,SAAS,EAC9C,GAAG,EAAE,MAAM,GACV,MAAM,GAAG,SAAS,CAWpB;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,0BAA0B,CAAC;IACpC,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,GAAG,sBAAsB,CAgDzB;AAED,wBAAgB,qBAAqB,CAAC,GAAG,SAAa,GAAG,kBAAkB,CAU1E;AAED,wBAAgB,4BAA4B,CAAC,MAAM,EAAE;IACnD,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,GAAG,kBAAkB,CAsBrB;AA0CD,wBAAgB,iCAAiC,IAAI,IAAI,CAIxD"}
|
|
@@ -11,17 +11,27 @@
|
|
|
11
11
|
* - later calls are refused locally while deferred, avoiding doomed subprocesses;
|
|
12
12
|
* - fallback models must be in the verified Gemini model set.
|
|
13
13
|
*/
|
|
14
|
-
import { GEMINI_DEFAULT_MODEL, isKnownGeminiModel } from '../models.js';
|
|
14
|
+
import { GEMINI_DEFAULT_MODEL, isKnownGeminiModel, KNOWN_GEMINI_MODELS } from '../models.js';
|
|
15
15
|
import fs from 'node:fs';
|
|
16
16
|
import path from 'node:path';
|
|
17
17
|
import { SafeFsExecutor } from '../../../../core/SafeFsExecutor.js';
|
|
18
18
|
const DEFAULT_IMMEDIATE_RETRIES = 1;
|
|
19
19
|
const DEFAULT_IMMEDIATE_RETRY_MAX_MS = 30_000;
|
|
20
20
|
const DEFAULT_BACKOFF_MS = 5_000;
|
|
21
|
+
// Switching to a DIFFERENT model (separate quota) needs no real backoff — the
|
|
22
|
+
// wait the exhausted model wanted does not apply to the fresh one. Use a tiny
|
|
23
|
+
// delay only to avoid a tight loop in pathological cases.
|
|
24
|
+
const DEFAULT_MODEL_SWITCH_BACKOFF_MS = 250;
|
|
21
25
|
const DEFAULT_UNKNOWN_RESET_MS = 15 * 60_000;
|
|
22
26
|
const RESET_GRACE_MS = 5_000;
|
|
23
27
|
let deferredUntil = 0;
|
|
24
28
|
let deferredReason = null;
|
|
29
|
+
// Per-model exhaustion windows (model id → epoch ms its quota is expected to
|
|
30
|
+
// reset). The known Gemini models draw on SEPARATE quotas, so a single model
|
|
31
|
+
// exhausting is NOT an account-wide block: we record the exhausted model here
|
|
32
|
+
// and switch to a model with headroom before globally deferring. Time-based, so
|
|
33
|
+
// entries self-clear once a model's reported reset window has passed.
|
|
34
|
+
const modelExhaustedUntil = new Map();
|
|
25
35
|
export function isGeminiCapacityError(message) {
|
|
26
36
|
if (!message)
|
|
27
37
|
return false;
|
|
@@ -64,6 +74,27 @@ export function resolveKnownGeminiFallback(requestedModel, config) {
|
|
|
64
74
|
return requestedModel;
|
|
65
75
|
return fallback;
|
|
66
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* Pick a known Gemini model to switch to when `exhaustedModel` hit its capacity
|
|
79
|
+
* limit. A candidate qualifies when it is (a) a known Gemini model, (b) not the
|
|
80
|
+
* just-exhausted model, and (c) not itself inside a recorded exhaustion window
|
|
81
|
+
* at `now`. Prefers an operator-configured fallback when it qualifies, otherwise
|
|
82
|
+
* the first known model with headroom. Returns undefined when every known model
|
|
83
|
+
* is exhausted — the genuine account-wide block case.
|
|
84
|
+
*/
|
|
85
|
+
export function pickGeminiFallbackModel(exhaustedModel, config, now) {
|
|
86
|
+
const hasHeadroom = (m) => !!m &&
|
|
87
|
+
isKnownGeminiModel(m) &&
|
|
88
|
+
m !== exhaustedModel &&
|
|
89
|
+
(modelExhaustedUntil.get(m) ?? 0) <= now;
|
|
90
|
+
if (hasHeadroom(config?.fallbackModel))
|
|
91
|
+
return config?.fallbackModel;
|
|
92
|
+
for (const m of KNOWN_GEMINI_MODELS) {
|
|
93
|
+
if (hasHeadroom(m))
|
|
94
|
+
return m;
|
|
95
|
+
}
|
|
96
|
+
return undefined;
|
|
97
|
+
}
|
|
67
98
|
export function decideGeminiCapacityPolicy(params) {
|
|
68
99
|
const { errorMessage, attempt, model, config } = params;
|
|
69
100
|
if (config?.enabled === false || !isGeminiCapacityError(errorMessage)) {
|
|
@@ -83,11 +114,29 @@ export function decideGeminiCapacityPolicy(params) {
|
|
|
83
114
|
};
|
|
84
115
|
}
|
|
85
116
|
const deferMs = retryAfterMs ?? DEFAULT_UNKNOWN_RESET_MS;
|
|
117
|
+
const now = params.now ?? Date.now();
|
|
118
|
+
// Per-model exhaustion is NOT an account-wide block — the known Gemini models
|
|
119
|
+
// (flash, pro) draw on SEPARATE quotas. Record this model's window, then switch
|
|
120
|
+
// to a model with headroom before globally deferring. The switch also means we
|
|
121
|
+
// do NOT write the global stop-state for a single-model exhaustion (the caller
|
|
122
|
+
// only records a deferral on action:'defer'), which is what made instar report
|
|
123
|
+
// a fully-available account as "blocked". Mirrors the codex auto-swap policy.
|
|
124
|
+
modelExhaustedUntil.set(model, now + deferMs);
|
|
125
|
+
const fallback = pickGeminiFallbackModel(model, config, now);
|
|
126
|
+
if (fallback) {
|
|
127
|
+
return {
|
|
128
|
+
action: 'retry',
|
|
129
|
+
retryAfterMs: DEFAULT_MODEL_SWITCH_BACKOFF_MS,
|
|
130
|
+
model: fallback,
|
|
131
|
+
reason: `gemini model ${model} exhausted; switching to ${fallback} (separate quota may have headroom)`,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
// Every known model is exhausted — a genuine account-wide block.
|
|
86
135
|
return {
|
|
87
136
|
action: 'defer',
|
|
88
137
|
retryAfterMs: deferMs,
|
|
89
138
|
model,
|
|
90
|
-
reason: `gemini capacity exhausted; deferring calls for ${deferMs}ms`,
|
|
139
|
+
reason: `gemini capacity exhausted on all known models; deferring calls for ${deferMs}ms`,
|
|
91
140
|
};
|
|
92
141
|
}
|
|
93
142
|
export function getGeminiCapacityGate(now = Date.now()) {
|
|
@@ -126,6 +175,12 @@ function writeGeminiQuotaState(params) {
|
|
|
126
175
|
usagePercent: 0,
|
|
127
176
|
fiveHourPercent: 100,
|
|
128
177
|
source: 'gemini-cli-capacity',
|
|
178
|
+
// 'account': this stop-state is only written once EVERY known model is
|
|
179
|
+
// exhausted (the genuine account-wide block). A single model exhausting no
|
|
180
|
+
// longer reaches here — the policy switches to a model with headroom first —
|
|
181
|
+
// so a reader can trust `recommendation:'stop'` here means the agent really
|
|
182
|
+
// is out of capacity, not just one model.
|
|
183
|
+
scope: 'account',
|
|
129
184
|
model: params.model,
|
|
130
185
|
blockedUntil: new Date(params.blockedUntil).toISOString(),
|
|
131
186
|
blockReason: params.reason,
|
|
@@ -152,5 +207,6 @@ function writeGeminiQuotaState(params) {
|
|
|
152
207
|
export function resetGeminiCapacityPolicyForTests() {
|
|
153
208
|
deferredUntil = 0;
|
|
154
209
|
deferredReason = null;
|
|
210
|
+
modelExhaustedUntil.clear();
|
|
155
211
|
}
|
|
156
212
|
//# sourceMappingURL=geminiCapacityPolicy.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"geminiCapacityPolicy.js","sourceRoot":"","sources":["../../../../../src/providers/adapters/gemini-cli/observability/geminiCapacityPolicy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"geminiCapacityPolicy.js","sourceRoot":"","sources":["../../../../../src/providers/adapters/gemini-cli/observability/geminiCapacityPolicy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAC7F,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAoCpE,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAC9C,MAAM,kBAAkB,GAAG,KAAK,CAAC;AACjC,8EAA8E;AAC9E,8EAA8E;AAC9E,0DAA0D;AAC1D,MAAM,+BAA+B,GAAG,GAAG,CAAC;AAC5C,MAAM,wBAAwB,GAAG,EAAE,GAAG,MAAM,CAAC;AAC7C,MAAM,cAAc,GAAG,KAAK,CAAC;AAE7B,IAAI,aAAa,GAAG,CAAC,CAAC;AACtB,IAAI,cAAc,GAAkB,IAAI,CAAC;AAEzC,6EAA6E;AAC7E,6EAA6E;AAC7E,8EAA8E;AAC9E,gFAAgF;AAChF,sEAAsE;AACtE,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;AAEtD,MAAM,UAAU,qBAAqB,CAAC,OAAkC;IACtE,IAAI,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC3B,MAAM,CAAC,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAChC,OAAO,CACL,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QACjB,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAC7B,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC;QAChC,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC;QAChC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7B,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;QACrB,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QAC/B,6BAA6B,CAAC,IAAI,CAAC,CAAC,CAAC,CACtC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,OAAkC;IACxE,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,MAAM,CAAC,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAEhC,IAAI,KAAK,GAAG,+MAA+M,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpO,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5F,CAAC;IAED,KAAK,GAAG,0KAA0K,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3L,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,KAAK,GAAG,8HAA8H,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/I,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,IAAI,CAAC,OAAe;IAC3B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IAChE,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,cAAsB,EACtB,MAA8C;IAE9C,MAAM,QAAQ,GAAG,MAAM,EAAE,aAAa,CAAC;IACvC,IAAI,CAAC,QAAQ,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC;QAAE,OAAO,cAAc,CAAC;IACtE,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CACrC,cAAsB,EACtB,MAA8C,EAC9C,GAAW;IAEX,MAAM,WAAW,GAAG,CAAC,CAAqB,EAAe,EAAE,CACzD,CAAC,CAAC,CAAC;QACH,kBAAkB,CAAC,CAAC,CAAC;QACrB,CAAC,KAAK,cAAc;QACpB,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC;IAC3C,IAAI,WAAW,CAAC,MAAM,EAAE,aAAa,CAAC;QAAE,OAAO,MAAM,EAAE,aAAa,CAAC;IACrE,KAAK,MAAM,CAAC,IAAI,mBAAmB,EAAE,CAAC;QACpC,IAAI,WAAW,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,MAM1C;IACC,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IACxD,IAAI,MAAM,EAAE,OAAO,KAAK,KAAK,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,EAAE,CAAC;QACtE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACjD,CAAC;IAED,MAAM,YAAY,GAAG,uBAAuB,CAAC,YAAY,CAAC,CAAC;IAC3D,MAAM,mBAAmB,GAAG,MAAM,EAAE,mBAAmB,IAAI,yBAAyB,CAAC;IACrF,MAAM,mBAAmB,GAAG,MAAM,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;IAE1F,IACE,OAAO,GAAG,mBAAmB;QAC7B,CAAC,YAAY,KAAK,SAAS,IAAI,YAAY,IAAI,mBAAmB,CAAC,EACnE,CAAC;QACD,MAAM,IAAI,GAAG,YAAY,IAAI,MAAM,EAAE,SAAS,IAAI,kBAAkB,CAAC;QACrE,OAAO;YACL,MAAM,EAAE,OAAO;YACf,YAAY,EAAE,IAAI;YAClB,KAAK,EAAE,0BAA0B,CAAC,KAAK,EAAE,MAAM,CAAC;YAChD,MAAM,EAAE,yCAAyC,IAAI,IAAI;SAC1D,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,YAAY,IAAI,wBAAwB,CAAC;IACzD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;IACrC,8EAA8E;IAC9E,gFAAgF;IAChF,+EAA+E;IAC/E,+EAA+E;IAC/E,+EAA+E;IAC/E,8EAA8E;IAC9E,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;IAC9C,MAAM,QAAQ,GAAG,uBAAuB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7D,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO;YACL,MAAM,EAAE,OAAO;YACf,YAAY,EAAE,+BAA+B;YAC7C,KAAK,EAAE,QAAQ;YACf,MAAM,EAAE,gBAAgB,KAAK,4BAA4B,QAAQ,qCAAqC;SACvG,CAAC;IACJ,CAAC;IACD,iEAAiE;IACjE,OAAO;QACL,MAAM,EAAE,OAAO;QACf,YAAY,EAAE,OAAO;QACrB,KAAK;QACL,MAAM,EAAE,sEAAsE,OAAO,IAAI;KAC1F,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;IACpD,IAAI,aAAa,IAAI,GAAG,EAAE,CAAC;QACzB,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC7E,CAAC;IACD,OAAO;QACL,KAAK,EAAE,KAAK;QACZ,YAAY,EAAE,aAAa,GAAG,GAAG;QACjC,aAAa;QACb,MAAM,EAAE,cAAc;KACvB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,MAM5C;IACC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;IACrC,aAAa,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,GAAG,cAAc,CAAC;IACxE,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC;IAC/B,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QAC1B,IAAI,CAAC;YACH,qBAAqB,CAAC;gBACpB,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,YAAY,EAAE,aAAa;gBAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,oBAAoB;gBAC3C,GAAG;aACJ,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CACV,2DACE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CACjD,EAAE,CACH,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,qBAAqB,CAAC,GAAG,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,qBAAqB,CAAC,MAM9B;IACC,MAAM,KAAK,GAAG;QACZ,YAAY,EAAE,CAAC;QACf,eAAe,EAAE,GAAG;QACpB,MAAM,EAAE,qBAAqB;QAC7B,uEAAuE;QACvE,2EAA2E;QAC3E,6EAA6E;QAC7E,4EAA4E;QAC5E,0CAA0C;QAC1C,KAAK,EAAE,SAAS;QAChB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,YAAY,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE;QACzD,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,WAAW,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE;QAC/C,cAAc,EAAE,MAAM;KACvB,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAChD,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,cAAc,IAAI,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;IACjG,IAAI,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACtD,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC;YACH,cAAc,CAAC,cAAc,CAAC,GAAG,EAAE;gBACjC,SAAS,EAAE,oDAAoD;aAChE,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC,CAAC,yBAAyB,CAAC,CAAC;QACrC,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iCAAiC;IAC/C,aAAa,GAAG,CAAC,CAAC;IAClB,cAAc,GAAG,IAAI,CAAC;IACtB,mBAAmB,CAAC,KAAK,EAAE,CAAC;AAC9B,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "./builtin-manifest.schema.json",
|
|
3
3
|
"schemaVersion": 1,
|
|
4
|
-
"generatedAt": "2026-06-
|
|
5
|
-
"instarVersion": "1.3.
|
|
4
|
+
"generatedAt": "2026-06-04T23:01:14.429Z",
|
|
5
|
+
"instarVersion": "1.3.252",
|
|
6
6
|
"entryCount": 198,
|
|
7
7
|
"entries": {
|
|
8
8
|
"hook:session-start": {
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: patch -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
When a Gemini call hit a per-model capacity limit (e.g. "You have exhausted your
|
|
9
|
+
capacity on this model. Quota resets after 46m"), instar's Gemini capacity policy
|
|
10
|
+
globally deferred on that model and wrote a `quota-state.json` snapshot with
|
|
11
|
+
`recommendation: 'stop'` / `fiveHourPercent: 100`. But the known Gemini models
|
|
12
|
+
(`gemini-2.5-flash`, `gemini-2.5-pro`) draw on SEPARATE quotas, so one model
|
|
13
|
+
exhausting was mis-recorded as a full account-wide block — and every reader of that
|
|
14
|
+
file (the agent, the mentor, the escalation monitor) concluded the whole Gemini
|
|
15
|
+
agent was down, even when the account had headroom on the other model. This produced
|
|
16
|
+
repeated false "agent is blocked" reports.
|
|
17
|
+
|
|
18
|
+
The policy now switches to a known model with headroom and retries before deferring
|
|
19
|
+
(mirroring the codex auto-swap-on-limit policy). Only when EVERY known model is
|
|
20
|
+
exhausted does it globally defer and write the stop-state — now tagged
|
|
21
|
+
`scope: 'account'`. The per-model exhaustion windows are tracked time-based and
|
|
22
|
+
self-clear, so there is no switch loop and no stale state. The model switch uses a
|
|
23
|
+
short 250ms backoff (a different model needs no wait).
|
|
24
|
+
|
|
25
|
+
## What to Tell Your User
|
|
26
|
+
|
|
27
|
+
If you run a Gemini-backed agent: when one Gemini model hits its limit, the agent now
|
|
28
|
+
quietly switches to the other model and keeps working, instead of going dark for the
|
|
29
|
+
length of that model's reset window. It only reports itself as out of capacity when
|
|
30
|
+
every model is genuinely exhausted. Nothing to do — it just stops the false "blocked"
|
|
31
|
+
reports and keeps the agent productive.
|
|
32
|
+
|
|
33
|
+
## Summary of New Capabilities
|
|
34
|
+
|
|
35
|
+
Gemini agents auto-fall-back across models on a per-model capacity limit (parity with
|
|
36
|
+
the codex auto-swap), and the durable quota-state now distinguishes a single-model
|
|
37
|
+
deferral from a genuine account-wide block (`scope: 'account'`).
|
|
38
|
+
|
|
39
|
+
## Evidence
|
|
40
|
+
|
|
41
|
+
- **Reproduction (live this session):** a Gemini agent's `gemini -p` probe returned
|
|
42
|
+
"You have exhausted your capacity on this model. Your quota will reset after 46m11s"
|
|
43
|
+
while the account `/stats` showed 20% remaining on the Auto model — i.e. one model
|
|
44
|
+
exhausted, account fine. The old policy wrote `recommendation: 'stop'` for this,
|
|
45
|
+
which read as a full block.
|
|
46
|
+
- **Before:** single-model exhaustion → global defer + `recommendation:'stop'` /
|
|
47
|
+
`fiveHourPercent:100`; the agent appeared fully blocked for the whole reset window.
|
|
48
|
+
- **After:** single-model exhaustion → switch to the model with headroom and keep
|
|
49
|
+
working; global stop-state (now `scope:'account'`) is written ONLY when every known
|
|
50
|
+
model is exhausted.
|
|
51
|
+
- **Tests:** 21 across three tiers — `tests/unit/geminiCapacityPolicy.test.ts`
|
|
52
|
+
(switch-then-defer, single-model-no-global-stop, `pickGeminiFallbackModel`,
|
|
53
|
+
self-clearing windows), `tests/integration/gemini-capacity-policy-integration.test.ts`,
|
|
54
|
+
and `tests/e2e/gemini-capacity-policy-lifecycle.test.ts` (both updated to assert the
|
|
55
|
+
two-model attempt + account scope), plus the escalation-monitor suite. `tsc --noEmit`
|
|
56
|
+
clean.
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# Side-Effects Review — Gemini per-model exhaustion switches model before deferring
|
|
2
|
+
|
|
3
|
+
**Version / slug:** `gemini-model-fallback-on-exhaustion`
|
|
4
|
+
**Date:** `2026-06-04`
|
|
5
|
+
**Author:** `echo`
|
|
6
|
+
**Second-pass reviewer:** `required (LLM-routing behavior change)`
|
|
7
|
+
|
|
8
|
+
## Summary of the change
|
|
9
|
+
|
|
10
|
+
When a Gemini call hit a capacity limit, `decideGeminiCapacityPolicy` (after its
|
|
11
|
+
short-window immediate-retry path) went straight to a GLOBAL `defer` on the SAME
|
|
12
|
+
model, and `recordGeminiCapacityDeferral` wrote a `quota-state.json` snapshot with
|
|
13
|
+
`fiveHourPercent: 100` / `recommendation: 'stop'`. But the known Gemini models —
|
|
14
|
+
`gemini-2.5-flash` and `gemini-2.5-pro` — draw on **separate** quotas. So one model
|
|
15
|
+
exhausting (e.g. "You have exhausted your capacity on this model. Quota resets after
|
|
16
|
+
46m") was recorded as a full account-wide block, and any reader of `quota-state.json`
|
|
17
|
+
(including the agent itself, the mentor, the escalation monitor) saw `recommendation:
|
|
18
|
+
'stop'` and concluded the whole Gemini agent was down — even when the account had
|
|
19
|
+
headroom on the other model. This caused repeated false "Gemi is blocked" reports
|
|
20
|
+
(confirmed live this session: a `gemini -p` probe returned that exact error while
|
|
21
|
+
`/stats` showed the account at 20% remaining on Auto).
|
|
22
|
+
|
|
23
|
+
The fix (mirrors the existing codex auto-swap-on-limit policy, #31): on per-model
|
|
24
|
+
exhaustion, record the exhausted model's reset window, then switch to a known model
|
|
25
|
+
with headroom and retry. Only when **every** known model is exhausted do we globally
|
|
26
|
+
defer and write the stop-state — now tagged `scope: 'account'`. Because the caller
|
|
27
|
+
records a deferral only on `action: 'defer'`, switching models also means a single-
|
|
28
|
+
model exhaustion no longer writes a global stop at all.
|
|
29
|
+
|
|
30
|
+
## Decision-point inventory
|
|
31
|
+
|
|
32
|
+
1. **`decideGeminiCapacityPolicy` (long-reset branch):** record `model` as exhausted
|
|
33
|
+
(time-based, `now + deferMs`); pick a fallback via `pickGeminiFallbackModel`; if
|
|
34
|
+
one exists → `retry` with that model; else → `defer`.
|
|
35
|
+
2. **`pickGeminiFallbackModel`:** a candidate qualifies when known, ≠ the exhausted
|
|
36
|
+
model, and not itself inside a recorded exhaustion window at `now`. Prefers an
|
|
37
|
+
operator-configured `fallbackModel`, else the first known model with headroom.
|
|
38
|
+
|
|
39
|
+
## 1. Over-block (what still defers / blocks)
|
|
40
|
+
|
|
41
|
+
- When ALL known models are in an exhaustion window → `defer` + the global stop-state
|
|
42
|
+
(now `scope: 'account'`). That is the genuine account-wide block and still gates
|
|
43
|
+
scheduler spawns exactly as before. (tests: "defers once ALL models exhausted",
|
|
44
|
+
integration "defers after a long quota reset", e2e lifecycle.)
|
|
45
|
+
- The short-window immediate-retry path (attempt < maxImmediateRetries, small reset)
|
|
46
|
+
is unchanged.
|
|
47
|
+
- Non-capacity errors are unchanged — `decide` returns `none` and the error surfaces.
|
|
48
|
+
|
|
49
|
+
## 2. Under-block (what it now allows that it didn't)
|
|
50
|
+
|
|
51
|
+
- A single model exhausting now triggers ONE extra Gemini spawn on the other model
|
|
52
|
+
(the switch) before any deferral. Worst-case cost when BOTH models are exhausted:
|
|
53
|
+
two spawns instead of one (flash then pro) before the global defer — bounded at 2
|
|
54
|
+
(one per known model; each is recorded exhausted so it is never retried within its
|
|
55
|
+
window). The model-switch retry uses a 250ms backoff (not the 5s same-model
|
|
56
|
+
backoff) since a different model needs no wait.
|
|
57
|
+
- Loop-safety: `modelExhaustedUntil` is time-based and self-clears once a window
|
|
58
|
+
passes, so there is no permanent state and no A→B→A switch loop (each model is
|
|
59
|
+
recorded before fallback selection; once both are recorded, `pickGeminiFallbackModel`
|
|
60
|
+
returns undefined → defer). (tests: "returns undefined once both exhausted",
|
|
61
|
+
"windows self-clear once the reset passes".)
|
|
62
|
+
|
|
63
|
+
## 3. Blast radius
|
|
64
|
+
|
|
65
|
+
- One file of logic: `src/providers/adapters/gemini-cli/observability/geminiCapacityPolicy.ts`
|
|
66
|
+
(new module state `modelExhaustedUntil`, new export `pickGeminiFallbackModel`, a
|
|
67
|
+
`now?` param on `decide`, the switch-before-defer branch, `scope` on the stop-state,
|
|
68
|
+
reset clears the map).
|
|
69
|
+
- NO consumer signature changes: both callers (`transport/oneShotCompletion.ts`,
|
|
70
|
+
`core/GeminiCliIntelligenceProvider.ts`) already switch to `decision.model` on
|
|
71
|
+
`action:'retry'` and record a deferral only on `action:'defer'`.
|
|
72
|
+
- Affects only Gemini-CLI agents. Claude / codex paths untouched. The added
|
|
73
|
+
`scope` field is additive (readers that don't know it are unaffected).
|
|
74
|
+
|
|
75
|
+
## 4. Reversibility
|
|
76
|
+
|
|
77
|
+
Fully reversible: revert the one policy file + the test updates. Module state is
|
|
78
|
+
in-process and time-based (no persistence); the `quota-state.json` shape only gains
|
|
79
|
+
an additive `scope` field. Verified: `tsc --noEmit` clean; 21 tests green across
|
|
80
|
+
unit + integration + e2e + escalation tiers (both the switch path and the genuine
|
|
81
|
+
all-models-defer path).
|