dsh-agy-link 0.6.56 → 0.6.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +111 -117
- package/dist/cli-image.mjs +665 -291
- package/dist/client.js +7 -2
- package/dist/index.js +894 -284
- package/docs/ARCHITECTURE.md +4 -4
- package/docs/CONFIGURATION.md +20 -14
- package/docs/KNOWN-GAPS.md +2 -8
- package/docs/README.en.md +76 -30
- package/docs/SECURITY-NOTES.md +2 -0
- package/docs/assets/console.png +0 -0
- package/docs/assets/context-metering.png +0 -0
- package/docs/assets/model-picker.png +0 -0
- package/docs/assets/settings.png +0 -0
- package/docs/assets/token-usage.png +0 -0
- package/package.json +1 -1
package/dist/cli-image.mjs
CHANGED
|
@@ -457,7 +457,7 @@ function extractAuthUrl(text) {
|
|
|
457
457
|
*/
|
|
458
458
|
function looksLikeHardRateLimit(text) {
|
|
459
459
|
if (!text) return false;
|
|
460
|
-
return /RESOURCE_EXHAUSTED|code[
|
|
460
|
+
return /RESOURCE_EXHAUSTED|\bQUOTA_EXHAUSTED\b|"?code"?\s*[:=]\s*429\b|"?status"?\s*[:=]\s*429\b|HTTP(?:\/[\d.]+)?[ :]?429\b|too many requests|individual quota reached|quota (?:exceeded|reached|exhausted|limit)|rate[ -]?limit(?:ed)? (?:exceeded|reached|hit)?|exceeded (?:your |the )?quota|has been exhausted|exhausted your (?:quota|capacity)|out of quota|check quota|insufficient quota|usage limit reached/i.test(text);
|
|
461
461
|
}
|
|
462
462
|
/**
|
|
463
463
|
* Failures that belong to the ACCOUNT (spent quota, refused region, broken
|
|
@@ -491,18 +491,19 @@ function looksLikeRateLimit(text) {
|
|
|
491
491
|
}
|
|
492
492
|
/**
|
|
493
493
|
* Parse reset duration in milliseconds from rate-limit / quota-exhausted error strings.
|
|
494
|
-
* Supports compact formats ("Resets in 21m25s", "
|
|
495
|
-
* verbose formats ("Resets in 15 minutes",
|
|
494
|
+
* Supports compact formats ("Resets in 21m25s", "reset after 4h58m46s",
|
|
495
|
+
* `"quotaResetDelay": "4h58m46.438s"`), verbose formats ("Resets in 15 minutes",
|
|
496
|
+
* "retry after 30 seconds"), and ISO timestamps.
|
|
496
497
|
*/
|
|
497
498
|
function parseResetDurationMs(text) {
|
|
498
499
|
if (!text) return void 0;
|
|
499
|
-
const compactMatch = text.match(/resets?\s+in\s
|
|
500
|
-
if (compactMatch && compactMatch[
|
|
500
|
+
const compactMatch = text.match(/(?:resets?\s+(?:in|after)|"(?:quotaResetDelay|retryDelay)"\s*:\s*")\s*((?:(\d+)\s*h)?\s*(?:(\d+)\s*m)?\s*(?:(\d+(?:\.\d+)?)\s*s)?)/i);
|
|
501
|
+
if (compactMatch && (compactMatch[2] || compactMatch[3] || compactMatch[4])) {
|
|
501
502
|
const hours = parseInt(compactMatch[2] || "0", 10);
|
|
502
503
|
const minutes = parseInt(compactMatch[3] || "0", 10);
|
|
503
|
-
const seconds =
|
|
504
|
-
const totalMs = (hours * 3600 + minutes * 60 + seconds) * 1e3;
|
|
505
|
-
if (totalMs > 0) return totalMs;
|
|
504
|
+
const seconds = parseFloat(compactMatch[4] || "0");
|
|
505
|
+
const totalMs = Math.round((hours * 3600 + minutes * 60 + seconds) * 1e3);
|
|
506
|
+
if (Number.isFinite(totalMs) && totalMs > 0) return totalMs;
|
|
506
507
|
}
|
|
507
508
|
const wordMatch = text.match(/(?:resets?|retry)\s+(?:in|after)\s+(\d+)\s*(hour|hr|minute|min|second|sec)s?/i);
|
|
508
509
|
if (wordMatch) {
|
|
@@ -518,6 +519,45 @@ function parseResetDurationMs(text) {
|
|
|
518
519
|
if (!Number.isNaN(parsed) && parsed > Date.now()) return parsed - Date.now();
|
|
519
520
|
}
|
|
520
521
|
}
|
|
522
|
+
const TRANSIENT_BURST_CAP_MS = 3e4;
|
|
523
|
+
/**
|
|
524
|
+
* True when a 429 error represents a short transient per-minute/per-second QPS burst
|
|
525
|
+
* (`reason: RATE_LIMIT_EXCEEDED` with reset <= 30s) rather than window quota exhaustion.
|
|
526
|
+
* Must NOT trigger long account cooldown or kill a running parent conversation.
|
|
527
|
+
*/
|
|
528
|
+
function isTransientBurstRateLimit(text) {
|
|
529
|
+
if (!text) return false;
|
|
530
|
+
if (!/\bRATE_LIMIT_EXCEEDED\b/i.test(text)) return false;
|
|
531
|
+
if (/\bQUOTA_EXHAUSTED\b|individual quota reached/i.test(text)) return false;
|
|
532
|
+
const resetMs = parseResetDurationMs(text);
|
|
533
|
+
return resetMs !== void 0 && resetMs > 0 && resetMs <= TRANSIENT_BURST_CAP_MS;
|
|
534
|
+
}
|
|
535
|
+
/** Extract Google RPC error metadata model slug (e.g. "gemini-3.1-flash-image"), if present. */
|
|
536
|
+
function extractQuotaErrorModel(text) {
|
|
537
|
+
if (!text) return void 0;
|
|
538
|
+
return text.match(/"model"\s*:\s*"([A-Za-z0-9._:/-]+)"/i)?.[1];
|
|
539
|
+
}
|
|
540
|
+
/**
|
|
541
|
+
* True when a failed result envelope carries a hard quota refusal (meaning the
|
|
542
|
+
* attempt must rotate accounts rather than soft-finish on any partial text).
|
|
543
|
+
*/
|
|
544
|
+
function isHardQuotaResultError(ok, error, response) {
|
|
545
|
+
if (ok) return false;
|
|
546
|
+
if (error && error.trim() !== "") return looksLikeHardRateLimit(error) && !isTransientBurstRateLimit(error);
|
|
547
|
+
return Boolean(response && looksLikeHardRateLimit(response) && !isTransientBurstRateLimit(response));
|
|
548
|
+
}
|
|
549
|
+
/**
|
|
550
|
+
* True when a failed result envelope represents a non-consumable account/policy
|
|
551
|
+
* failure even when partial response text was already buffered before the error.
|
|
552
|
+
*/
|
|
553
|
+
function isNonConsumableResultError(ok, error, response) {
|
|
554
|
+
if (ok) return false;
|
|
555
|
+
if (isHardQuotaResultError(ok, error, response)) return true;
|
|
556
|
+
if (looksLikeTosViolation(error) || looksLikeTosViolation(response)) return true;
|
|
557
|
+
if (looksLikeValidationRequired(error) || looksLikeValidationRequired(response)) return true;
|
|
558
|
+
if (error && (looksLikeEligibilityFailure(error) || looksLikeAuthFailure(error))) return true;
|
|
559
|
+
return false;
|
|
560
|
+
}
|
|
521
561
|
//#endregion
|
|
522
562
|
//#region src/common/config.ts
|
|
523
563
|
function dshHome() {
|
|
@@ -1159,6 +1199,21 @@ function simplifyRatio(width, height) {
|
|
|
1159
1199
|
}
|
|
1160
1200
|
//#endregion
|
|
1161
1201
|
//#region src/features/image/service.ts
|
|
1202
|
+
function isHardQuotaErrorText(text) {
|
|
1203
|
+
return Boolean(text && text.trim() !== "" && looksLikeHardRateLimit(text) && !isTransientBurstRateLimit(text) && !looksLikeTosViolation(text));
|
|
1204
|
+
}
|
|
1205
|
+
function detectHardQuotaError(res, replyError) {
|
|
1206
|
+
if (!res.ok) {
|
|
1207
|
+
const err = res.error ?? "";
|
|
1208
|
+
if (looksLikeTosViolation(err)) return null;
|
|
1209
|
+
if (res.code === Err.RATE_LIMIT || res.code === Err.QUOTA || isHardQuotaErrorText(err)) return isTransientBurstRateLimit(err) ? null : err || "Antigravity quota exhausted";
|
|
1210
|
+
return null;
|
|
1211
|
+
}
|
|
1212
|
+
const toolHit = res.toolCalls.find((t) => (t.name === "generate_image" || t.name === "invoke_subagent") && isHardQuotaErrorText(t.error));
|
|
1213
|
+
if (toolHit?.error) return toolHit.error;
|
|
1214
|
+
if (isHardQuotaErrorText(replyError)) return replyError;
|
|
1215
|
+
return null;
|
|
1216
|
+
}
|
|
1162
1217
|
function emptyImageResult(req, model, effort) {
|
|
1163
1218
|
return {
|
|
1164
1219
|
ok: false,
|
|
@@ -1316,6 +1371,8 @@ var ImageService = class {
|
|
|
1316
1371
|
let last = ctx.last;
|
|
1317
1372
|
let effort = target.effort;
|
|
1318
1373
|
let effortFallbackUsed = false;
|
|
1374
|
+
let quotaHopsUsed = 0;
|
|
1375
|
+
const maxQuotaHops = Math.min(Math.max(0, Math.round(this.deps.poolQuotaHops?.() ?? 0)), 16);
|
|
1319
1376
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
1320
1377
|
const startedAt = this.now();
|
|
1321
1378
|
last.effort = effort;
|
|
@@ -1328,9 +1385,34 @@ var ImageService = class {
|
|
|
1328
1385
|
imagePaths: refs,
|
|
1329
1386
|
...signal === void 0 ? {} : { signal }
|
|
1330
1387
|
});
|
|
1331
|
-
last.attempts = attempt;
|
|
1388
|
+
last.attempts = attempt + quotaHopsUsed;
|
|
1332
1389
|
last.durationMs = this.now() - startedAt;
|
|
1333
1390
|
if (res.conversationId !== void 0) last.conversationId = res.conversationId;
|
|
1391
|
+
if (!res.ok && looksLikeTosViolation(res.error)) return {
|
|
1392
|
+
...last,
|
|
1393
|
+
ok: false,
|
|
1394
|
+
code: "AGY_ERROR",
|
|
1395
|
+
error: res.error ?? "account suspended (TOS_VIOLATION)"
|
|
1396
|
+
};
|
|
1397
|
+
const reply = res.ok ? parseImageReply(res.text) : { error: void 0 };
|
|
1398
|
+
const quotaErr = detectHardQuotaError(res, reply.error);
|
|
1399
|
+
if (quotaErr !== null) {
|
|
1400
|
+
last = {
|
|
1401
|
+
...last,
|
|
1402
|
+
ok: false,
|
|
1403
|
+
code: "NO_ACCOUNT",
|
|
1404
|
+
error: quotaErr
|
|
1405
|
+
};
|
|
1406
|
+
const errModel = extractQuotaErrorModel(quotaErr) ?? target.model;
|
|
1407
|
+
const canSwitch = this.deps.onQuotaExhausted ? this.deps.onQuotaExhausted(res.accountId, errModel, quotaErr) : true;
|
|
1408
|
+
if (canSwitch && quotaHopsUsed < maxQuotaHops) {
|
|
1409
|
+
quotaHopsUsed++;
|
|
1410
|
+
attempt--;
|
|
1411
|
+
continue;
|
|
1412
|
+
}
|
|
1413
|
+
if (canSwitch && attempt < maxAttempts) continue;
|
|
1414
|
+
return last;
|
|
1415
|
+
}
|
|
1334
1416
|
if (!res.ok) {
|
|
1335
1417
|
const failure = res.error ?? "";
|
|
1336
1418
|
if (!effortFallbackUsed && effort !== "" && /--effort/i.test(failure) && /not supported/i.test(failure)) {
|
|
@@ -1360,7 +1442,6 @@ var ImageService = class {
|
|
|
1360
1442
|
if (attempt < maxAttempts) continue;
|
|
1361
1443
|
return last;
|
|
1362
1444
|
}
|
|
1363
|
-
const reply = parseImageReply(res.text);
|
|
1364
1445
|
if (reply.error !== void 0) {
|
|
1365
1446
|
last = {
|
|
1366
1447
|
...last,
|
|
@@ -1559,6 +1640,14 @@ async function runViaAdapter(adapter, call) {
|
|
|
1559
1640
|
agyImagePrompt: call.prompt,
|
|
1560
1641
|
agyPermissionMode: call.permissionMode,
|
|
1561
1642
|
agyOnAccount: (info) => {
|
|
1643
|
+
if (accountId !== void 0 && typeof info.id === "string" && info.id !== accountId) {
|
|
1644
|
+
toolCalls.length = 0;
|
|
1645
|
+
subagents.length = 0;
|
|
1646
|
+
textParts.length = 0;
|
|
1647
|
+
responseText = "";
|
|
1648
|
+
conversationId = void 0;
|
|
1649
|
+
failure = null;
|
|
1650
|
+
}
|
|
1562
1651
|
if (typeof info.id === "string") accountId = info.id;
|
|
1563
1652
|
if (typeof info.home === "string" && info.home !== "") home = info.home;
|
|
1564
1653
|
},
|
|
@@ -1628,121 +1717,6 @@ async function runViaAdapter(adapter, call) {
|
|
|
1628
1717
|
};
|
|
1629
1718
|
}
|
|
1630
1719
|
//#endregion
|
|
1631
|
-
//#region src/common/pool-types.ts
|
|
1632
|
-
/** Check if a model identifier belongs to the Gemini 4 Argon family. */
|
|
1633
|
-
function isArgonModel(modelId) {
|
|
1634
|
-
if (!modelId) return false;
|
|
1635
|
-
return modelId.trim().toLowerCase().includes("argon");
|
|
1636
|
-
}
|
|
1637
|
-
/**
|
|
1638
|
-
* Map a model slug to its backend quota counter family.
|
|
1639
|
-
*
|
|
1640
|
-
* Gemini 4 Argon (Pro/Ultra-only) bills against its OWN weekly quota pool —
|
|
1641
|
-
* never the plain Gemini one. Its ids look like `gemini-4-argon-low/-medium`
|
|
1642
|
-
* and MUST be detected before the generic `gemini-` prefix below, otherwise
|
|
1643
|
-
* argon usage would silently drain/report the Gemini counter.
|
|
1644
|
-
*/
|
|
1645
|
-
function modelFamilyOf(modelId) {
|
|
1646
|
-
if (!modelId) return "unknown";
|
|
1647
|
-
const id = modelId.toLowerCase();
|
|
1648
|
-
if (id.includes("argon")) return "argon";
|
|
1649
|
-
if (id.startsWith("claude-") || id.includes("claude")) return "anthropic";
|
|
1650
|
-
if (id.startsWith("gemini-") || id.startsWith("gemma-") || id.includes("gemini")) return "google";
|
|
1651
|
-
if (id.startsWith("gpt-") || id.startsWith("openai/") || id.includes("gpt-oss")) return "openai";
|
|
1652
|
-
return "unknown";
|
|
1653
|
-
}
|
|
1654
|
-
/**
|
|
1655
|
-
* Determine whether a managed account has confirmed access to the Gemini 4 Argon family.
|
|
1656
|
-
*
|
|
1657
|
-
* An account qualifies for Argon when:
|
|
1658
|
-
* 1. It carries an authoritative Argon quota bucket.
|
|
1659
|
-
* 2. It has a dedicated Argon group recorded by the pool.
|
|
1660
|
-
* 3. Its Google per-model quota breakdown includes an Argon model.
|
|
1661
|
-
* 4. Or its discovered catalog includes Argon (when no negative quota breakdown is present).
|
|
1662
|
-
*/
|
|
1663
|
-
function accountHasArgon(acc, pool, catalog) {
|
|
1664
|
-
if (!acc) return false;
|
|
1665
|
-
if (acc.quotas?.argon !== void 0) return true;
|
|
1666
|
-
if (pool && typeof pool.hasDedicatedGroup === "function" && pool.hasDedicatedGroup(acc.id, "argon")) return true;
|
|
1667
|
-
if (acc.quotas?.google?.models?.some((m) => isArgonModel(m.modelId))) return true;
|
|
1668
|
-
if (Array.isArray(acc.quotas?.google?.models)) return false;
|
|
1669
|
-
if (catalog && catalog.source === "discovered") return catalog.models?.some((m) => isArgonModel(m.id)) ?? false;
|
|
1670
|
-
return false;
|
|
1671
|
-
}
|
|
1672
|
-
const EFFORT_SUFFIX_RE = /-(?:tiered-)?(?:low|medium|high|xhigh|tiered|thinking)$/i;
|
|
1673
|
-
/**
|
|
1674
|
-
* Return the un-tiered base model slug (e.g. claude-opus-4-6-thinking -> claude-opus-4-6,
|
|
1675
|
-
* claude-opus-5-5-high -> claude-opus-5-5).
|
|
1676
|
-
*/
|
|
1677
|
-
function baseModelSlug(id) {
|
|
1678
|
-
const s = id.trim().toLowerCase().replace(/\./g, "-");
|
|
1679
|
-
if (s === "claude-opus-4-6" || s === "claude-opus-4-8" || s === "claude-opus-4-5") return "claude-opus-4-6";
|
|
1680
|
-
if (s === "claude-sonnet-4-5") return "claude-sonnet-4-6";
|
|
1681
|
-
return s.replace(EFFORT_SUFFIX_RE, "");
|
|
1682
|
-
}
|
|
1683
|
-
/**
|
|
1684
|
-
* Test whether a requested model matches a candidate model id (accounting for
|
|
1685
|
-
* aliases, effort variants, and base model families).
|
|
1686
|
-
*/
|
|
1687
|
-
function modelMatches(requestedModel, candidateModelId) {
|
|
1688
|
-
if (!requestedModel || !candidateModelId) return false;
|
|
1689
|
-
const r = requestedModel.trim().toLowerCase();
|
|
1690
|
-
const c = candidateModelId.trim().toLowerCase();
|
|
1691
|
-
if (r === c) return true;
|
|
1692
|
-
return baseModelSlug(r) === baseModelSlug(c);
|
|
1693
|
-
}
|
|
1694
|
-
function quotaSupportsModel(acc, clean, pool, catalog) {
|
|
1695
|
-
const familyQuotaModels = acc.quotas?.[modelFamilyOf(clean)]?.models;
|
|
1696
|
-
if (Array.isArray(familyQuotaModels) && familyQuotaModels.some((m) => modelMatches(clean, m.modelId))) return true;
|
|
1697
|
-
const lower = clean.toLowerCase();
|
|
1698
|
-
if (lower === "gemini-pro-orchestrator-neon") return Array.isArray(familyQuotaModels) && familyQuotaModels.some((m) => modelMatches("gemini-4-neon", m.modelId));
|
|
1699
|
-
if (lower === "gemini-pro-orchestrator-g1-pro") return accountHasArgon(acc, pool, catalog);
|
|
1700
|
-
return false;
|
|
1701
|
-
}
|
|
1702
|
-
/**
|
|
1703
|
-
* Determine whether a managed account supports a specific model.
|
|
1704
|
-
* If the account's quota or discovered catalog enumerates models for that family,
|
|
1705
|
-
* the list is authoritative: the model must match an available entry.
|
|
1706
|
-
*/
|
|
1707
|
-
function accountSupportsModel(acc, model, pool, catalog) {
|
|
1708
|
-
if (!acc) return false;
|
|
1709
|
-
if (!model || model.trim() === "") return true;
|
|
1710
|
-
const clean = model.trim();
|
|
1711
|
-
if (isArgonModel(clean)) return accountHasArgon(acc, pool, catalog);
|
|
1712
|
-
if (catalog && catalog.source === "discovered" && Array.isArray(catalog.models) && catalog.models.length > 0) {
|
|
1713
|
-
if (catalog.models.some((m) => modelMatches(clean, m.id))) return true;
|
|
1714
|
-
if (quotaSupportsModel(acc, clean, pool, catalog)) return true;
|
|
1715
|
-
return false;
|
|
1716
|
-
}
|
|
1717
|
-
const family = modelFamilyOf(clean);
|
|
1718
|
-
const familyQuota = acc.quotas?.[family];
|
|
1719
|
-
if (familyQuota) {
|
|
1720
|
-
if (Array.isArray(familyQuota.models) && familyQuota.models.length > 0) return quotaSupportsModel(acc, clean, pool, catalog);
|
|
1721
|
-
return true;
|
|
1722
|
-
}
|
|
1723
|
-
if (acc.quotas && Object.keys(acc.quotas).length > 0 && acc.quotas.google !== void 0) return false;
|
|
1724
|
-
return true;
|
|
1725
|
-
}
|
|
1726
|
-
/**
|
|
1727
|
-
* A flagged account whose flag is old enough (or whose flag predates the
|
|
1728
|
-
* authMarkedAt bookkeeping) may be re-verified in the background. If the
|
|
1729
|
-
* stored token still works the flag clears itself; if not, the flag is
|
|
1730
|
-
* simply re-stamped.
|
|
1731
|
-
*/
|
|
1732
|
-
/** A terminal account ban (Google TOS_VIOLATION) is not a transient auth error. */
|
|
1733
|
-
function isTerminalAuthError(message) {
|
|
1734
|
-
return typeof message === "string" && /\bTOS_VIOLATION\b|violation of Terms of Service/i.test(message);
|
|
1735
|
-
}
|
|
1736
|
-
function defaultPoolData() {
|
|
1737
|
-
return {
|
|
1738
|
-
version: 1,
|
|
1739
|
-
mode: "sequential",
|
|
1740
|
-
defaultCooldownMs: 9e5,
|
|
1741
|
-
maxCooldownMs: 36e5,
|
|
1742
|
-
accounts: []
|
|
1743
|
-
};
|
|
1744
|
-
}
|
|
1745
|
-
//#endregion
|
|
1746
1720
|
//#region src/runtime/agy/bin.ts
|
|
1747
1721
|
const IS_WIN$3 = process.platform === "win32";
|
|
1748
1722
|
/** Executable candidates for one PATH entry, per-platform. Exported for tests. */
|
|
@@ -1999,6 +1973,9 @@ function renderCompactionMarker(notice) {
|
|
|
1999
1973
|
//#endregion
|
|
2000
1974
|
//#region src/host/recording.ts
|
|
2001
1975
|
const waiters = Symbol("waiters");
|
|
1976
|
+
function isConsumableResult(ev) {
|
|
1977
|
+
return (ev.ok || ev.response !== "") && !isNonConsumableResultError(ev.ok, ev.error, ev.response);
|
|
1978
|
+
}
|
|
2002
1979
|
/** One agy run's append-only event log plus its settlement state. */
|
|
2003
1980
|
var RunRecording = class {
|
|
2004
1981
|
now;
|
|
@@ -2154,10 +2131,10 @@ var RunRecording = class {
|
|
|
2154
2131
|
get hasResult() {
|
|
2155
2132
|
return this.events.some((e) => e.kind === "result");
|
|
2156
2133
|
}
|
|
2157
|
-
/** Whether a result event that EventMapper will finish on (ok, or non-empty error response) was recorded. */
|
|
2134
|
+
/** Whether a result event that EventMapper will finish on (ok, or non-empty consumable error response) was recorded. */
|
|
2158
2135
|
get hasConsumableResult() {
|
|
2159
2136
|
const r = this.getResultEvent();
|
|
2160
|
-
return r !== null && (r
|
|
2137
|
+
return r !== null && isConsumableResult(r);
|
|
2161
2138
|
}
|
|
2162
2139
|
/**
|
|
2163
2140
|
* Whether any recorded event at or after `from` can finish a span cleanly
|
|
@@ -2170,7 +2147,7 @@ var RunRecording = class {
|
|
|
2170
2147
|
for (let i = Math.max(0, from); i < this.events.length; i++) {
|
|
2171
2148
|
const ev = this.events[i];
|
|
2172
2149
|
if (ev === void 0) continue;
|
|
2173
|
-
if (ev.kind === "result" && (ev
|
|
2150
|
+
if (ev.kind === "result" && isConsumableResult(ev)) return true;
|
|
2174
2151
|
if (cutOnTool && ev.kind === "step" && ev.stepKind === "tool" && ev.tool !== void 0 && (ev.state === "DONE" || ev.state === "ERROR" || ev.state === "CANCELED" || ev.state === "CANCELLED" || ev.state === "RUNNING" || ev.state === void 0 && (ev.tool.output !== void 0 || ev.tool.error !== void 0))) return true;
|
|
2175
2152
|
}
|
|
2176
2153
|
return false;
|
|
@@ -2181,22 +2158,21 @@ var RunRecording = class {
|
|
|
2181
2158
|
const ev = this.events[i];
|
|
2182
2159
|
if (ev !== void 0 && ev.kind === "result") return {
|
|
2183
2160
|
ok: ev.ok,
|
|
2184
|
-
response: ev.response
|
|
2161
|
+
response: ev.response,
|
|
2162
|
+
...ev.error === void 0 ? {} : { error: ev.error }
|
|
2185
2163
|
};
|
|
2186
2164
|
}
|
|
2187
2165
|
return null;
|
|
2188
2166
|
}
|
|
2189
2167
|
/**
|
|
2190
2168
|
* True when at least one consumable `result` event (`ok` or non-empty
|
|
2191
|
-
* `response
|
|
2192
|
-
* `failureInfo === null` and no consumable result at `>= from` would always
|
|
2193
|
-
* fail with `agy stream ended without a result event` if re-driven.
|
|
2169
|
+
* `response`, excluding hard quota / policy errors) exists at index >= `from`.
|
|
2194
2170
|
*/
|
|
2195
2171
|
hasConsumableResultFrom(from) {
|
|
2196
2172
|
const start = Math.max(0, from);
|
|
2197
2173
|
for (let i = start; i < this.events.length; i++) {
|
|
2198
2174
|
const ev = this.events[i];
|
|
2199
|
-
if (ev !== void 0 && ev.kind === "result" && (ev
|
|
2175
|
+
if (ev !== void 0 && ev.kind === "result" && isConsumableResult(ev)) return true;
|
|
2200
2176
|
}
|
|
2201
2177
|
return false;
|
|
2202
2178
|
}
|
|
@@ -3032,7 +3008,7 @@ function startAgyProcess(opts) {
|
|
|
3032
3008
|
},
|
|
3033
3009
|
kill: (reason) => {
|
|
3034
3010
|
if (reason === "timeout") state.timedOut = true;
|
|
3035
|
-
else state.aborted = true;
|
|
3011
|
+
else if (reason === "abort") state.aborted = true;
|
|
3036
3012
|
if (state.watchdog) clearTimeout(state.watchdog);
|
|
3037
3013
|
if (state.turnCap) clearTimeout(state.turnCap);
|
|
3038
3014
|
if (state.idleGraceTimer) clearTimeout(state.idleGraceTimer);
|
|
@@ -4590,7 +4566,7 @@ var EventMapper = class {
|
|
|
4590
4566
|
return;
|
|
4591
4567
|
}
|
|
4592
4568
|
if (!ev.ok) {
|
|
4593
|
-
if (ev.response === "") return;
|
|
4569
|
+
if (ev.response === "" || isNonConsumableResultError(ev.ok, ev.error, ev.response)) return;
|
|
4594
4570
|
const respText = this.formatAssistantText(ev.response);
|
|
4595
4571
|
if (!this.sawTextStep) {
|
|
4596
4572
|
yield* this.ensureBlock("text");
|
|
@@ -5777,6 +5753,121 @@ var SpawnLimiter = class {
|
|
|
5777
5753
|
}
|
|
5778
5754
|
};
|
|
5779
5755
|
//#endregion
|
|
5756
|
+
//#region src/common/pool-types.ts
|
|
5757
|
+
/** Check if a model identifier belongs to the Gemini 4 Argon family. */
|
|
5758
|
+
function isArgonModel(modelId) {
|
|
5759
|
+
if (!modelId) return false;
|
|
5760
|
+
return modelId.trim().toLowerCase().includes("argon");
|
|
5761
|
+
}
|
|
5762
|
+
/**
|
|
5763
|
+
* Map a model slug to its backend quota counter family.
|
|
5764
|
+
*
|
|
5765
|
+
* Gemini 4 Argon (Pro/Ultra-only) bills against its OWN weekly quota pool —
|
|
5766
|
+
* never the plain Gemini one. Its ids look like `gemini-4-argon-low/-medium`
|
|
5767
|
+
* and MUST be detected before the generic `gemini-` prefix below, otherwise
|
|
5768
|
+
* argon usage would silently drain/report the Gemini counter.
|
|
5769
|
+
*/
|
|
5770
|
+
function modelFamilyOf(modelId) {
|
|
5771
|
+
if (!modelId) return "unknown";
|
|
5772
|
+
const id = modelId.toLowerCase();
|
|
5773
|
+
if (id.includes("argon")) return "argon";
|
|
5774
|
+
if (id.startsWith("claude-") || id.includes("claude")) return "anthropic";
|
|
5775
|
+
if (id.startsWith("gemini-") || id.startsWith("gemma-") || id.includes("gemini")) return "google";
|
|
5776
|
+
if (id.startsWith("gpt-") || id.startsWith("openai/") || id.includes("gpt-oss")) return "openai";
|
|
5777
|
+
return "unknown";
|
|
5778
|
+
}
|
|
5779
|
+
/**
|
|
5780
|
+
* Determine whether a managed account has confirmed access to the Gemini 4 Argon family.
|
|
5781
|
+
*
|
|
5782
|
+
* An account qualifies for Argon when:
|
|
5783
|
+
* 1. It carries an authoritative Argon quota bucket.
|
|
5784
|
+
* 2. It has a dedicated Argon group recorded by the pool.
|
|
5785
|
+
* 3. Its Google per-model quota breakdown includes an Argon model.
|
|
5786
|
+
* 4. Or its discovered catalog includes Argon (when no negative quota breakdown is present).
|
|
5787
|
+
*/
|
|
5788
|
+
function accountHasArgon(acc, pool, catalog) {
|
|
5789
|
+
if (!acc) return false;
|
|
5790
|
+
if (acc.quotas?.argon !== void 0) return true;
|
|
5791
|
+
if (pool && typeof pool.hasDedicatedGroup === "function" && pool.hasDedicatedGroup(acc.id, "argon")) return true;
|
|
5792
|
+
if (acc.quotas?.google?.models?.some((m) => isArgonModel(m.modelId))) return true;
|
|
5793
|
+
if (Array.isArray(acc.quotas?.google?.models)) return false;
|
|
5794
|
+
if (catalog && catalog.source === "discovered") return catalog.models?.some((m) => isArgonModel(m.id)) ?? false;
|
|
5795
|
+
return false;
|
|
5796
|
+
}
|
|
5797
|
+
const EFFORT_SUFFIX_RE = /-(?:tiered-)?(?:low|medium|high|xhigh|tiered|thinking)$/i;
|
|
5798
|
+
/**
|
|
5799
|
+
* Return the un-tiered base model slug (e.g. claude-opus-4-6-thinking -> claude-opus-4-6,
|
|
5800
|
+
* claude-opus-5-5-high -> claude-opus-5-5).
|
|
5801
|
+
*/
|
|
5802
|
+
function baseModelSlug(id) {
|
|
5803
|
+
const s = id.trim().toLowerCase().replace(/\./g, "-");
|
|
5804
|
+
if (s === "claude-opus-4-6" || s === "claude-opus-4-8" || s === "claude-opus-4-5") return "claude-opus-4-6";
|
|
5805
|
+
if (s === "claude-sonnet-4-5") return "claude-sonnet-4-6";
|
|
5806
|
+
return s.replace(EFFORT_SUFFIX_RE, "");
|
|
5807
|
+
}
|
|
5808
|
+
/**
|
|
5809
|
+
* Test whether a requested model matches a candidate model id (accounting for
|
|
5810
|
+
* aliases, effort variants, and base model families).
|
|
5811
|
+
*/
|
|
5812
|
+
function modelMatches(requestedModel, candidateModelId) {
|
|
5813
|
+
if (!requestedModel || !candidateModelId) return false;
|
|
5814
|
+
const r = requestedModel.trim().toLowerCase();
|
|
5815
|
+
const c = candidateModelId.trim().toLowerCase();
|
|
5816
|
+
if (r === c) return true;
|
|
5817
|
+
return baseModelSlug(r) === baseModelSlug(c);
|
|
5818
|
+
}
|
|
5819
|
+
function quotaSupportsModel(acc, clean, pool, catalog) {
|
|
5820
|
+
const familyQuotaModels = acc.quotas?.[modelFamilyOf(clean)]?.models;
|
|
5821
|
+
if (Array.isArray(familyQuotaModels) && familyQuotaModels.some((m) => modelMatches(clean, m.modelId))) return true;
|
|
5822
|
+
const lower = clean.toLowerCase();
|
|
5823
|
+
if (lower === "gemini-pro-orchestrator-neon") return Array.isArray(familyQuotaModels) && familyQuotaModels.some((m) => modelMatches("gemini-4-neon", m.modelId));
|
|
5824
|
+
if (lower === "gemini-pro-orchestrator-g1-pro") return accountHasArgon(acc, pool, catalog);
|
|
5825
|
+
return false;
|
|
5826
|
+
}
|
|
5827
|
+
/**
|
|
5828
|
+
* Determine whether a managed account supports a specific model.
|
|
5829
|
+
* If the account's quota or discovered catalog enumerates models for that family,
|
|
5830
|
+
* the list is authoritative: the model must match an available entry.
|
|
5831
|
+
*/
|
|
5832
|
+
function accountSupportsModel(acc, model, pool, catalog) {
|
|
5833
|
+
if (!acc) return false;
|
|
5834
|
+
if (!model || model.trim() === "") return true;
|
|
5835
|
+
const clean = model.trim();
|
|
5836
|
+
if (isArgonModel(clean)) return accountHasArgon(acc, pool, catalog);
|
|
5837
|
+
if (catalog && catalog.source === "discovered" && Array.isArray(catalog.models) && catalog.models.length > 0) {
|
|
5838
|
+
if (catalog.models.some((m) => modelMatches(clean, m.id))) return true;
|
|
5839
|
+
if (quotaSupportsModel(acc, clean, pool, catalog)) return true;
|
|
5840
|
+
return false;
|
|
5841
|
+
}
|
|
5842
|
+
const family = modelFamilyOf(clean);
|
|
5843
|
+
const familyQuota = acc.quotas?.[family];
|
|
5844
|
+
if (familyQuota) {
|
|
5845
|
+
if (Array.isArray(familyQuota.models) && familyQuota.models.length > 0) return quotaSupportsModel(acc, clean, pool, catalog);
|
|
5846
|
+
return true;
|
|
5847
|
+
}
|
|
5848
|
+
if (acc.quotas && Object.keys(acc.quotas).length > 0 && acc.quotas.google !== void 0) return false;
|
|
5849
|
+
return true;
|
|
5850
|
+
}
|
|
5851
|
+
/**
|
|
5852
|
+
* A flagged account whose flag is old enough (or whose flag predates the
|
|
5853
|
+
* authMarkedAt bookkeeping) may be re-verified in the background. If the
|
|
5854
|
+
* stored token still works the flag clears itself; if not, the flag is
|
|
5855
|
+
* simply re-stamped.
|
|
5856
|
+
*/
|
|
5857
|
+
/** A terminal account ban (Google TOS_VIOLATION) is not a transient auth error. */
|
|
5858
|
+
function isTerminalAuthError(message) {
|
|
5859
|
+
return typeof message === "string" && /\bTOS_VIOLATION\b|violation of Terms of Service/i.test(message);
|
|
5860
|
+
}
|
|
5861
|
+
function defaultPoolData() {
|
|
5862
|
+
return {
|
|
5863
|
+
version: 1,
|
|
5864
|
+
mode: "sequential",
|
|
5865
|
+
defaultCooldownMs: 9e5,
|
|
5866
|
+
maxCooldownMs: 36e5,
|
|
5867
|
+
accounts: []
|
|
5868
|
+
};
|
|
5869
|
+
}
|
|
5870
|
+
//#endregion
|
|
5780
5871
|
//#region src/common/atomic-write.ts
|
|
5781
5872
|
let seq = 0;
|
|
5782
5873
|
/** Unique temp path beside `file` (same directory, so rename stays atomic). */
|
|
@@ -6695,15 +6786,32 @@ function providerRetryPolicyFor(pool) {
|
|
|
6695
6786
|
Err.OVERLOADED
|
|
6696
6787
|
];
|
|
6697
6788
|
if (enabledAccounts >= 2) retryableCodes.push(Err.AUTH);
|
|
6789
|
+
const poolHops = Math.min(Math.max(1, enabledAccounts - 1), 16);
|
|
6698
6790
|
return {
|
|
6699
6791
|
mode: "normal",
|
|
6700
|
-
maxRetries: Math.max(
|
|
6792
|
+
maxRetries: Math.max(poolHops, NETWORK_RETRY_DELAYS_MS.length),
|
|
6701
6793
|
retryableCodes,
|
|
6702
6794
|
initialDelayMs: 2e3,
|
|
6703
6795
|
maxDelayMs: 45e3,
|
|
6704
6796
|
jitterRatio: .1
|
|
6705
6797
|
};
|
|
6706
6798
|
}
|
|
6799
|
+
function isAuxCall(options) {
|
|
6800
|
+
return (options.purpose ?? "chat") !== "chat" || typeof options.system === "string" && options.system.includes("REVIEW_POLICY");
|
|
6801
|
+
}
|
|
6802
|
+
function canRetryAuxQuota(deps, options, emitted, finish, quotaHops, rawMsg) {
|
|
6803
|
+
if (!isAuxCall(options) || emitted || options.signal?.aborted) return false;
|
|
6804
|
+
if (finish.reason.kind !== "error" || looksLikeTosViolation(rawMsg)) return false;
|
|
6805
|
+
const code = finish.reason.failure?.code;
|
|
6806
|
+
if (code !== Err.RATE_LIMIT && code !== Err.ELIGIBILITY && code !== Err.AUTH) return false;
|
|
6807
|
+
const pool = deps.pool;
|
|
6808
|
+
if (!pool) return false;
|
|
6809
|
+
const enabledCount = pool.getAccounts().filter((a) => a.enabled).length;
|
|
6810
|
+
if (quotaHops >= Math.min(Math.max(0, enabledCount - 1), 16)) return false;
|
|
6811
|
+
const model = options.model || deps.defaultModel?.() || "gemini";
|
|
6812
|
+
const fam = modelFamilyOf(model);
|
|
6813
|
+
return pool.selectAccount(fam, model) !== null;
|
|
6814
|
+
}
|
|
6707
6815
|
/**
|
|
6708
6816
|
* Effort self-healing wrapper around streamAttempt (issue #67).
|
|
6709
6817
|
*
|
|
@@ -6715,13 +6823,25 @@ function providerRetryPolicyFor(pool) {
|
|
|
6715
6823
|
* becomes one extra spawn. The refusal text itself is the evidence that WE
|
|
6716
6824
|
* passed the flag, so the replay does not depend on who requested the effort
|
|
6717
6825
|
* (an explicit request, `defaultEffort`, or the aux channel's pinning).
|
|
6826
|
+
*
|
|
6827
|
+
* Also rotates across spare pool accounts for auxiliary routes (`isAux`) that
|
|
6828
|
+
* bypass `dsh-llm-retry`, and switches accounts when a model is unrecognized.
|
|
6718
6829
|
*/
|
|
6719
6830
|
async function* streamWithEffortRetry(deps, options) {
|
|
6720
|
-
|
|
6831
|
+
let suppressEffort = false;
|
|
6832
|
+
let effortSelfHealed = false;
|
|
6833
|
+
let modelSelfHealed = false;
|
|
6834
|
+
let quotaHops = 0;
|
|
6835
|
+
const isAux = isAuxCall(options);
|
|
6836
|
+
const poolEnabled = deps.pool ? deps.pool.getAccounts().filter((a) => a.enabled).length : 0;
|
|
6837
|
+
const maxAuxHops = Math.min(Math.max(0, poolEnabled - 1), 16);
|
|
6838
|
+
for (;;) {
|
|
6721
6839
|
let emitted = false;
|
|
6840
|
+
const bufferAux = isAux && quotaHops < maxAuxHops;
|
|
6841
|
+
const buffered = [];
|
|
6722
6842
|
const telemetry = [];
|
|
6723
6843
|
let finish = null;
|
|
6724
|
-
for await (const chunk of deps.streamAttempt(options,
|
|
6844
|
+
for await (const chunk of deps.streamAttempt(options, suppressEffort)) {
|
|
6725
6845
|
if (chunk.type === "finish") {
|
|
6726
6846
|
finish = chunk;
|
|
6727
6847
|
continue;
|
|
@@ -6730,27 +6850,51 @@ async function* streamWithEffortRetry(deps, options) {
|
|
|
6730
6850
|
telemetry.push(chunk);
|
|
6731
6851
|
continue;
|
|
6732
6852
|
}
|
|
6853
|
+
if (bufferAux) {
|
|
6854
|
+
buffered.push(chunk);
|
|
6855
|
+
continue;
|
|
6856
|
+
}
|
|
6733
6857
|
emitted = true;
|
|
6734
6858
|
yield chunk;
|
|
6735
6859
|
}
|
|
6736
6860
|
if (finish === null) {
|
|
6861
|
+
for (const chunk of buffered) yield chunk;
|
|
6737
6862
|
for (const chunk of telemetry) yield chunk;
|
|
6738
6863
|
return;
|
|
6739
6864
|
}
|
|
6740
|
-
if (
|
|
6865
|
+
if (!emitted && finish.reason.kind === "error") {
|
|
6741
6866
|
const failure = finish.reason.failure;
|
|
6742
6867
|
const rawMsg = String(failure?.message ?? "");
|
|
6743
|
-
|
|
6744
|
-
|
|
6745
|
-
if (
|
|
6746
|
-
|
|
6868
|
+
if (buffered.length === 0) {
|
|
6869
|
+
const refused = parseEffortUnsupported(rawMsg);
|
|
6870
|
+
if (!effortSelfHealed && refused !== null) {
|
|
6871
|
+
effortSelfHealed = true;
|
|
6872
|
+
suppressEffort = true;
|
|
6873
|
+
if (deps.catalogAdvertisesEffort?.(refused) !== true) deps.learnEffortUnsupported(refused);
|
|
6874
|
+
continue;
|
|
6875
|
+
}
|
|
6876
|
+
const unrecog = parseModelNotRecognized(rawMsg);
|
|
6877
|
+
if (!modelSelfHealed && unrecog !== null) {
|
|
6878
|
+
modelSelfHealed = true;
|
|
6879
|
+
if (deps.onModelUnsupported) {
|
|
6880
|
+
deps.onModelUnsupported(unrecog);
|
|
6881
|
+
continue;
|
|
6882
|
+
}
|
|
6883
|
+
if (deps.pool) {
|
|
6884
|
+
const fam = modelFamilyOf(unrecog);
|
|
6885
|
+
const currentId = deps.pool.getActiveAccountId(fam);
|
|
6886
|
+
if (currentId) deps.pool.markModelUnsupported(currentId, unrecog);
|
|
6887
|
+
deps.pool.selectAccount(fam, unrecog);
|
|
6888
|
+
continue;
|
|
6889
|
+
}
|
|
6890
|
+
}
|
|
6747
6891
|
}
|
|
6748
|
-
|
|
6749
|
-
|
|
6750
|
-
deps.onModelUnsupported(unrecog);
|
|
6892
|
+
if (canRetryAuxQuota(deps, options, emitted, finish, quotaHops, rawMsg)) {
|
|
6893
|
+
quotaHops++;
|
|
6751
6894
|
continue;
|
|
6752
6895
|
}
|
|
6753
6896
|
}
|
|
6897
|
+
for (const chunk of buffered) yield chunk;
|
|
6754
6898
|
for (const chunk of telemetry) yield chunk;
|
|
6755
6899
|
yield finish;
|
|
6756
6900
|
return;
|
|
@@ -6953,7 +7097,7 @@ async function finalizeAttempt(input) {
|
|
|
6953
7097
|
emitRunTelemetry(input);
|
|
6954
7098
|
}
|
|
6955
7099
|
function applySuccess(input) {
|
|
6956
|
-
if (input.account) input.pool?.recordSuccess(input.account.id, input.family);
|
|
7100
|
+
if (input.account && !input.cooledInLiveFamilies?.has(input.family)) input.pool?.recordSuccess(input.account.id, input.family);
|
|
6957
7101
|
input.dropFailoverBudget(input.sessionKey);
|
|
6958
7102
|
input.clearRateLimitFailover(input.sessionKey);
|
|
6959
7103
|
input.clearNetworkStreak(input.sessionKey);
|
|
@@ -6982,7 +7126,14 @@ function applyFailure(input, failure) {
|
|
|
6982
7126
|
*/
|
|
6983
7127
|
function rotateOnFailure(input, failure) {
|
|
6984
7128
|
const { account, pool, family, outcome } = input;
|
|
6985
|
-
|
|
7129
|
+
const qf = input.quotaFailover;
|
|
7130
|
+
const targetFamily = qf?.family ?? family;
|
|
7131
|
+
const resetMs = parseResetDurationMs(input.rawErrText) ?? parseResetDurationMs(failure.message);
|
|
7132
|
+
const serverResetTime = qf?.resetTime ?? (resetMs && resetMs > 0 ? new Date(Date.now() + resetMs).toISOString() : void 0);
|
|
7133
|
+
if (account && !outcome.aborted && (looksLikeHardRateLimit(input.rawErrText) || Boolean(qf))) {
|
|
7134
|
+
if (!qf?.cooledInLive) pool?.recordFailure(account.id, targetFamily, failure.message, serverResetTime);
|
|
7135
|
+
if (qf?.crossFamily) pool?.advanceAccount(account.id, family, input.activeModel);
|
|
7136
|
+
}
|
|
6986
7137
|
if (account && input.silentTimeout) pool?.advanceAccount(account.id, family);
|
|
6987
7138
|
if (account && failure.code === Err.ELIGIBILITY) {
|
|
6988
7139
|
const unrecog = parseModelNotRecognized(failure.message ?? "");
|
|
@@ -7002,7 +7153,7 @@ function rotateOnFailure(input, failure) {
|
|
|
7002
7153
|
* persisted as resumable-interrupted, then write it.
|
|
7003
7154
|
*/
|
|
7004
7155
|
function persistFailureBinding(input, failure) {
|
|
7005
|
-
const staleConversation = failure.code === Err.AUTH || failure.message !== "" && /conversation.*(not found|invalid|not recognized|expired|does not exist)|session.*(expired|invalid)/i.test(failure.message);
|
|
7156
|
+
const staleConversation = failure.code === Err.AUTH || input.quotaFailover?.crossFamily === true || failure.message !== "" && /conversation.*(not found|invalid|not recognized|expired|does not exist)|session.*(expired|invalid)/i.test(failure.message);
|
|
7006
7157
|
const interruptedTurn = failure.kind === "aborted" || input.outcome.timedOut === true;
|
|
7007
7158
|
if (input.persistBinding && input.sessionAccountKey !== "" && staleConversation) input.store.delete(input.sessionAccountKey);
|
|
7008
7159
|
else if (input.persistBinding && input.sessionAccountKey !== "" && !input.superseded) {
|
|
@@ -8333,11 +8484,16 @@ function isWeeklyQuotaBenched(quota, clearedAt, now) {
|
|
|
8333
8484
|
* `updatedAt` of the snapshot it overrode; the gate stays off only while the
|
|
8334
8485
|
* cached row is no newer than that stamp, so a fresh poll always re-arms it.
|
|
8335
8486
|
*/
|
|
8336
|
-
function isQuotaBenched(acc, family, now) {
|
|
8487
|
+
function isQuotaBenched(acc, family, now, model) {
|
|
8337
8488
|
const quota = acc.quotas?.[family] ?? (family === "argon" ? acc.quotas?.google : void 0);
|
|
8338
8489
|
if (!quota) return false;
|
|
8339
8490
|
const clearedAt = acc.quotaGateClearedAt?.[family];
|
|
8340
|
-
|
|
8491
|
+
if (is5hQuotaBenched(quota, clearedAt, now) || isWeeklyQuotaBenched(quota, clearedAt, now)) return true;
|
|
8492
|
+
if (!model || model.trim() === "" || !Array.isArray(quota.models)) return false;
|
|
8493
|
+
if (clearedAt !== void 0 && (quota.updatedAt ?? 0) <= clearedAt) return false;
|
|
8494
|
+
const matched = quota.models.filter((m) => modelMatches(model, m.modelId));
|
|
8495
|
+
if (matched.length === 0) return false;
|
|
8496
|
+
return matched.every((m) => isWindowBenched(m.remainingFraction, m.resetTime ?? quota.resetTime, LOW_QUOTA_WEEKLY, now));
|
|
8341
8497
|
}
|
|
8342
8498
|
/**
|
|
8343
8499
|
* True when the account cannot operate due to auth expiration, web verification
|
|
@@ -8424,7 +8580,7 @@ function explainIneligibility(acc, family, now, pool, model) {
|
|
|
8424
8580
|
if (model && pool?.isModelSupported && !pool.isModelSupported(acc, model)) return "model-unsupported";
|
|
8425
8581
|
const cd = acc.cooldowns?.[family];
|
|
8426
8582
|
if (cd && cd.cooldownUntil > now) return "cooldown";
|
|
8427
|
-
if (isQuotaBenched(acc, family, now)) return "quota";
|
|
8583
|
+
if (isQuotaBenched(acc, family, now, model)) return "quota";
|
|
8428
8584
|
return null;
|
|
8429
8585
|
}
|
|
8430
8586
|
/**
|
|
@@ -8854,7 +9010,7 @@ var PoolAdmin = class {
|
|
|
8854
9010
|
consecutiveFailures: failures
|
|
8855
9011
|
};
|
|
8856
9012
|
const famPinned = data.activeAccountIds?.[family];
|
|
8857
|
-
if (famPinned === id || famPinned === void 0 && data.lastActiveAccountId === id) this.advanceAccount(id, family);
|
|
9013
|
+
if (famPinned === id || famPinned === void 0 && (data.lastActiveAccountId === id || data.lastActiveAccountId === void 0)) this.advanceAccount(id, family);
|
|
8858
9014
|
this.deps.persist();
|
|
8859
9015
|
}
|
|
8860
9016
|
/**
|
|
@@ -34942,6 +35098,62 @@ function noteFailure(log, what, err) {
|
|
|
34942
35098
|
if (code === "ENOENT" || code === "EACCES" || code === "EPERM" || code === "EBUSY") return;
|
|
34943
35099
|
log?.(`subagent tracker: ${what} — ${String(err)}`);
|
|
34944
35100
|
}
|
|
35101
|
+
function createTranscriptCursor(offset = 0) {
|
|
35102
|
+
return {
|
|
35103
|
+
offset,
|
|
35104
|
+
decoder: new StringDecoder("utf8"),
|
|
35105
|
+
rest: ""
|
|
35106
|
+
};
|
|
35107
|
+
}
|
|
35108
|
+
function fileSizeOrZero(logPath) {
|
|
35109
|
+
try {
|
|
35110
|
+
const st = statSync(logPath);
|
|
35111
|
+
return st.isFile() ? st.size : 0;
|
|
35112
|
+
} catch {
|
|
35113
|
+
return 0;
|
|
35114
|
+
}
|
|
35115
|
+
}
|
|
35116
|
+
function readTranscriptDelta(logPath, cursor) {
|
|
35117
|
+
let size;
|
|
35118
|
+
try {
|
|
35119
|
+
const st = statSync(logPath);
|
|
35120
|
+
if (!st.isFile()) return [];
|
|
35121
|
+
size = st.size;
|
|
35122
|
+
} catch {
|
|
35123
|
+
return [];
|
|
35124
|
+
}
|
|
35125
|
+
if (size < cursor.offset) {
|
|
35126
|
+
cursor.offset = 0;
|
|
35127
|
+
cursor.decoder = new StringDecoder("utf8");
|
|
35128
|
+
cursor.rest = "";
|
|
35129
|
+
}
|
|
35130
|
+
if (size === cursor.offset) return [];
|
|
35131
|
+
const length = size - cursor.offset;
|
|
35132
|
+
const buffer = Buffer.allocUnsafe(length);
|
|
35133
|
+
let bytesRead = 0;
|
|
35134
|
+
try {
|
|
35135
|
+
const fd = openSync(logPath, "r");
|
|
35136
|
+
try {
|
|
35137
|
+
bytesRead = readSync(fd, buffer, 0, length, cursor.offset);
|
|
35138
|
+
} finally {
|
|
35139
|
+
closeSync(fd);
|
|
35140
|
+
}
|
|
35141
|
+
} catch {
|
|
35142
|
+
return [];
|
|
35143
|
+
}
|
|
35144
|
+
if (bytesRead <= 0) return [];
|
|
35145
|
+
cursor.offset += bytesRead;
|
|
35146
|
+
const parts = (cursor.rest + cursor.decoder.write(buffer.subarray(0, bytesRead))).split("\n");
|
|
35147
|
+
cursor.rest = parts.pop() ?? "";
|
|
35148
|
+
return parts.filter((line) => line.trim() !== "");
|
|
35149
|
+
}
|
|
35150
|
+
function extractAuthoritativeSubagentQuotaError(text, fromParentSystemMessage) {
|
|
35151
|
+
if (!text) return false;
|
|
35152
|
+
if (!looksLikeHardRateLimit(text) || isTransientBurstRateLimit(text) || looksLikeTosViolation(text)) return false;
|
|
35153
|
+
if (fromParentSystemMessage) return /The subagent\b[\s\S]*?\bencountered an error/i.test(text);
|
|
35154
|
+
if (/Command exited with code\b/i.test(text)) return false;
|
|
35155
|
+
return /image failed to generate|failed to generate content|cloudcode-pa\.googleapis\.com|RESOURCE_EXHAUSTED|QUOTA_EXHAUSTED|Individual quota reached|agent executor error/i.test(text);
|
|
35156
|
+
}
|
|
34945
35157
|
function isActiveSubagentState(state) {
|
|
34946
35158
|
return state === void 0 || state === "" || state === "SUBAGENT_STATE_ALIVE";
|
|
34947
35159
|
}
|
|
@@ -35017,6 +35229,8 @@ var SubagentTracker = class {
|
|
|
35017
35229
|
active = false;
|
|
35018
35230
|
polling = false;
|
|
35019
35231
|
pollIntervalMs;
|
|
35232
|
+
parentCursor = createTranscriptCursor();
|
|
35233
|
+
parentLines = 0;
|
|
35020
35234
|
constructor(opts) {
|
|
35021
35235
|
this.opts = opts;
|
|
35022
35236
|
this.conversationId = opts.conversationId ?? "";
|
|
@@ -35030,6 +35244,8 @@ var SubagentTracker = class {
|
|
|
35030
35244
|
this.conversationId = cid;
|
|
35031
35245
|
this.brainDir = null;
|
|
35032
35246
|
this.baselined = false;
|
|
35247
|
+
this.parentCursor = createTranscriptCursor();
|
|
35248
|
+
this.parentLines = 0;
|
|
35033
35249
|
this.taskActivity.reset();
|
|
35034
35250
|
if (this.active) {
|
|
35035
35251
|
if (this.opts.resumed === true) {
|
|
@@ -35055,6 +35271,8 @@ var SubagentTracker = class {
|
|
|
35055
35271
|
try {
|
|
35056
35272
|
if (!this.brainDir) this.brainDir = findBrainDir(this.conversationId, this.accountHome);
|
|
35057
35273
|
if (!this.brainDir) return;
|
|
35274
|
+
const parentLog = join(this.brainDir, ".system_generated", "logs", "transcript.jsonl");
|
|
35275
|
+
this.parentCursor = createTranscriptCursor(fileSizeOrZero(parentLog));
|
|
35058
35276
|
this.taskActivity.baseline(this.brainDir);
|
|
35059
35277
|
this.scanSubagentsForConv(this.conversationId, true);
|
|
35060
35278
|
for (const cid of Array.from(this.trackedSubagents.keys())) this.scanSubagentsForConv(cid, true);
|
|
@@ -35073,14 +35291,19 @@ var SubagentTracker = class {
|
|
|
35073
35291
|
}
|
|
35074
35292
|
}
|
|
35075
35293
|
}
|
|
35076
|
-
/**
|
|
35077
|
-
|
|
35078
|
-
|
|
35294
|
+
/** Perform one final synchronous scan even after `stop()` halted the interval. */
|
|
35295
|
+
flush() {
|
|
35296
|
+
this.poll(true);
|
|
35297
|
+
}
|
|
35298
|
+
/** Run one scan across parent transcript, known conversations, and subagents. */
|
|
35299
|
+
poll(force = false) {
|
|
35300
|
+
if (this.polling || !this.active && !force) return;
|
|
35079
35301
|
if (!this.conversationId) return;
|
|
35080
35302
|
this.polling = true;
|
|
35081
35303
|
try {
|
|
35082
35304
|
if (!this.brainDir) this.brainDir = findBrainDir(this.conversationId, this.accountHome);
|
|
35083
35305
|
if (!this.brainDir) return;
|
|
35306
|
+
this.pollParentTranscript();
|
|
35084
35307
|
this.taskActivity.poll(this.brainDir, (ev) => {
|
|
35085
35308
|
this.emitSubagentEvent(ev.stepKey, ev.text, ev.raw);
|
|
35086
35309
|
});
|
|
@@ -35093,6 +35316,30 @@ var SubagentTracker = class {
|
|
|
35093
35316
|
this.polling = false;
|
|
35094
35317
|
}
|
|
35095
35318
|
}
|
|
35319
|
+
pollParentTranscript() {
|
|
35320
|
+
if (!this.brainDir) return;
|
|
35321
|
+
const logPath = join(this.brainDir, ".system_generated", "logs", "transcript.jsonl");
|
|
35322
|
+
for (const line of readTranscriptDelta(logPath, this.parentCursor)) try {
|
|
35323
|
+
const o = JSON.parse(line);
|
|
35324
|
+
this.parentLines += 1;
|
|
35325
|
+
if (o.source !== "SYSTEM" || o.type !== "SYSTEM_MESSAGE" || typeof o.content !== "string") continue;
|
|
35326
|
+
if (!extractAuthoritativeSubagentQuotaError(o.content, true)) continue;
|
|
35327
|
+
const qKey = `parent-quota-${typeof o.step_index === "number" ? o.step_index : this.parentLines}`;
|
|
35328
|
+
if (this.seenStepKeys.has(qKey)) continue;
|
|
35329
|
+
const role = o.content.match(/The subagent\s+(.+?)\s+(?:\([^)]+\)\s+)?(?:with id|encountered an error)/i)?.[1]?.trim();
|
|
35330
|
+
const model = extractQuotaErrorModel(o.content);
|
|
35331
|
+
const resetMs = parseResetDurationMs(o.content);
|
|
35332
|
+
const prefix = role ? `[子代理 · ${role}]` : "[子代理]";
|
|
35333
|
+
this.emitSubagentEvent(qKey, `${prefix} 额度已用尽,正在准备切换账号...\n`, o);
|
|
35334
|
+
this.opts.onQuotaExhausted?.({
|
|
35335
|
+
source: "parent-system-message",
|
|
35336
|
+
errorText: o.content,
|
|
35337
|
+
...role ? { role } : {},
|
|
35338
|
+
...model ? { model } : {},
|
|
35339
|
+
...resetMs !== void 0 ? { resetMs } : {}
|
|
35340
|
+
});
|
|
35341
|
+
} catch {}
|
|
35342
|
+
}
|
|
35096
35343
|
emitSubagentEvent(stepKey, text, raw) {
|
|
35097
35344
|
if (this.seenStepKeys.has(stepKey)) return;
|
|
35098
35345
|
this.seenStepKeys.add(stepKey);
|
|
@@ -35142,14 +35389,8 @@ var SubagentTracker = class {
|
|
|
35142
35389
|
const typeName = typeof desc?.typeName === "string" && desc.typeName !== "" ? desc.typeName : "Subagent";
|
|
35143
35390
|
const subBrainDir = findBrainDir(subId, this.accountHome) ?? join(bDir, "..", subId);
|
|
35144
35391
|
const active = isActiveSubagentState(rawState);
|
|
35145
|
-
|
|
35146
|
-
|
|
35147
|
-
const logPath = join(subBrainDir, ".system_generated", "logs", "transcript.jsonl");
|
|
35148
|
-
const st = statSync(logPath);
|
|
35149
|
-
if (st.isFile()) initialOffset = st.size;
|
|
35150
|
-
} catch {
|
|
35151
|
-
initialOffset = 0;
|
|
35152
|
-
}
|
|
35392
|
+
const logPath = join(subBrainDir, ".system_generated", "logs", "transcript.jsonl");
|
|
35393
|
+
const initialOffset = baselineOnly ? fileSizeOrZero(logPath) : 0;
|
|
35153
35394
|
const tracked = {
|
|
35154
35395
|
conversationId: subId,
|
|
35155
35396
|
role,
|
|
@@ -35157,9 +35398,7 @@ var SubagentTracker = class {
|
|
|
35157
35398
|
brainDir: subBrainDir,
|
|
35158
35399
|
state: rawState,
|
|
35159
35400
|
terminal: !active,
|
|
35160
|
-
|
|
35161
|
-
decoder: new StringDecoder("utf8"),
|
|
35162
|
-
rest: "",
|
|
35401
|
+
cursor: createTranscriptCursor(initialOffset),
|
|
35163
35402
|
lines: 0
|
|
35164
35403
|
};
|
|
35165
35404
|
this.trackedSubagents.set(subId, tracked);
|
|
@@ -35168,6 +35407,7 @@ var SubagentTracker = class {
|
|
|
35168
35407
|
else if (active) this.emitSubagentEvent(spawnKey, `[子代理 · ${role}] 已启动协同专员 (${typeName})\n`, obj);
|
|
35169
35408
|
else {
|
|
35170
35409
|
this.seenStepKeys.add(spawnKey);
|
|
35410
|
+
this.pollSubagentTranscript(tracked, true);
|
|
35171
35411
|
if (rawState !== void 0) this.emitSubagentEvent(`subagent-state-${subId}-${rawState}`, `[子代理 · ${role}] 协同专员未保持运行 (${rawState})\n`, obj);
|
|
35172
35412
|
}
|
|
35173
35413
|
} catch (err) {
|
|
@@ -35175,70 +35415,161 @@ var SubagentTracker = class {
|
|
|
35175
35415
|
}
|
|
35176
35416
|
}
|
|
35177
35417
|
}
|
|
35178
|
-
|
|
35179
|
-
* Read the transcript lines appended since the last poll.
|
|
35180
|
-
*
|
|
35181
|
-
* agy rewrites nothing and only appends, so the tracker keeps a byte offset
|
|
35182
|
-
* instead of re-reading (and re-splitting) a transcript that grows to hundreds
|
|
35183
|
-
* of kilobytes: each poll costs one bounded read of the new bytes.
|
|
35184
|
-
*/
|
|
35185
|
-
readNewLines(sub) {
|
|
35418
|
+
pollSubagentTranscript(sub, quotaOnly = false) {
|
|
35186
35419
|
const logPath = join(sub.brainDir, ".system_generated", "logs", "transcript.jsonl");
|
|
35187
|
-
|
|
35188
|
-
try {
|
|
35189
|
-
const st = statSync(logPath);
|
|
35190
|
-
if (!st.isFile()) return [];
|
|
35191
|
-
size = st.size;
|
|
35192
|
-
} catch {
|
|
35193
|
-
return [];
|
|
35194
|
-
}
|
|
35195
|
-
if (size < sub.offset) {
|
|
35196
|
-
sub.offset = 0;
|
|
35197
|
-
sub.decoder = new StringDecoder("utf8");
|
|
35198
|
-
sub.rest = "";
|
|
35199
|
-
}
|
|
35200
|
-
if (size === sub.offset) return [];
|
|
35201
|
-
const length = size - sub.offset;
|
|
35202
|
-
const buffer = Buffer.allocUnsafe(length);
|
|
35203
|
-
let bytesRead = 0;
|
|
35204
|
-
try {
|
|
35205
|
-
const fd = openSync(logPath, "r");
|
|
35206
|
-
try {
|
|
35207
|
-
bytesRead = readSync(fd, buffer, 0, length, sub.offset);
|
|
35208
|
-
} finally {
|
|
35209
|
-
closeSync(fd);
|
|
35210
|
-
}
|
|
35211
|
-
} catch {
|
|
35212
|
-
return [];
|
|
35213
|
-
}
|
|
35214
|
-
if (bytesRead <= 0) return [];
|
|
35215
|
-
sub.offset += bytesRead;
|
|
35216
|
-
const parts = (sub.rest + sub.decoder.write(buffer.subarray(0, bytesRead))).split("\n");
|
|
35217
|
-
sub.rest = parts.pop() ?? "";
|
|
35218
|
-
return parts.filter((line) => line.trim() !== "");
|
|
35219
|
-
}
|
|
35220
|
-
pollSubagentTranscript(sub) {
|
|
35221
|
-
for (const line of this.readNewLines(sub)) try {
|
|
35420
|
+
for (const line of readTranscriptDelta(logPath, sub.cursor)) try {
|
|
35222
35421
|
const o = JSON.parse(line);
|
|
35223
35422
|
sub.lines += 1;
|
|
35224
35423
|
const stepIdx = typeof o.step_index === "number" ? o.step_index : sub.lines;
|
|
35225
35424
|
const baseKey = `subagent-${sub.conversationId}-${stepIdx}`;
|
|
35226
|
-
if (
|
|
35227
|
-
|
|
35228
|
-
|
|
35229
|
-
|
|
35230
|
-
|
|
35231
|
-
|
|
35232
|
-
|
|
35233
|
-
|
|
35234
|
-
|
|
35235
|
-
|
|
35236
|
-
|
|
35237
|
-
|
|
35425
|
+
if (!quotaOnly) {
|
|
35426
|
+
if (Array.isArray(o.tool_calls) && o.tool_calls.length > 0) for (let tcIdx = 0; tcIdx < o.tool_calls.length; tcIdx++) {
|
|
35427
|
+
const tc = o.tool_calls[tcIdx];
|
|
35428
|
+
const toolName = typeof tc?.name === "string" ? tc.name : "";
|
|
35429
|
+
if (toolName === "") continue;
|
|
35430
|
+
const msg = formatSubagentAction(sub.role, toolName, tc.args);
|
|
35431
|
+
this.emitSubagentEvent(`${baseKey}-tc-${tcIdx}`, `${msg}\n`, o);
|
|
35432
|
+
}
|
|
35433
|
+
else if (o.type === "PLANNER_RESPONSE" && typeof o.content === "string" && o.content.trim() !== "") {
|
|
35434
|
+
const firstLine = o.content.trim().split("\n")[0].replace(/^#+\s*/, "").slice(0, 100);
|
|
35435
|
+
if (firstLine !== "") this.emitSubagentEvent(`${baseKey}-content`, `[子代理 · ${sub.role}] 阶段进展: ${firstLine}\n`, o);
|
|
35436
|
+
}
|
|
35437
|
+
}
|
|
35438
|
+
if (isTerminalTranscriptStatus(o.status)) {
|
|
35439
|
+
if (!quotaOnly) this.emitSubagentEvent(`${baseKey}-status`, `[子代理 · ${sub.role}] 执行异常/中止 (${o.status})\n`, o);
|
|
35440
|
+
if (/^(ERROR|FAILED)$/i.test(o.status.trim()) && typeof o.content === "string" && extractAuthoritativeSubagentQuotaError(o.content, false)) {
|
|
35441
|
+
const qKey = `subagent-quota-${sub.conversationId}-${stepIdx}`;
|
|
35442
|
+
if (!this.seenStepKeys.has(qKey)) {
|
|
35443
|
+
const model = extractQuotaErrorModel(o.content);
|
|
35444
|
+
const resetMs = parseResetDurationMs(o.content);
|
|
35445
|
+
this.emitSubagentEvent(qKey, `[子代理 · ${sub.role}] 额度已用尽,正在准备切换账号...\n`, o);
|
|
35446
|
+
this.opts.onQuotaExhausted?.({
|
|
35447
|
+
source: "subagent-transcript",
|
|
35448
|
+
errorText: o.content,
|
|
35449
|
+
role: sub.role,
|
|
35450
|
+
conversationId: sub.conversationId,
|
|
35451
|
+
...model ? { model } : {},
|
|
35452
|
+
...resetMs !== void 0 ? { resetMs } : {}
|
|
35453
|
+
});
|
|
35454
|
+
}
|
|
35455
|
+
}
|
|
35456
|
+
}
|
|
35238
35457
|
} catch {}
|
|
35239
35458
|
}
|
|
35240
35459
|
};
|
|
35241
35460
|
//#endregion
|
|
35461
|
+
//#region src/provider/quota-failover.ts
|
|
35462
|
+
const DIRECT_QUOTA_TOOLS = /* @__PURE__ */ new Set([
|
|
35463
|
+
"generate_image",
|
|
35464
|
+
"invoke_subagent",
|
|
35465
|
+
"swarm",
|
|
35466
|
+
"run_workflow"
|
|
35467
|
+
]);
|
|
35468
|
+
/**
|
|
35469
|
+
* Inspect one completed/failed tool step for an authoritative hard quota error
|
|
35470
|
+
* emitted directly by agy's image-generation or subagent delegation tool.
|
|
35471
|
+
*/
|
|
35472
|
+
function isDirectToolQuotaExhausted(ev) {
|
|
35473
|
+
if (ev.kind !== "step" || ev.stepKind !== "tool" || !ev.tool) return null;
|
|
35474
|
+
if (ev.state !== "ERROR" && !ev.tool.error) return null;
|
|
35475
|
+
const unwrapped = unwrapMcpTool(ev.tool.name, ev.tool.args);
|
|
35476
|
+
if (!DIRECT_QUOTA_TOOLS.has(unwrapped.tool)) return null;
|
|
35477
|
+
const err = [ev.tool.error, typeof ev.tool.output === "string" ? ev.tool.output : ""].filter(Boolean).join(" ").trim();
|
|
35478
|
+
if (err === "") return null;
|
|
35479
|
+
if (!looksLikeHardRateLimit(err) || isTransientBurstRateLimit(err) || looksLikeTosViolation(err)) return null;
|
|
35480
|
+
return {
|
|
35481
|
+
toolName: unwrapped.tool,
|
|
35482
|
+
errorText: err
|
|
35483
|
+
};
|
|
35484
|
+
}
|
|
35485
|
+
/**
|
|
35486
|
+
* Find an alternative healthy account that can serve BOTH the current turn's
|
|
35487
|
+
* `(inputFamily, activeModel)` and (if different) the exhausted `exhaustedFamily`.
|
|
35488
|
+
*/
|
|
35489
|
+
function findAlternativeAccount(pool, currentAccountId, inputFamily, activeModel, exhaustedFamily) {
|
|
35490
|
+
const now = Date.now();
|
|
35491
|
+
return pool.getAccounts().find((alt) => {
|
|
35492
|
+
if (alt.id === currentAccountId) return false;
|
|
35493
|
+
if (!(typeof pool.isAccountEligibleForModel === "function" ? pool.isAccountEligibleForModel(alt, inputFamily, activeModel, now) : pool.isAccountEligible(alt, inputFamily, now))) return false;
|
|
35494
|
+
if (exhaustedFamily !== inputFamily && !pool.isAccountEligible(alt, exhaustedFamily, now)) return false;
|
|
35495
|
+
return true;
|
|
35496
|
+
});
|
|
35497
|
+
}
|
|
35498
|
+
function createLiveQuotaController(ctx) {
|
|
35499
|
+
const cooled = /* @__PURE__ */ new Set();
|
|
35500
|
+
let state = null;
|
|
35501
|
+
let triggeredKill = false;
|
|
35502
|
+
const handleQuotaExhausted = (info) => {
|
|
35503
|
+
const raw = info.errorText.trim();
|
|
35504
|
+
if (raw === "" || isTransientBurstRateLimit(raw) || looksLikeTosViolation(raw)) return;
|
|
35505
|
+
const detectedModel = info.model ?? extractQuotaErrorModel(raw);
|
|
35506
|
+
const modelFam = detectedModel ? modelFamilyOf(detectedModel) : void 0;
|
|
35507
|
+
const targetFamily = modelFam && modelFam !== "unknown" ? modelFam : ctx.inputFamily;
|
|
35508
|
+
const resetMs = info.resetMs ?? parseResetDurationMs(raw);
|
|
35509
|
+
const resetTime = resetMs && resetMs > 0 ? new Date(Date.now() + resetMs).toISOString() : void 0;
|
|
35510
|
+
let cooledNow = false;
|
|
35511
|
+
if (ctx.account && ctx.pool && !cooled.has(targetFamily)) {
|
|
35512
|
+
cooled.add(targetFamily);
|
|
35513
|
+
ctx.pool.recordFailure(ctx.account.id, targetFamily, raw, resetTime);
|
|
35514
|
+
cooledNow = true;
|
|
35515
|
+
}
|
|
35516
|
+
const alt = ctx.account && ctx.pool ? findAlternativeAccount(ctx.pool, ctx.account.id, ctx.inputFamily, ctx.activeModel, targetFamily) : void 0;
|
|
35517
|
+
const subject = info.role ? `子代理 (${info.role})` : "子代理/工具";
|
|
35518
|
+
if (alt && !triggeredKill) {
|
|
35519
|
+
triggeredKill = true;
|
|
35520
|
+
state = {
|
|
35521
|
+
family: targetFamily,
|
|
35522
|
+
crossFamily: targetFamily !== ctx.inputFamily,
|
|
35523
|
+
errorText: raw.slice(0, 300),
|
|
35524
|
+
rawErrorText: raw,
|
|
35525
|
+
...resetTime === void 0 ? {} : { resetTime },
|
|
35526
|
+
cooledInLive: cooledNow || cooled.has(targetFamily)
|
|
35527
|
+
};
|
|
35528
|
+
ctx.emitNotice(`live-quota-switch-${ctx.account?.id ?? "none"}-${targetFamily}`, `[agy] ${subject}额度耗尽,已自动切换至可用账号 ${alt.alias || alt.id}...\n`);
|
|
35529
|
+
ctx.onTriggerKill();
|
|
35530
|
+
} else if (!alt) ctx.emitNotice(`live-quota-none-${ctx.account?.id ?? "none"}-${targetFamily}`, `[agy] ${subject}额度耗尽,当前无其他可用账号可切换\n`);
|
|
35531
|
+
};
|
|
35532
|
+
return {
|
|
35533
|
+
handleQuotaExhausted,
|
|
35534
|
+
failoverState: () => state,
|
|
35535
|
+
cooledFamilies: () => cooled
|
|
35536
|
+
};
|
|
35537
|
+
}
|
|
35538
|
+
function reportDirectToolQuota(quotaCtrl, ev) {
|
|
35539
|
+
if (!quotaCtrl) return;
|
|
35540
|
+
const direct = isDirectToolQuotaExhausted(ev);
|
|
35541
|
+
if (!direct) return;
|
|
35542
|
+
const m = extractQuotaErrorModel(direct.errorText);
|
|
35543
|
+
const r = parseResetDurationMs(direct.errorText);
|
|
35544
|
+
quotaCtrl.handleQuotaExhausted({
|
|
35545
|
+
source: "tool-step",
|
|
35546
|
+
errorText: direct.errorText,
|
|
35547
|
+
...m ? { model: m } : {},
|
|
35548
|
+
...r !== void 0 ? { resetMs: r } : {}
|
|
35549
|
+
});
|
|
35550
|
+
}
|
|
35551
|
+
function poolQuotaHopsFor(pool) {
|
|
35552
|
+
if (!pool) return 0;
|
|
35553
|
+
return Math.min(Math.max(0, pool.getAccounts().filter((a) => a.enabled).length - 1), 16);
|
|
35554
|
+
}
|
|
35555
|
+
function createPoolQuotaHooks(pool) {
|
|
35556
|
+
return {
|
|
35557
|
+
poolQuotaHops: () => poolQuotaHopsFor(pool),
|
|
35558
|
+
onQuotaExhausted: (accountId, model, errorText) => {
|
|
35559
|
+
const errFam = modelFamilyOf(extractQuotaErrorModel(errorText) || model);
|
|
35560
|
+
const mainFam = modelFamilyOf(model);
|
|
35561
|
+
const targetFam = errFam !== "unknown" ? errFam : mainFam;
|
|
35562
|
+
const resetMs = parseResetDurationMs(errorText);
|
|
35563
|
+
const resetTime = resetMs && resetMs > 0 ? new Date(Date.now() + resetMs).toISOString() : void 0;
|
|
35564
|
+
if (accountId) {
|
|
35565
|
+
pool.recordFailure(accountId, targetFam, errorText, resetTime);
|
|
35566
|
+
if (targetFam !== mainFam) pool.advanceAccount(accountId, mainFam, model);
|
|
35567
|
+
}
|
|
35568
|
+
return pool.selectAccount(mainFam, model) !== null;
|
|
35569
|
+
}
|
|
35570
|
+
};
|
|
35571
|
+
}
|
|
35572
|
+
//#endregion
|
|
35242
35573
|
//#region src/provider/attempt.ts
|
|
35243
35574
|
function brief$1(s) {
|
|
35244
35575
|
const flat = s.trim().replace(/\s+/g, " ");
|
|
@@ -35329,7 +35660,7 @@ function buildSpawnArgs(input, printTimeoutMinutes, log) {
|
|
|
35329
35660
|
};
|
|
35330
35661
|
}
|
|
35331
35662
|
/** Create the recording, parser, HOME snapshot and tool watchdog for one run. */
|
|
35332
|
-
function createAttemptRecorder(deps, cfg, account, initialConversationId, trackSubagents = true) {
|
|
35663
|
+
function createAttemptRecorder(deps, cfg, account, initialConversationId, trackSubagents = true, quotaCtx) {
|
|
35333
35664
|
const rec = deps.runs.create();
|
|
35334
35665
|
rec.accountHome = account && account.dir ? account.dir : void 0;
|
|
35335
35666
|
const conversationsDir = conversationsDirFor(rec.accountHome);
|
|
@@ -35337,17 +35668,33 @@ function createAttemptRecorder(deps, cfg, account, initialConversationId, trackS
|
|
|
35337
35668
|
const parser = new StreamJsonParser();
|
|
35338
35669
|
deps.onParser?.(parser);
|
|
35339
35670
|
const watchdog = createToolWatchdog(deps.log, cfg);
|
|
35671
|
+
let attachedProc = null;
|
|
35672
|
+
let pendingQuotaKill = false;
|
|
35673
|
+
const quotaCtrl = quotaCtx ? createLiveQuotaController({
|
|
35674
|
+
pool: deps.pool,
|
|
35675
|
+
account,
|
|
35676
|
+
inputFamily: quotaCtx.family,
|
|
35677
|
+
activeModel: quotaCtx.activeModel,
|
|
35678
|
+
emitNotice: (stepKey, text) => rec.append({
|
|
35679
|
+
kind: "step",
|
|
35680
|
+
stepKey,
|
|
35681
|
+
stepKind: "subagent",
|
|
35682
|
+
text,
|
|
35683
|
+
raw: {}
|
|
35684
|
+
}),
|
|
35685
|
+
onTriggerKill: () => {
|
|
35686
|
+
if (attachedProc) attachedProc.kill("quota");
|
|
35687
|
+
else pendingQuotaKill = true;
|
|
35688
|
+
}
|
|
35689
|
+
}) : null;
|
|
35340
35690
|
const tracker = trackSubagents ? new SubagentTracker({
|
|
35341
35691
|
conversationId: initialConversationId,
|
|
35342
35692
|
resumed: Boolean(initialConversationId),
|
|
35343
35693
|
accountHome: rec.accountHome,
|
|
35344
35694
|
log: deps.log,
|
|
35345
|
-
onEvent: (ev) =>
|
|
35346
|
-
|
|
35347
|
-
|
|
35348
|
-
noteActivity: () => {
|
|
35349
|
-
watchdog.markEvent();
|
|
35350
|
-
}
|
|
35695
|
+
onEvent: (ev) => rec.append(ev),
|
|
35696
|
+
...quotaCtrl ? { onQuotaExhausted: (info) => quotaCtrl.handleQuotaExhausted(info) } : {},
|
|
35697
|
+
noteActivity: () => watchdog.markEvent()
|
|
35351
35698
|
}) : void 0;
|
|
35352
35699
|
const state = {
|
|
35353
35700
|
rec,
|
|
@@ -35357,8 +35704,10 @@ function createAttemptRecorder(deps, cfg, account, initialConversationId, trackS
|
|
|
35357
35704
|
streamCid: initialConversationId ?? null,
|
|
35358
35705
|
printTimeoutMinutes: watchdog.printTimeoutMinutes,
|
|
35359
35706
|
attachProcess: (proc) => {
|
|
35707
|
+
attachedProc = proc;
|
|
35360
35708
|
watchdog.attach(proc);
|
|
35361
35709
|
tracker?.start();
|
|
35710
|
+
if (pendingQuotaKill) proc.kill("quota");
|
|
35362
35711
|
proc.outcome.finally(() => tracker?.stop());
|
|
35363
35712
|
},
|
|
35364
35713
|
startTracker: () => {
|
|
@@ -35378,6 +35727,7 @@ function createAttemptRecorder(deps, cfg, account, initialConversationId, trackS
|
|
|
35378
35727
|
}
|
|
35379
35728
|
watchdog.trackStep(ev);
|
|
35380
35729
|
rec.append(ev);
|
|
35730
|
+
reportDirectToolQuota(quotaCtrl, ev);
|
|
35381
35731
|
if (typeof onEvent === "function") try {
|
|
35382
35732
|
onEvent(ev);
|
|
35383
35733
|
} catch {}
|
|
@@ -35395,10 +35745,15 @@ function createAttemptRecorder(deps, cfg, account, initialConversationId, trackS
|
|
|
35395
35745
|
} catch {}
|
|
35396
35746
|
}
|
|
35397
35747
|
},
|
|
35748
|
+
flushTracker: () => {
|
|
35749
|
+
tracker?.flush();
|
|
35750
|
+
},
|
|
35398
35751
|
stop: () => {
|
|
35399
35752
|
watchdog.stop();
|
|
35400
35753
|
tracker?.stop();
|
|
35401
35754
|
},
|
|
35755
|
+
quotaFailover: () => quotaCtrl?.failoverState() ?? null,
|
|
35756
|
+
cooledInLiveFamilies: () => quotaCtrl?.cooledFamilies() ?? /* @__PURE__ */ new Set(),
|
|
35402
35757
|
conversationId: () => state.streamCid ?? diffConversations(before, conversationsDir).conversationId
|
|
35403
35758
|
};
|
|
35404
35759
|
return state;
|
|
@@ -35968,19 +36323,24 @@ async function* dispatchCredentialSlot(deps, input) {
|
|
|
35968
36323
|
}
|
|
35969
36324
|
/** Evidence + failure for one settled run (pure classification inputs). */
|
|
35970
36325
|
function classifyRun(input) {
|
|
35971
|
-
const { cfg, outcome, parser, rec, runLogHome, runLogsBefore } = input;
|
|
36326
|
+
const { cfg, outcome, parser, rec, runLogHome, runLogsBefore, quotaFailover } = input;
|
|
35972
36327
|
const r = rec.getResultEvent();
|
|
35973
36328
|
const u = rec.lastUsage ?? rec.finalUsage({});
|
|
35974
36329
|
const zeroUsageTos = r !== null && (u?.input_tokens ?? 0) === 0 && (u?.output_tokens ?? 0) === 0 && looksLikeTosViolation(r.response);
|
|
35975
36330
|
const errorResponseTos = r !== null && !r.ok && looksLikeTosViolation(r.response);
|
|
35976
|
-
const
|
|
36331
|
+
const errorResponseHardQuota = r !== null && isHardQuotaResultError(r.ok, r.error, r.response);
|
|
36332
|
+
const liveQuota = quotaFailover !== null;
|
|
36333
|
+
const consumable = !liveQuota && r !== null && (r.ok || r.response !== "") && !zeroUsageTos && !errorResponseTos && !errorResponseHardQuota;
|
|
36334
|
+
const runLogText = readAgyRunLogText(runLogHome, runLogsBefore);
|
|
36335
|
+
const logQuotaHint = !consumable && !liveQuota ? runLogText.split(/\r?\n/).filter((l) => /(?:errorreport\.go|session\.go).*?(?:RESOURCE_EXHAUSTED|Individual quota reached|QUOTA_EXHAUSTED)/i.test(l)).slice(-2).join(" ") : "";
|
|
35977
36336
|
const rawErrText = [
|
|
35978
36337
|
outcome.stderrTail,
|
|
35979
36338
|
parser.stats.lastResultError,
|
|
35980
|
-
!consumable && r && (!r.ok || zeroUsageTos) ? r.response : ""
|
|
36339
|
+
!consumable && r && (!r.ok || zeroUsageTos) ? r.error ?? r.response : "",
|
|
36340
|
+
liveQuota ? quotaFailover.rawErrorText : "",
|
|
36341
|
+
logQuotaHint
|
|
35981
36342
|
].filter(Boolean).join(" ");
|
|
35982
|
-
const isRateLimit = looksLikeRateLimit(rawErrText);
|
|
35983
|
-
const runLogText = readAgyRunLogText(runLogHome, runLogsBefore);
|
|
36343
|
+
const isRateLimit = liveQuota || looksLikeRateLimit(rawErrText);
|
|
35984
36344
|
const keyringJitter = looksLikeAuthVerificationJitter(runLogText);
|
|
35985
36345
|
const transportFailure = isTransportFailure({
|
|
35986
36346
|
rawErrText,
|
|
@@ -35989,20 +36349,27 @@ function classifyRun(input) {
|
|
|
35989
36349
|
stdoutHead: outcome.stdout,
|
|
35990
36350
|
runLogText
|
|
35991
36351
|
});
|
|
35992
|
-
const silentTimeout = outcome.timedOut === true && outcome.stdout.trim() === "";
|
|
36352
|
+
const silentTimeout = !liveQuota && outcome.timedOut === true && outcome.stdout.trim() === "";
|
|
36353
|
+
const failure = classifyFailure({
|
|
36354
|
+
cfg,
|
|
36355
|
+
outcome: liveQuota ? {
|
|
36356
|
+
...outcome,
|
|
36357
|
+
capped: false,
|
|
36358
|
+
timedOut: false,
|
|
36359
|
+
idleGraceTerminated: false
|
|
36360
|
+
} : outcome,
|
|
36361
|
+
parser,
|
|
36362
|
+
sawAuthFailure: liveQuota ? false : sawAuthFailure(parser, outcome),
|
|
36363
|
+
consumable,
|
|
36364
|
+
transportFailure: liveQuota ? false : transportFailure,
|
|
36365
|
+
keyringJitter: liveQuota ? false : keyringJitter,
|
|
36366
|
+
rawErrText,
|
|
36367
|
+
isRateLimit,
|
|
36368
|
+
silentTimeout
|
|
36369
|
+
});
|
|
36370
|
+
if (failure && liveQuota && failure.code === Err.RATE_LIMIT && parser.stats.lastResultError === void 0 && outcome.stderrTail === "") failure.message = "Google Antigravity quota / rate limit reached (subagent/tool): " + quotaFailover.errorText;
|
|
35993
36371
|
return {
|
|
35994
|
-
failure
|
|
35995
|
-
cfg,
|
|
35996
|
-
outcome,
|
|
35997
|
-
parser,
|
|
35998
|
-
sawAuthFailure: sawAuthFailure(parser, outcome),
|
|
35999
|
-
consumable,
|
|
36000
|
-
transportFailure,
|
|
36001
|
-
keyringJitter,
|
|
36002
|
-
rawErrText,
|
|
36003
|
-
isRateLimit,
|
|
36004
|
-
silentTimeout
|
|
36005
|
-
}),
|
|
36372
|
+
failure,
|
|
36006
36373
|
rawErrText,
|
|
36007
36374
|
runLogText,
|
|
36008
36375
|
silentTimeout
|
|
@@ -36025,12 +36392,15 @@ async function handleAttemptOutcome(deps, input, ctx) {
|
|
|
36025
36392
|
const { cfg, sessionKey } = input;
|
|
36026
36393
|
const outcome = await ctx.proc.outcome;
|
|
36027
36394
|
slot.releaseOnce();
|
|
36395
|
+
recorder.flushTracker();
|
|
36028
36396
|
recorder.stop();
|
|
36029
36397
|
deps.unregisterActiveRun(sessionKey, rec);
|
|
36030
36398
|
deps.clearInFlightPrompt(sessionKey, spawn.prompt);
|
|
36031
36399
|
recorder.drain(input.agyOnEvent);
|
|
36400
|
+
const quotaFailover = recorder.quotaFailover();
|
|
36401
|
+
const cooledInLiveFamilies = recorder.cooledInLiveFamilies();
|
|
36032
36402
|
const conversationId = recorder.conversationId();
|
|
36033
|
-
if (!rec.hasConsumableResult && outcome.idleGraceTerminated) {
|
|
36403
|
+
if (!rec.hasConsumableResult && outcome.idleGraceTerminated && quotaFailover === null) {
|
|
36034
36404
|
const recovered = conversationId === null ? "" : recoverFinalReply(conversationId, rec.accountHome);
|
|
36035
36405
|
const synthResult = {
|
|
36036
36406
|
kind: "result",
|
|
@@ -36049,41 +36419,47 @@ async function handleAttemptOutcome(deps, input, ctx) {
|
|
|
36049
36419
|
parser,
|
|
36050
36420
|
rec,
|
|
36051
36421
|
runLogHome,
|
|
36052
|
-
runLogsBefore
|
|
36422
|
+
runLogsBefore,
|
|
36423
|
+
quotaFailover
|
|
36053
36424
|
});
|
|
36054
36425
|
if (failure !== null) {
|
|
36055
36426
|
if (failure.code === Err.NETWORK) failure.retryAfterMs = networkRetryDelayMs(deps.noteNetworkFailure(sessionKey));
|
|
36056
36427
|
else deps.clearNetworkStreak(sessionKey);
|
|
36057
36428
|
}
|
|
36058
|
-
rec.settle(failure);
|
|
36059
36429
|
const superseded = deps.isSuperseded(sessionKey, rec);
|
|
36060
|
-
|
|
36061
|
-
|
|
36062
|
-
|
|
36063
|
-
|
|
36064
|
-
|
|
36065
|
-
|
|
36066
|
-
|
|
36067
|
-
|
|
36068
|
-
|
|
36069
|
-
|
|
36070
|
-
|
|
36071
|
-
|
|
36072
|
-
|
|
36073
|
-
|
|
36074
|
-
|
|
36075
|
-
|
|
36076
|
-
|
|
36077
|
-
|
|
36078
|
-
|
|
36079
|
-
|
|
36080
|
-
|
|
36081
|
-
|
|
36082
|
-
|
|
36083
|
-
|
|
36084
|
-
|
|
36085
|
-
|
|
36086
|
-
|
|
36430
|
+
try {
|
|
36431
|
+
await finalizeAttempt({
|
|
36432
|
+
rec,
|
|
36433
|
+
outcome,
|
|
36434
|
+
failure,
|
|
36435
|
+
store: deps.store,
|
|
36436
|
+
...deps.pool === void 0 ? {} : { pool: deps.pool },
|
|
36437
|
+
...input.account === void 0 ? {} : { account: input.account },
|
|
36438
|
+
family: input.family,
|
|
36439
|
+
persistBinding: input.persistBinding,
|
|
36440
|
+
sessionAccountKey: input.sessionAccountKey,
|
|
36441
|
+
...input.binding === void 0 ? {} : { binding: input.binding },
|
|
36442
|
+
conversationId,
|
|
36443
|
+
messages: input.messages,
|
|
36444
|
+
model: input.model,
|
|
36445
|
+
activeModel: input.activeModel,
|
|
36446
|
+
superseded,
|
|
36447
|
+
newlySentImageIds: input.newlySentImageIds,
|
|
36448
|
+
rawErrText,
|
|
36449
|
+
runLogText,
|
|
36450
|
+
silentTimeout,
|
|
36451
|
+
sessionKey,
|
|
36452
|
+
quotaFailover,
|
|
36453
|
+
cooledInLiveFamilies,
|
|
36454
|
+
clearNetworkStreak: (key) => deps.clearNetworkStreak(key),
|
|
36455
|
+
dropFailoverBudget: (key) => deps.dropFailoverBudget(key),
|
|
36456
|
+
clearRateLimitFailover: (key) => deps.clearRateLimitFailover(key),
|
|
36457
|
+
sharesSingleEgress: () => deps.sharesSingleEgress(),
|
|
36458
|
+
...deps.onRun === void 0 ? {} : { onRun: deps.onRun }
|
|
36459
|
+
});
|
|
36460
|
+
} finally {
|
|
36461
|
+
rec.settle(failure);
|
|
36462
|
+
}
|
|
36087
36463
|
}
|
|
36088
36464
|
/**
|
|
36089
36465
|
* Run one attempt: recorder → slot dispatch → spawn → concurrent outcome.
|
|
@@ -36091,7 +36467,10 @@ async function handleAttemptOutcome(deps, input, ctx) {
|
|
|
36091
36467
|
* queue notices; the outcome closure runs detached exactly as before.
|
|
36092
36468
|
*/
|
|
36093
36469
|
async function* runAgyAttempt(deps, input) {
|
|
36094
|
-
const recorder = createAttemptRecorder(deps, input.cfg, input.account, input.binding?.conversationId
|
|
36470
|
+
const recorder = createAttemptRecorder(deps, input.cfg, input.account, input.binding?.conversationId, true, {
|
|
36471
|
+
family: input.family,
|
|
36472
|
+
activeModel: input.activeModel
|
|
36473
|
+
});
|
|
36095
36474
|
const spawn = buildSpawnArgs(input, recorder.printTimeoutMinutes, deps.log);
|
|
36096
36475
|
const slot = yield* dispatchCredentialSlot(deps, {
|
|
36097
36476
|
...input,
|
|
@@ -36258,21 +36637,14 @@ var AgyAdapter = class extends LlmAdapter {
|
|
|
36258
36637
|
buildArgs(opts) {
|
|
36259
36638
|
return buildAgyArgs(opts);
|
|
36260
36639
|
}
|
|
36261
|
-
/** Effort and
|
|
36640
|
+
/** Effort, model, and aux-quota self-healing wrapper around streamAttempt (issue #67, PR #100). */
|
|
36262
36641
|
async *stream(options) {
|
|
36263
36642
|
yield* streamWithEffortRetry({
|
|
36264
36643
|
streamAttempt: (opts, suppressEffort) => this.streamAttempt(opts, suppressEffort),
|
|
36265
36644
|
learnEffortUnsupported: (model) => this.learnEffortUnsupported(model),
|
|
36266
36645
|
catalogAdvertisesEffort: (model) => this.deps.catalog.advertisedEfforts(model).length > 0,
|
|
36267
|
-
|
|
36268
|
-
|
|
36269
|
-
const pool = this.deps.pool;
|
|
36270
|
-
if (pool) {
|
|
36271
|
-
const currentId = pool.getActiveAccountId(fam);
|
|
36272
|
-
if (currentId) pool.markModelUnsupported(currentId, unrecog);
|
|
36273
|
-
pool.selectAccount(fam, unrecog);
|
|
36274
|
-
}
|
|
36275
|
-
}
|
|
36646
|
+
pool: this.deps.pool,
|
|
36647
|
+
defaultModel: () => this.deps.getConfig().defaultModel
|
|
36276
36648
|
}, options);
|
|
36277
36649
|
}
|
|
36278
36650
|
async *streamAttempt(options, suppressEffort = false) {
|
|
@@ -36688,12 +37060,12 @@ function createImageCatalog(cfg) {
|
|
|
36688
37060
|
}, cfg.fallbackModels, 3e5, void 0, { cacheFile: join(stateDir(), "model-catalog.json") });
|
|
36689
37061
|
}
|
|
36690
37062
|
/** Build the real adapter (same deps as the smoke scripts / host). */
|
|
36691
|
-
function createImageAdapter(cfg, log, catalog = createImageCatalog(cfg)) {
|
|
37063
|
+
function createImageAdapter(cfg, log, catalog = createImageCatalog(cfg), pool = new AccountPoolManager()) {
|
|
36692
37064
|
return new AgyAdapter({
|
|
36693
37065
|
getConfig: () => cfg,
|
|
36694
37066
|
catalog,
|
|
36695
37067
|
store: new SessionStore(join(stateDir(), "cli-image-sessions.json")),
|
|
36696
|
-
pool
|
|
37068
|
+
pool,
|
|
36697
37069
|
bin: () => resolveAgyBin(cfg),
|
|
36698
37070
|
acquire: () => Promise.resolve(() => void 0),
|
|
36699
37071
|
runs: new RunRegistry(),
|
|
@@ -36706,12 +37078,14 @@ function createDefaultImageService(args, _req, writeErr) {
|
|
|
36706
37078
|
if (cfg.workspaceRoot === "") cfg.workspaceRoot = process.cwd();
|
|
36707
37079
|
const bin = resolveAgyBin(cfg);
|
|
36708
37080
|
const catalog = createImageCatalog(cfg);
|
|
36709
|
-
const
|
|
37081
|
+
const pool = new AccountPoolManager();
|
|
37082
|
+
const adapter = createImageAdapter(cfg, args.quiet ? void 0 : (m) => writeErr("[agy] " + m + "\n"), catalog, pool);
|
|
36710
37083
|
return new ImageService({
|
|
36711
37084
|
cfg: () => cfg,
|
|
36712
37085
|
bin: () => bin,
|
|
36713
37086
|
run: (call) => runViaAdapter(adapter, call),
|
|
36714
37087
|
catalog: () => catalog.get(),
|
|
37088
|
+
...createPoolQuotaHooks(pool),
|
|
36715
37089
|
...args.quiet ? {} : { log: (m) => writeErr("[image] " + m + "\n") }
|
|
36716
37090
|
});
|
|
36717
37091
|
}
|