dsh-agy-link 0.6.55 → 0.6.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +111 -117
- package/dist/cli-image.mjs +919 -306
- package/dist/client.js +7 -2
- package/dist/index.js +1172 -299
- package/docs/ARCHITECTURE.md +4 -4
- package/docs/CONFIGURATION.md +20 -14
- package/docs/KNOWN-GAPS.md +2 -8
- package/docs/README.en.md +76 -30
- package/docs/SECURITY-NOTES.md +2 -0
- package/docs/assets/console.png +0 -0
- package/docs/assets/context-metering.png +0 -0
- package/docs/assets/model-picker.png +0 -0
- package/docs/assets/settings.png +0 -0
- package/docs/assets/token-usage.png +0 -0
- package/package.json +1 -1
package/dist/cli-image.mjs
CHANGED
|
@@ -457,7 +457,7 @@ function extractAuthUrl(text) {
|
|
|
457
457
|
*/
|
|
458
458
|
function looksLikeHardRateLimit(text) {
|
|
459
459
|
if (!text) return false;
|
|
460
|
-
return /RESOURCE_EXHAUSTED|code[
|
|
460
|
+
return /RESOURCE_EXHAUSTED|\bQUOTA_EXHAUSTED\b|"?code"?\s*[:=]\s*429\b|"?status"?\s*[:=]\s*429\b|HTTP(?:\/[\d.]+)?[ :]?429\b|too many requests|individual quota reached|quota (?:exceeded|reached|exhausted|limit)|rate[ -]?limit(?:ed)? (?:exceeded|reached|hit)?|exceeded (?:your |the )?quota|has been exhausted|exhausted your (?:quota|capacity)|out of quota|check quota|insufficient quota|usage limit reached/i.test(text);
|
|
461
461
|
}
|
|
462
462
|
/**
|
|
463
463
|
* Failures that belong to the ACCOUNT (spent quota, refused region, broken
|
|
@@ -491,18 +491,19 @@ function looksLikeRateLimit(text) {
|
|
|
491
491
|
}
|
|
492
492
|
/**
|
|
493
493
|
* Parse reset duration in milliseconds from rate-limit / quota-exhausted error strings.
|
|
494
|
-
* Supports compact formats ("Resets in 21m25s", "
|
|
495
|
-
* verbose formats ("Resets in 15 minutes",
|
|
494
|
+
* Supports compact formats ("Resets in 21m25s", "reset after 4h58m46s",
|
|
495
|
+
* `"quotaResetDelay": "4h58m46.438s"`), verbose formats ("Resets in 15 minutes",
|
|
496
|
+
* "retry after 30 seconds"), and ISO timestamps.
|
|
496
497
|
*/
|
|
497
498
|
function parseResetDurationMs(text) {
|
|
498
499
|
if (!text) return void 0;
|
|
499
|
-
const compactMatch = text.match(/resets?\s+in\s
|
|
500
|
-
if (compactMatch && compactMatch[
|
|
500
|
+
const compactMatch = text.match(/(?:resets?\s+(?:in|after)|"(?:quotaResetDelay|retryDelay)"\s*:\s*")\s*((?:(\d+)\s*h)?\s*(?:(\d+)\s*m)?\s*(?:(\d+(?:\.\d+)?)\s*s)?)/i);
|
|
501
|
+
if (compactMatch && (compactMatch[2] || compactMatch[3] || compactMatch[4])) {
|
|
501
502
|
const hours = parseInt(compactMatch[2] || "0", 10);
|
|
502
503
|
const minutes = parseInt(compactMatch[3] || "0", 10);
|
|
503
|
-
const seconds =
|
|
504
|
-
const totalMs = (hours * 3600 + minutes * 60 + seconds) * 1e3;
|
|
505
|
-
if (totalMs > 0) return totalMs;
|
|
504
|
+
const seconds = parseFloat(compactMatch[4] || "0");
|
|
505
|
+
const totalMs = Math.round((hours * 3600 + minutes * 60 + seconds) * 1e3);
|
|
506
|
+
if (Number.isFinite(totalMs) && totalMs > 0) return totalMs;
|
|
506
507
|
}
|
|
507
508
|
const wordMatch = text.match(/(?:resets?|retry)\s+(?:in|after)\s+(\d+)\s*(hour|hr|minute|min|second|sec)s?/i);
|
|
508
509
|
if (wordMatch) {
|
|
@@ -518,6 +519,45 @@ function parseResetDurationMs(text) {
|
|
|
518
519
|
if (!Number.isNaN(parsed) && parsed > Date.now()) return parsed - Date.now();
|
|
519
520
|
}
|
|
520
521
|
}
|
|
522
|
+
const TRANSIENT_BURST_CAP_MS = 3e4;
|
|
523
|
+
/**
|
|
524
|
+
* True when a 429 error represents a short transient per-minute/per-second QPS burst
|
|
525
|
+
* (`reason: RATE_LIMIT_EXCEEDED` with reset <= 30s) rather than window quota exhaustion.
|
|
526
|
+
* Must NOT trigger long account cooldown or kill a running parent conversation.
|
|
527
|
+
*/
|
|
528
|
+
function isTransientBurstRateLimit(text) {
|
|
529
|
+
if (!text) return false;
|
|
530
|
+
if (!/\bRATE_LIMIT_EXCEEDED\b/i.test(text)) return false;
|
|
531
|
+
if (/\bQUOTA_EXHAUSTED\b|individual quota reached/i.test(text)) return false;
|
|
532
|
+
const resetMs = parseResetDurationMs(text);
|
|
533
|
+
return resetMs !== void 0 && resetMs > 0 && resetMs <= TRANSIENT_BURST_CAP_MS;
|
|
534
|
+
}
|
|
535
|
+
/** Extract Google RPC error metadata model slug (e.g. "gemini-3.1-flash-image"), if present. */
|
|
536
|
+
function extractQuotaErrorModel(text) {
|
|
537
|
+
if (!text) return void 0;
|
|
538
|
+
return text.match(/"model"\s*:\s*"([A-Za-z0-9._:/-]+)"/i)?.[1];
|
|
539
|
+
}
|
|
540
|
+
/**
|
|
541
|
+
* True when a failed result envelope carries a hard quota refusal (meaning the
|
|
542
|
+
* attempt must rotate accounts rather than soft-finish on any partial text).
|
|
543
|
+
*/
|
|
544
|
+
function isHardQuotaResultError(ok, error, response) {
|
|
545
|
+
if (ok) return false;
|
|
546
|
+
if (error && error.trim() !== "") return looksLikeHardRateLimit(error) && !isTransientBurstRateLimit(error);
|
|
547
|
+
return Boolean(response && looksLikeHardRateLimit(response) && !isTransientBurstRateLimit(response));
|
|
548
|
+
}
|
|
549
|
+
/**
|
|
550
|
+
* True when a failed result envelope represents a non-consumable account/policy
|
|
551
|
+
* failure even when partial response text was already buffered before the error.
|
|
552
|
+
*/
|
|
553
|
+
function isNonConsumableResultError(ok, error, response) {
|
|
554
|
+
if (ok) return false;
|
|
555
|
+
if (isHardQuotaResultError(ok, error, response)) return true;
|
|
556
|
+
if (looksLikeTosViolation(error) || looksLikeTosViolation(response)) return true;
|
|
557
|
+
if (looksLikeValidationRequired(error) || looksLikeValidationRequired(response)) return true;
|
|
558
|
+
if (error && (looksLikeEligibilityFailure(error) || looksLikeAuthFailure(error))) return true;
|
|
559
|
+
return false;
|
|
560
|
+
}
|
|
521
561
|
//#endregion
|
|
522
562
|
//#region src/common/config.ts
|
|
523
563
|
function dshHome() {
|
|
@@ -1159,6 +1199,21 @@ function simplifyRatio(width, height) {
|
|
|
1159
1199
|
}
|
|
1160
1200
|
//#endregion
|
|
1161
1201
|
//#region src/features/image/service.ts
|
|
1202
|
+
function isHardQuotaErrorText(text) {
|
|
1203
|
+
return Boolean(text && text.trim() !== "" && looksLikeHardRateLimit(text) && !isTransientBurstRateLimit(text) && !looksLikeTosViolation(text));
|
|
1204
|
+
}
|
|
1205
|
+
function detectHardQuotaError(res, replyError) {
|
|
1206
|
+
if (!res.ok) {
|
|
1207
|
+
const err = res.error ?? "";
|
|
1208
|
+
if (looksLikeTosViolation(err)) return null;
|
|
1209
|
+
if (res.code === Err.RATE_LIMIT || res.code === Err.QUOTA || isHardQuotaErrorText(err)) return isTransientBurstRateLimit(err) ? null : err || "Antigravity quota exhausted";
|
|
1210
|
+
return null;
|
|
1211
|
+
}
|
|
1212
|
+
const toolHit = res.toolCalls.find((t) => (t.name === "generate_image" || t.name === "invoke_subagent") && isHardQuotaErrorText(t.error));
|
|
1213
|
+
if (toolHit?.error) return toolHit.error;
|
|
1214
|
+
if (isHardQuotaErrorText(replyError)) return replyError;
|
|
1215
|
+
return null;
|
|
1216
|
+
}
|
|
1162
1217
|
function emptyImageResult(req, model, effort) {
|
|
1163
1218
|
return {
|
|
1164
1219
|
ok: false,
|
|
@@ -1316,6 +1371,8 @@ var ImageService = class {
|
|
|
1316
1371
|
let last = ctx.last;
|
|
1317
1372
|
let effort = target.effort;
|
|
1318
1373
|
let effortFallbackUsed = false;
|
|
1374
|
+
let quotaHopsUsed = 0;
|
|
1375
|
+
const maxQuotaHops = Math.min(Math.max(0, Math.round(this.deps.poolQuotaHops?.() ?? 0)), 16);
|
|
1319
1376
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
1320
1377
|
const startedAt = this.now();
|
|
1321
1378
|
last.effort = effort;
|
|
@@ -1328,9 +1385,34 @@ var ImageService = class {
|
|
|
1328
1385
|
imagePaths: refs,
|
|
1329
1386
|
...signal === void 0 ? {} : { signal }
|
|
1330
1387
|
});
|
|
1331
|
-
last.attempts = attempt;
|
|
1388
|
+
last.attempts = attempt + quotaHopsUsed;
|
|
1332
1389
|
last.durationMs = this.now() - startedAt;
|
|
1333
1390
|
if (res.conversationId !== void 0) last.conversationId = res.conversationId;
|
|
1391
|
+
if (!res.ok && looksLikeTosViolation(res.error)) return {
|
|
1392
|
+
...last,
|
|
1393
|
+
ok: false,
|
|
1394
|
+
code: "AGY_ERROR",
|
|
1395
|
+
error: res.error ?? "account suspended (TOS_VIOLATION)"
|
|
1396
|
+
};
|
|
1397
|
+
const reply = res.ok ? parseImageReply(res.text) : { error: void 0 };
|
|
1398
|
+
const quotaErr = detectHardQuotaError(res, reply.error);
|
|
1399
|
+
if (quotaErr !== null) {
|
|
1400
|
+
last = {
|
|
1401
|
+
...last,
|
|
1402
|
+
ok: false,
|
|
1403
|
+
code: "NO_ACCOUNT",
|
|
1404
|
+
error: quotaErr
|
|
1405
|
+
};
|
|
1406
|
+
const errModel = extractQuotaErrorModel(quotaErr) ?? target.model;
|
|
1407
|
+
const canSwitch = this.deps.onQuotaExhausted ? this.deps.onQuotaExhausted(res.accountId, errModel, quotaErr) : true;
|
|
1408
|
+
if (canSwitch && quotaHopsUsed < maxQuotaHops) {
|
|
1409
|
+
quotaHopsUsed++;
|
|
1410
|
+
attempt--;
|
|
1411
|
+
continue;
|
|
1412
|
+
}
|
|
1413
|
+
if (canSwitch && attempt < maxAttempts) continue;
|
|
1414
|
+
return last;
|
|
1415
|
+
}
|
|
1334
1416
|
if (!res.ok) {
|
|
1335
1417
|
const failure = res.error ?? "";
|
|
1336
1418
|
if (!effortFallbackUsed && effort !== "" && /--effort/i.test(failure) && /not supported/i.test(failure)) {
|
|
@@ -1360,7 +1442,6 @@ var ImageService = class {
|
|
|
1360
1442
|
if (attempt < maxAttempts) continue;
|
|
1361
1443
|
return last;
|
|
1362
1444
|
}
|
|
1363
|
-
const reply = parseImageReply(res.text);
|
|
1364
1445
|
if (reply.error !== void 0) {
|
|
1365
1446
|
last = {
|
|
1366
1447
|
...last,
|
|
@@ -1559,6 +1640,14 @@ async function runViaAdapter(adapter, call) {
|
|
|
1559
1640
|
agyImagePrompt: call.prompt,
|
|
1560
1641
|
agyPermissionMode: call.permissionMode,
|
|
1561
1642
|
agyOnAccount: (info) => {
|
|
1643
|
+
if (accountId !== void 0 && typeof info.id === "string" && info.id !== accountId) {
|
|
1644
|
+
toolCalls.length = 0;
|
|
1645
|
+
subagents.length = 0;
|
|
1646
|
+
textParts.length = 0;
|
|
1647
|
+
responseText = "";
|
|
1648
|
+
conversationId = void 0;
|
|
1649
|
+
failure = null;
|
|
1650
|
+
}
|
|
1562
1651
|
if (typeof info.id === "string") accountId = info.id;
|
|
1563
1652
|
if (typeof info.home === "string" && info.home !== "") home = info.home;
|
|
1564
1653
|
},
|
|
@@ -1628,121 +1717,6 @@ async function runViaAdapter(adapter, call) {
|
|
|
1628
1717
|
};
|
|
1629
1718
|
}
|
|
1630
1719
|
//#endregion
|
|
1631
|
-
//#region src/common/pool-types.ts
|
|
1632
|
-
/** Check if a model identifier belongs to the Gemini 4 Argon family. */
|
|
1633
|
-
function isArgonModel(modelId) {
|
|
1634
|
-
if (!modelId) return false;
|
|
1635
|
-
return modelId.trim().toLowerCase().includes("argon");
|
|
1636
|
-
}
|
|
1637
|
-
/**
|
|
1638
|
-
* Map a model slug to its backend quota counter family.
|
|
1639
|
-
*
|
|
1640
|
-
* Gemini 4 Argon (Pro/Ultra-only) bills against its OWN weekly quota pool —
|
|
1641
|
-
* never the plain Gemini one. Its ids look like `gemini-4-argon-low/-medium`
|
|
1642
|
-
* and MUST be detected before the generic `gemini-` prefix below, otherwise
|
|
1643
|
-
* argon usage would silently drain/report the Gemini counter.
|
|
1644
|
-
*/
|
|
1645
|
-
function modelFamilyOf(modelId) {
|
|
1646
|
-
if (!modelId) return "unknown";
|
|
1647
|
-
const id = modelId.toLowerCase();
|
|
1648
|
-
if (id.includes("argon")) return "argon";
|
|
1649
|
-
if (id.startsWith("claude-") || id.includes("claude")) return "anthropic";
|
|
1650
|
-
if (id.startsWith("gemini-") || id.startsWith("gemma-") || id.includes("gemini")) return "google";
|
|
1651
|
-
if (id.startsWith("gpt-") || id.startsWith("openai/") || id.includes("gpt-oss")) return "openai";
|
|
1652
|
-
return "unknown";
|
|
1653
|
-
}
|
|
1654
|
-
/**
|
|
1655
|
-
* Determine whether a managed account has confirmed access to the Gemini 4 Argon family.
|
|
1656
|
-
*
|
|
1657
|
-
* An account qualifies for Argon when:
|
|
1658
|
-
* 1. It carries an authoritative Argon quota bucket.
|
|
1659
|
-
* 2. It has a dedicated Argon group recorded by the pool.
|
|
1660
|
-
* 3. Its Google per-model quota breakdown includes an Argon model.
|
|
1661
|
-
* 4. Or its discovered catalog includes Argon (when no negative quota breakdown is present).
|
|
1662
|
-
*/
|
|
1663
|
-
function accountHasArgon(acc, pool, catalog) {
|
|
1664
|
-
if (!acc) return false;
|
|
1665
|
-
if (acc.quotas?.argon !== void 0) return true;
|
|
1666
|
-
if (pool && typeof pool.hasDedicatedGroup === "function" && pool.hasDedicatedGroup(acc.id, "argon")) return true;
|
|
1667
|
-
if (acc.quotas?.google?.models?.some((m) => isArgonModel(m.modelId))) return true;
|
|
1668
|
-
if (Array.isArray(acc.quotas?.google?.models)) return false;
|
|
1669
|
-
if (catalog && catalog.source === "discovered") return catalog.models?.some((m) => isArgonModel(m.id)) ?? false;
|
|
1670
|
-
return false;
|
|
1671
|
-
}
|
|
1672
|
-
const EFFORT_SUFFIX_RE = /-(?:tiered-)?(?:low|medium|high|xhigh|tiered|thinking)$/i;
|
|
1673
|
-
/**
|
|
1674
|
-
* Return the un-tiered base model slug (e.g. claude-opus-4-6-thinking -> claude-opus-4-6,
|
|
1675
|
-
* claude-opus-5-5-high -> claude-opus-5-5).
|
|
1676
|
-
*/
|
|
1677
|
-
function baseModelSlug(id) {
|
|
1678
|
-
const s = id.trim().toLowerCase().replace(/\./g, "-");
|
|
1679
|
-
if (s === "claude-opus-4-6" || s === "claude-opus-4-8" || s === "claude-opus-4-5") return "claude-opus-4-6";
|
|
1680
|
-
if (s === "claude-sonnet-4-5") return "claude-sonnet-4-6";
|
|
1681
|
-
return s.replace(EFFORT_SUFFIX_RE, "");
|
|
1682
|
-
}
|
|
1683
|
-
/**
|
|
1684
|
-
* Test whether a requested model matches a candidate model id (accounting for
|
|
1685
|
-
* aliases, effort variants, and base model families).
|
|
1686
|
-
*/
|
|
1687
|
-
function modelMatches(requestedModel, candidateModelId) {
|
|
1688
|
-
if (!requestedModel || !candidateModelId) return false;
|
|
1689
|
-
const r = requestedModel.trim().toLowerCase();
|
|
1690
|
-
const c = candidateModelId.trim().toLowerCase();
|
|
1691
|
-
if (r === c) return true;
|
|
1692
|
-
return baseModelSlug(r) === baseModelSlug(c);
|
|
1693
|
-
}
|
|
1694
|
-
function quotaSupportsModel(acc, clean, pool, catalog) {
|
|
1695
|
-
const familyQuotaModels = acc.quotas?.[modelFamilyOf(clean)]?.models;
|
|
1696
|
-
if (Array.isArray(familyQuotaModels) && familyQuotaModels.some((m) => modelMatches(clean, m.modelId))) return true;
|
|
1697
|
-
const lower = clean.toLowerCase();
|
|
1698
|
-
if (lower === "gemini-pro-orchestrator-neon") return Array.isArray(familyQuotaModels) && familyQuotaModels.some((m) => modelMatches("gemini-4-neon", m.modelId));
|
|
1699
|
-
if (lower === "gemini-pro-orchestrator-g1-pro") return accountHasArgon(acc, pool, catalog);
|
|
1700
|
-
return false;
|
|
1701
|
-
}
|
|
1702
|
-
/**
|
|
1703
|
-
* Determine whether a managed account supports a specific model.
|
|
1704
|
-
* If the account's quota or discovered catalog enumerates models for that family,
|
|
1705
|
-
* the list is authoritative: the model must match an available entry.
|
|
1706
|
-
*/
|
|
1707
|
-
function accountSupportsModel(acc, model, pool, catalog) {
|
|
1708
|
-
if (!acc) return false;
|
|
1709
|
-
if (!model || model.trim() === "") return true;
|
|
1710
|
-
const clean = model.trim();
|
|
1711
|
-
if (isArgonModel(clean)) return accountHasArgon(acc, pool, catalog);
|
|
1712
|
-
if (catalog && catalog.source === "discovered" && Array.isArray(catalog.models) && catalog.models.length > 0) {
|
|
1713
|
-
if (catalog.models.some((m) => modelMatches(clean, m.id))) return true;
|
|
1714
|
-
if (quotaSupportsModel(acc, clean, pool, catalog)) return true;
|
|
1715
|
-
return false;
|
|
1716
|
-
}
|
|
1717
|
-
const family = modelFamilyOf(clean);
|
|
1718
|
-
const familyQuota = acc.quotas?.[family];
|
|
1719
|
-
if (familyQuota) {
|
|
1720
|
-
if (Array.isArray(familyQuota.models) && familyQuota.models.length > 0) return quotaSupportsModel(acc, clean, pool, catalog);
|
|
1721
|
-
return true;
|
|
1722
|
-
}
|
|
1723
|
-
if (acc.quotas && Object.keys(acc.quotas).length > 0 && acc.quotas.google !== void 0) return false;
|
|
1724
|
-
return true;
|
|
1725
|
-
}
|
|
1726
|
-
/**
|
|
1727
|
-
* A flagged account whose flag is old enough (or whose flag predates the
|
|
1728
|
-
* authMarkedAt bookkeeping) may be re-verified in the background. If the
|
|
1729
|
-
* stored token still works the flag clears itself; if not, the flag is
|
|
1730
|
-
* simply re-stamped.
|
|
1731
|
-
*/
|
|
1732
|
-
/** A terminal account ban (Google TOS_VIOLATION) is not a transient auth error. */
|
|
1733
|
-
function isTerminalAuthError(message) {
|
|
1734
|
-
return typeof message === "string" && /\bTOS_VIOLATION\b|violation of Terms of Service/i.test(message);
|
|
1735
|
-
}
|
|
1736
|
-
function defaultPoolData() {
|
|
1737
|
-
return {
|
|
1738
|
-
version: 1,
|
|
1739
|
-
mode: "sequential",
|
|
1740
|
-
defaultCooldownMs: 9e5,
|
|
1741
|
-
maxCooldownMs: 36e5,
|
|
1742
|
-
accounts: []
|
|
1743
|
-
};
|
|
1744
|
-
}
|
|
1745
|
-
//#endregion
|
|
1746
1720
|
//#region src/runtime/agy/bin.ts
|
|
1747
1721
|
const IS_WIN$3 = process.platform === "win32";
|
|
1748
1722
|
/** Executable candidates for one PATH entry, per-platform. Exported for tests. */
|
|
@@ -1999,6 +1973,9 @@ function renderCompactionMarker(notice) {
|
|
|
1999
1973
|
//#endregion
|
|
2000
1974
|
//#region src/host/recording.ts
|
|
2001
1975
|
const waiters = Symbol("waiters");
|
|
1976
|
+
function isConsumableResult(ev) {
|
|
1977
|
+
return (ev.ok || ev.response !== "") && !isNonConsumableResultError(ev.ok, ev.error, ev.response);
|
|
1978
|
+
}
|
|
2002
1979
|
/** One agy run's append-only event log plus its settlement state. */
|
|
2003
1980
|
var RunRecording = class {
|
|
2004
1981
|
now;
|
|
@@ -2154,10 +2131,10 @@ var RunRecording = class {
|
|
|
2154
2131
|
get hasResult() {
|
|
2155
2132
|
return this.events.some((e) => e.kind === "result");
|
|
2156
2133
|
}
|
|
2157
|
-
/** Whether a result event that EventMapper will finish on (ok, or non-empty error response) was recorded. */
|
|
2134
|
+
/** Whether a result event that EventMapper will finish on (ok, or non-empty consumable error response) was recorded. */
|
|
2158
2135
|
get hasConsumableResult() {
|
|
2159
2136
|
const r = this.getResultEvent();
|
|
2160
|
-
return r !== null && (r
|
|
2137
|
+
return r !== null && isConsumableResult(r);
|
|
2161
2138
|
}
|
|
2162
2139
|
/**
|
|
2163
2140
|
* Whether any recorded event at or after `from` can finish a span cleanly
|
|
@@ -2170,7 +2147,7 @@ var RunRecording = class {
|
|
|
2170
2147
|
for (let i = Math.max(0, from); i < this.events.length; i++) {
|
|
2171
2148
|
const ev = this.events[i];
|
|
2172
2149
|
if (ev === void 0) continue;
|
|
2173
|
-
if (ev.kind === "result" && (ev
|
|
2150
|
+
if (ev.kind === "result" && isConsumableResult(ev)) return true;
|
|
2174
2151
|
if (cutOnTool && ev.kind === "step" && ev.stepKind === "tool" && ev.tool !== void 0 && (ev.state === "DONE" || ev.state === "ERROR" || ev.state === "CANCELED" || ev.state === "CANCELLED" || ev.state === "RUNNING" || ev.state === void 0 && (ev.tool.output !== void 0 || ev.tool.error !== void 0))) return true;
|
|
2175
2152
|
}
|
|
2176
2153
|
return false;
|
|
@@ -2181,22 +2158,21 @@ var RunRecording = class {
|
|
|
2181
2158
|
const ev = this.events[i];
|
|
2182
2159
|
if (ev !== void 0 && ev.kind === "result") return {
|
|
2183
2160
|
ok: ev.ok,
|
|
2184
|
-
response: ev.response
|
|
2161
|
+
response: ev.response,
|
|
2162
|
+
...ev.error === void 0 ? {} : { error: ev.error }
|
|
2185
2163
|
};
|
|
2186
2164
|
}
|
|
2187
2165
|
return null;
|
|
2188
2166
|
}
|
|
2189
2167
|
/**
|
|
2190
2168
|
* True when at least one consumable `result` event (`ok` or non-empty
|
|
2191
|
-
* `response
|
|
2192
|
-
* `failureInfo === null` and no consumable result at `>= from` would always
|
|
2193
|
-
* fail with `agy stream ended without a result event` if re-driven.
|
|
2169
|
+
* `response`, excluding hard quota / policy errors) exists at index >= `from`.
|
|
2194
2170
|
*/
|
|
2195
2171
|
hasConsumableResultFrom(from) {
|
|
2196
2172
|
const start = Math.max(0, from);
|
|
2197
2173
|
for (let i = start; i < this.events.length; i++) {
|
|
2198
2174
|
const ev = this.events[i];
|
|
2199
|
-
if (ev !== void 0 && ev.kind === "result" && (ev
|
|
2175
|
+
if (ev !== void 0 && ev.kind === "result" && isConsumableResult(ev)) return true;
|
|
2200
2176
|
}
|
|
2201
2177
|
return false;
|
|
2202
2178
|
}
|
|
@@ -3032,7 +3008,7 @@ function startAgyProcess(opts) {
|
|
|
3032
3008
|
},
|
|
3033
3009
|
kill: (reason) => {
|
|
3034
3010
|
if (reason === "timeout") state.timedOut = true;
|
|
3035
|
-
else state.aborted = true;
|
|
3011
|
+
else if (reason === "abort") state.aborted = true;
|
|
3036
3012
|
if (state.watchdog) clearTimeout(state.watchdog);
|
|
3037
3013
|
if (state.turnCap) clearTimeout(state.turnCap);
|
|
3038
3014
|
if (state.idleGraceTimer) clearTimeout(state.idleGraceTimer);
|
|
@@ -4298,6 +4274,29 @@ function buildToolCallBlock(input) {
|
|
|
4298
4274
|
}
|
|
4299
4275
|
//#endregion
|
|
4300
4276
|
//#region src/provider/mapper/event-mapper.ts
|
|
4277
|
+
function formatRunningToolFallback(toolName, rawArgs) {
|
|
4278
|
+
const unwrapped = unwrapMcpTool(toolName, rawArgs);
|
|
4279
|
+
const a = unwrapped.input;
|
|
4280
|
+
const str = (...keys) => {
|
|
4281
|
+
for (const k of keys) {
|
|
4282
|
+
const v = a[k];
|
|
4283
|
+
if (typeof v === "string" && v.trim() !== "") return v.trim();
|
|
4284
|
+
if (typeof v === "number" && Number.isFinite(v)) return String(v);
|
|
4285
|
+
}
|
|
4286
|
+
return "";
|
|
4287
|
+
};
|
|
4288
|
+
if (unwrapped.tool === "schedule") {
|
|
4289
|
+
const cron = str("CronExpression", "cron_expression");
|
|
4290
|
+
const dur = str("DurationSeconds", "duration_seconds");
|
|
4291
|
+
const prompt = str("Prompt", "prompt");
|
|
4292
|
+
const timing = cron ? `cron ${cron}` : dur ? `${dur}s timer` : "schedule";
|
|
4293
|
+
return prompt ? `Schedule active (${timing}): ${prompt}` : `Schedule active (${timing})`;
|
|
4294
|
+
}
|
|
4295
|
+
const cmd = str("CommandLine", "command_line", "command", "cmd");
|
|
4296
|
+
const taskId = str("TaskId", "taskId", "task_id", "bgTaskId");
|
|
4297
|
+
if (cmd !== "") return `Tool is running as a background task${taskId ? ` (${taskId})` : ""}: ${cmd}`;
|
|
4298
|
+
return taskId ? `Tool is running as a background task (${taskId})` : "Tool is running as a background task";
|
|
4299
|
+
}
|
|
4301
4300
|
var EventMapper = class {
|
|
4302
4301
|
opts;
|
|
4303
4302
|
blockIdx = 0;
|
|
@@ -4540,7 +4539,7 @@ var EventMapper = class {
|
|
|
4540
4539
|
}
|
|
4541
4540
|
return;
|
|
4542
4541
|
}
|
|
4543
|
-
if (ev.state === "RUNNING" && ev.tool.output === void 0) ev.tool.output = ev.text !== "" ? ev.text :
|
|
4542
|
+
if (ev.state === "RUNNING" && (ev.tool.output === void 0 || ev.tool.output === "")) ev.tool.output = ev.text !== "" ? ev.text : formatRunningToolFallback(ev.tool.name, ev.tool.args);
|
|
4544
4543
|
if (this.announcedTools.has(ev.stepKey)) return;
|
|
4545
4544
|
this.announcedTools.add(ev.stepKey);
|
|
4546
4545
|
if (!this.opts.cutOnTool) return;
|
|
@@ -4567,7 +4566,7 @@ var EventMapper = class {
|
|
|
4567
4566
|
return;
|
|
4568
4567
|
}
|
|
4569
4568
|
if (!ev.ok) {
|
|
4570
|
-
if (ev.response === "") return;
|
|
4569
|
+
if (ev.response === "" || isNonConsumableResultError(ev.ok, ev.error, ev.response)) return;
|
|
4571
4570
|
const respText = this.formatAssistantText(ev.response);
|
|
4572
4571
|
if (!this.sawTextStep) {
|
|
4573
4572
|
yield* this.ensureBlock("text");
|
|
@@ -5754,6 +5753,121 @@ var SpawnLimiter = class {
|
|
|
5754
5753
|
}
|
|
5755
5754
|
};
|
|
5756
5755
|
//#endregion
|
|
5756
|
+
//#region src/common/pool-types.ts
|
|
5757
|
+
/** Check if a model identifier belongs to the Gemini 4 Argon family. */
|
|
5758
|
+
function isArgonModel(modelId) {
|
|
5759
|
+
if (!modelId) return false;
|
|
5760
|
+
return modelId.trim().toLowerCase().includes("argon");
|
|
5761
|
+
}
|
|
5762
|
+
/**
|
|
5763
|
+
* Map a model slug to its backend quota counter family.
|
|
5764
|
+
*
|
|
5765
|
+
* Gemini 4 Argon (Pro/Ultra-only) bills against its OWN weekly quota pool —
|
|
5766
|
+
* never the plain Gemini one. Its ids look like `gemini-4-argon-low/-medium`
|
|
5767
|
+
* and MUST be detected before the generic `gemini-` prefix below, otherwise
|
|
5768
|
+
* argon usage would silently drain/report the Gemini counter.
|
|
5769
|
+
*/
|
|
5770
|
+
function modelFamilyOf(modelId) {
|
|
5771
|
+
if (!modelId) return "unknown";
|
|
5772
|
+
const id = modelId.toLowerCase();
|
|
5773
|
+
if (id.includes("argon")) return "argon";
|
|
5774
|
+
if (id.startsWith("claude-") || id.includes("claude")) return "anthropic";
|
|
5775
|
+
if (id.startsWith("gemini-") || id.startsWith("gemma-") || id.includes("gemini")) return "google";
|
|
5776
|
+
if (id.startsWith("gpt-") || id.startsWith("openai/") || id.includes("gpt-oss")) return "openai";
|
|
5777
|
+
return "unknown";
|
|
5778
|
+
}
|
|
5779
|
+
/**
|
|
5780
|
+
* Determine whether a managed account has confirmed access to the Gemini 4 Argon family.
|
|
5781
|
+
*
|
|
5782
|
+
* An account qualifies for Argon when:
|
|
5783
|
+
* 1. It carries an authoritative Argon quota bucket.
|
|
5784
|
+
* 2. It has a dedicated Argon group recorded by the pool.
|
|
5785
|
+
* 3. Its Google per-model quota breakdown includes an Argon model.
|
|
5786
|
+
* 4. Or its discovered catalog includes Argon (when no negative quota breakdown is present).
|
|
5787
|
+
*/
|
|
5788
|
+
function accountHasArgon(acc, pool, catalog) {
|
|
5789
|
+
if (!acc) return false;
|
|
5790
|
+
if (acc.quotas?.argon !== void 0) return true;
|
|
5791
|
+
if (pool && typeof pool.hasDedicatedGroup === "function" && pool.hasDedicatedGroup(acc.id, "argon")) return true;
|
|
5792
|
+
if (acc.quotas?.google?.models?.some((m) => isArgonModel(m.modelId))) return true;
|
|
5793
|
+
if (Array.isArray(acc.quotas?.google?.models)) return false;
|
|
5794
|
+
if (catalog && catalog.source === "discovered") return catalog.models?.some((m) => isArgonModel(m.id)) ?? false;
|
|
5795
|
+
return false;
|
|
5796
|
+
}
|
|
5797
|
+
const EFFORT_SUFFIX_RE = /-(?:tiered-)?(?:low|medium|high|xhigh|tiered|thinking)$/i;
|
|
5798
|
+
/**
|
|
5799
|
+
* Return the un-tiered base model slug (e.g. claude-opus-4-6-thinking -> claude-opus-4-6,
|
|
5800
|
+
* claude-opus-5-5-high -> claude-opus-5-5).
|
|
5801
|
+
*/
|
|
5802
|
+
function baseModelSlug(id) {
|
|
5803
|
+
const s = id.trim().toLowerCase().replace(/\./g, "-");
|
|
5804
|
+
if (s === "claude-opus-4-6" || s === "claude-opus-4-8" || s === "claude-opus-4-5") return "claude-opus-4-6";
|
|
5805
|
+
if (s === "claude-sonnet-4-5") return "claude-sonnet-4-6";
|
|
5806
|
+
return s.replace(EFFORT_SUFFIX_RE, "");
|
|
5807
|
+
}
|
|
5808
|
+
/**
|
|
5809
|
+
* Test whether a requested model matches a candidate model id (accounting for
|
|
5810
|
+
* aliases, effort variants, and base model families).
|
|
5811
|
+
*/
|
|
5812
|
+
function modelMatches(requestedModel, candidateModelId) {
|
|
5813
|
+
if (!requestedModel || !candidateModelId) return false;
|
|
5814
|
+
const r = requestedModel.trim().toLowerCase();
|
|
5815
|
+
const c = candidateModelId.trim().toLowerCase();
|
|
5816
|
+
if (r === c) return true;
|
|
5817
|
+
return baseModelSlug(r) === baseModelSlug(c);
|
|
5818
|
+
}
|
|
5819
|
+
function quotaSupportsModel(acc, clean, pool, catalog) {
|
|
5820
|
+
const familyQuotaModels = acc.quotas?.[modelFamilyOf(clean)]?.models;
|
|
5821
|
+
if (Array.isArray(familyQuotaModels) && familyQuotaModels.some((m) => modelMatches(clean, m.modelId))) return true;
|
|
5822
|
+
const lower = clean.toLowerCase();
|
|
5823
|
+
if (lower === "gemini-pro-orchestrator-neon") return Array.isArray(familyQuotaModels) && familyQuotaModels.some((m) => modelMatches("gemini-4-neon", m.modelId));
|
|
5824
|
+
if (lower === "gemini-pro-orchestrator-g1-pro") return accountHasArgon(acc, pool, catalog);
|
|
5825
|
+
return false;
|
|
5826
|
+
}
|
|
5827
|
+
/**
|
|
5828
|
+
* Determine whether a managed account supports a specific model.
|
|
5829
|
+
* If the account's quota or discovered catalog enumerates models for that family,
|
|
5830
|
+
* the list is authoritative: the model must match an available entry.
|
|
5831
|
+
*/
|
|
5832
|
+
function accountSupportsModel(acc, model, pool, catalog) {
|
|
5833
|
+
if (!acc) return false;
|
|
5834
|
+
if (!model || model.trim() === "") return true;
|
|
5835
|
+
const clean = model.trim();
|
|
5836
|
+
if (isArgonModel(clean)) return accountHasArgon(acc, pool, catalog);
|
|
5837
|
+
if (catalog && catalog.source === "discovered" && Array.isArray(catalog.models) && catalog.models.length > 0) {
|
|
5838
|
+
if (catalog.models.some((m) => modelMatches(clean, m.id))) return true;
|
|
5839
|
+
if (quotaSupportsModel(acc, clean, pool, catalog)) return true;
|
|
5840
|
+
return false;
|
|
5841
|
+
}
|
|
5842
|
+
const family = modelFamilyOf(clean);
|
|
5843
|
+
const familyQuota = acc.quotas?.[family];
|
|
5844
|
+
if (familyQuota) {
|
|
5845
|
+
if (Array.isArray(familyQuota.models) && familyQuota.models.length > 0) return quotaSupportsModel(acc, clean, pool, catalog);
|
|
5846
|
+
return true;
|
|
5847
|
+
}
|
|
5848
|
+
if (acc.quotas && Object.keys(acc.quotas).length > 0 && acc.quotas.google !== void 0) return false;
|
|
5849
|
+
return true;
|
|
5850
|
+
}
|
|
5851
|
+
/**
|
|
5852
|
+
* A flagged account whose flag is old enough (or whose flag predates the
|
|
5853
|
+
* authMarkedAt bookkeeping) may be re-verified in the background. If the
|
|
5854
|
+
* stored token still works the flag clears itself; if not, the flag is
|
|
5855
|
+
* simply re-stamped.
|
|
5856
|
+
*/
|
|
5857
|
+
/** A terminal account ban (Google TOS_VIOLATION) is not a transient auth error. */
|
|
5858
|
+
function isTerminalAuthError(message) {
|
|
5859
|
+
return typeof message === "string" && /\bTOS_VIOLATION\b|violation of Terms of Service/i.test(message);
|
|
5860
|
+
}
|
|
5861
|
+
function defaultPoolData() {
|
|
5862
|
+
return {
|
|
5863
|
+
version: 1,
|
|
5864
|
+
mode: "sequential",
|
|
5865
|
+
defaultCooldownMs: 9e5,
|
|
5866
|
+
maxCooldownMs: 36e5,
|
|
5867
|
+
accounts: []
|
|
5868
|
+
};
|
|
5869
|
+
}
|
|
5870
|
+
//#endregion
|
|
5757
5871
|
//#region src/common/atomic-write.ts
|
|
5758
5872
|
let seq = 0;
|
|
5759
5873
|
/** Unique temp path beside `file` (same directory, so rename stays atomic). */
|
|
@@ -6672,15 +6786,32 @@ function providerRetryPolicyFor(pool) {
|
|
|
6672
6786
|
Err.OVERLOADED
|
|
6673
6787
|
];
|
|
6674
6788
|
if (enabledAccounts >= 2) retryableCodes.push(Err.AUTH);
|
|
6789
|
+
const poolHops = Math.min(Math.max(1, enabledAccounts - 1), 16);
|
|
6675
6790
|
return {
|
|
6676
6791
|
mode: "normal",
|
|
6677
|
-
maxRetries: Math.max(
|
|
6792
|
+
maxRetries: Math.max(poolHops, NETWORK_RETRY_DELAYS_MS.length),
|
|
6678
6793
|
retryableCodes,
|
|
6679
6794
|
initialDelayMs: 2e3,
|
|
6680
6795
|
maxDelayMs: 45e3,
|
|
6681
6796
|
jitterRatio: .1
|
|
6682
6797
|
};
|
|
6683
6798
|
}
|
|
6799
|
+
function isAuxCall(options) {
|
|
6800
|
+
return (options.purpose ?? "chat") !== "chat" || typeof options.system === "string" && options.system.includes("REVIEW_POLICY");
|
|
6801
|
+
}
|
|
6802
|
+
function canRetryAuxQuota(deps, options, emitted, finish, quotaHops, rawMsg) {
|
|
6803
|
+
if (!isAuxCall(options) || emitted || options.signal?.aborted) return false;
|
|
6804
|
+
if (finish.reason.kind !== "error" || looksLikeTosViolation(rawMsg)) return false;
|
|
6805
|
+
const code = finish.reason.failure?.code;
|
|
6806
|
+
if (code !== Err.RATE_LIMIT && code !== Err.ELIGIBILITY && code !== Err.AUTH) return false;
|
|
6807
|
+
const pool = deps.pool;
|
|
6808
|
+
if (!pool) return false;
|
|
6809
|
+
const enabledCount = pool.getAccounts().filter((a) => a.enabled).length;
|
|
6810
|
+
if (quotaHops >= Math.min(Math.max(0, enabledCount - 1), 16)) return false;
|
|
6811
|
+
const model = options.model || deps.defaultModel?.() || "gemini";
|
|
6812
|
+
const fam = modelFamilyOf(model);
|
|
6813
|
+
return pool.selectAccount(fam, model) !== null;
|
|
6814
|
+
}
|
|
6684
6815
|
/**
|
|
6685
6816
|
* Effort self-healing wrapper around streamAttempt (issue #67).
|
|
6686
6817
|
*
|
|
@@ -6692,13 +6823,25 @@ function providerRetryPolicyFor(pool) {
|
|
|
6692
6823
|
* becomes one extra spawn. The refusal text itself is the evidence that WE
|
|
6693
6824
|
* passed the flag, so the replay does not depend on who requested the effort
|
|
6694
6825
|
* (an explicit request, `defaultEffort`, or the aux channel's pinning).
|
|
6826
|
+
*
|
|
6827
|
+
* Also rotates across spare pool accounts for auxiliary routes (`isAux`) that
|
|
6828
|
+
* bypass `dsh-llm-retry`, and switches accounts when a model is unrecognized.
|
|
6695
6829
|
*/
|
|
6696
6830
|
async function* streamWithEffortRetry(deps, options) {
|
|
6697
|
-
|
|
6831
|
+
let suppressEffort = false;
|
|
6832
|
+
let effortSelfHealed = false;
|
|
6833
|
+
let modelSelfHealed = false;
|
|
6834
|
+
let quotaHops = 0;
|
|
6835
|
+
const isAux = isAuxCall(options);
|
|
6836
|
+
const poolEnabled = deps.pool ? deps.pool.getAccounts().filter((a) => a.enabled).length : 0;
|
|
6837
|
+
const maxAuxHops = Math.min(Math.max(0, poolEnabled - 1), 16);
|
|
6838
|
+
for (;;) {
|
|
6698
6839
|
let emitted = false;
|
|
6840
|
+
const bufferAux = isAux && quotaHops < maxAuxHops;
|
|
6841
|
+
const buffered = [];
|
|
6699
6842
|
const telemetry = [];
|
|
6700
6843
|
let finish = null;
|
|
6701
|
-
for await (const chunk of deps.streamAttempt(options,
|
|
6844
|
+
for await (const chunk of deps.streamAttempt(options, suppressEffort)) {
|
|
6702
6845
|
if (chunk.type === "finish") {
|
|
6703
6846
|
finish = chunk;
|
|
6704
6847
|
continue;
|
|
@@ -6707,27 +6850,51 @@ async function* streamWithEffortRetry(deps, options) {
|
|
|
6707
6850
|
telemetry.push(chunk);
|
|
6708
6851
|
continue;
|
|
6709
6852
|
}
|
|
6853
|
+
if (bufferAux) {
|
|
6854
|
+
buffered.push(chunk);
|
|
6855
|
+
continue;
|
|
6856
|
+
}
|
|
6710
6857
|
emitted = true;
|
|
6711
6858
|
yield chunk;
|
|
6712
6859
|
}
|
|
6713
6860
|
if (finish === null) {
|
|
6861
|
+
for (const chunk of buffered) yield chunk;
|
|
6714
6862
|
for (const chunk of telemetry) yield chunk;
|
|
6715
6863
|
return;
|
|
6716
6864
|
}
|
|
6717
|
-
if (
|
|
6865
|
+
if (!emitted && finish.reason.kind === "error") {
|
|
6718
6866
|
const failure = finish.reason.failure;
|
|
6719
6867
|
const rawMsg = String(failure?.message ?? "");
|
|
6720
|
-
|
|
6721
|
-
|
|
6722
|
-
if (
|
|
6723
|
-
|
|
6868
|
+
if (buffered.length === 0) {
|
|
6869
|
+
const refused = parseEffortUnsupported(rawMsg);
|
|
6870
|
+
if (!effortSelfHealed && refused !== null) {
|
|
6871
|
+
effortSelfHealed = true;
|
|
6872
|
+
suppressEffort = true;
|
|
6873
|
+
if (deps.catalogAdvertisesEffort?.(refused) !== true) deps.learnEffortUnsupported(refused);
|
|
6874
|
+
continue;
|
|
6875
|
+
}
|
|
6876
|
+
const unrecog = parseModelNotRecognized(rawMsg);
|
|
6877
|
+
if (!modelSelfHealed && unrecog !== null) {
|
|
6878
|
+
modelSelfHealed = true;
|
|
6879
|
+
if (deps.onModelUnsupported) {
|
|
6880
|
+
deps.onModelUnsupported(unrecog);
|
|
6881
|
+
continue;
|
|
6882
|
+
}
|
|
6883
|
+
if (deps.pool) {
|
|
6884
|
+
const fam = modelFamilyOf(unrecog);
|
|
6885
|
+
const currentId = deps.pool.getActiveAccountId(fam);
|
|
6886
|
+
if (currentId) deps.pool.markModelUnsupported(currentId, unrecog);
|
|
6887
|
+
deps.pool.selectAccount(fam, unrecog);
|
|
6888
|
+
continue;
|
|
6889
|
+
}
|
|
6890
|
+
}
|
|
6724
6891
|
}
|
|
6725
|
-
|
|
6726
|
-
|
|
6727
|
-
deps.onModelUnsupported(unrecog);
|
|
6892
|
+
if (canRetryAuxQuota(deps, options, emitted, finish, quotaHops, rawMsg)) {
|
|
6893
|
+
quotaHops++;
|
|
6728
6894
|
continue;
|
|
6729
6895
|
}
|
|
6730
6896
|
}
|
|
6897
|
+
for (const chunk of buffered) yield chunk;
|
|
6731
6898
|
for (const chunk of telemetry) yield chunk;
|
|
6732
6899
|
yield finish;
|
|
6733
6900
|
return;
|
|
@@ -6930,7 +7097,7 @@ async function finalizeAttempt(input) {
|
|
|
6930
7097
|
emitRunTelemetry(input);
|
|
6931
7098
|
}
|
|
6932
7099
|
function applySuccess(input) {
|
|
6933
|
-
if (input.account) input.pool?.recordSuccess(input.account.id, input.family);
|
|
7100
|
+
if (input.account && !input.cooledInLiveFamilies?.has(input.family)) input.pool?.recordSuccess(input.account.id, input.family);
|
|
6934
7101
|
input.dropFailoverBudget(input.sessionKey);
|
|
6935
7102
|
input.clearRateLimitFailover(input.sessionKey);
|
|
6936
7103
|
input.clearNetworkStreak(input.sessionKey);
|
|
@@ -6959,7 +7126,14 @@ function applyFailure(input, failure) {
|
|
|
6959
7126
|
*/
|
|
6960
7127
|
function rotateOnFailure(input, failure) {
|
|
6961
7128
|
const { account, pool, family, outcome } = input;
|
|
6962
|
-
|
|
7129
|
+
const qf = input.quotaFailover;
|
|
7130
|
+
const targetFamily = qf?.family ?? family;
|
|
7131
|
+
const resetMs = parseResetDurationMs(input.rawErrText) ?? parseResetDurationMs(failure.message);
|
|
7132
|
+
const serverResetTime = qf?.resetTime ?? (resetMs && resetMs > 0 ? new Date(Date.now() + resetMs).toISOString() : void 0);
|
|
7133
|
+
if (account && !outcome.aborted && (looksLikeHardRateLimit(input.rawErrText) || Boolean(qf))) {
|
|
7134
|
+
if (!qf?.cooledInLive) pool?.recordFailure(account.id, targetFamily, failure.message, serverResetTime);
|
|
7135
|
+
if (qf?.crossFamily) pool?.advanceAccount(account.id, family, input.activeModel);
|
|
7136
|
+
}
|
|
6963
7137
|
if (account && input.silentTimeout) pool?.advanceAccount(account.id, family);
|
|
6964
7138
|
if (account && failure.code === Err.ELIGIBILITY) {
|
|
6965
7139
|
const unrecog = parseModelNotRecognized(failure.message ?? "");
|
|
@@ -6979,7 +7153,7 @@ function rotateOnFailure(input, failure) {
|
|
|
6979
7153
|
* persisted as resumable-interrupted, then write it.
|
|
6980
7154
|
*/
|
|
6981
7155
|
function persistFailureBinding(input, failure) {
|
|
6982
|
-
const staleConversation = failure.code === Err.AUTH || failure.message !== "" && /conversation.*(not found|invalid|not recognized|expired|does not exist)|session.*(expired|invalid)/i.test(failure.message);
|
|
7156
|
+
const staleConversation = failure.code === Err.AUTH || input.quotaFailover?.crossFamily === true || failure.message !== "" && /conversation.*(not found|invalid|not recognized|expired|does not exist)|session.*(expired|invalid)/i.test(failure.message);
|
|
6983
7157
|
const interruptedTurn = failure.kind === "aborted" || input.outcome.timedOut === true;
|
|
6984
7158
|
if (input.persistBinding && input.sessionAccountKey !== "" && staleConversation) input.store.delete(input.sessionAccountKey);
|
|
6985
7159
|
else if (input.persistBinding && input.sessionAccountKey !== "" && !input.superseded) {
|
|
@@ -8310,11 +8484,16 @@ function isWeeklyQuotaBenched(quota, clearedAt, now) {
|
|
|
8310
8484
|
* `updatedAt` of the snapshot it overrode; the gate stays off only while the
|
|
8311
8485
|
* cached row is no newer than that stamp, so a fresh poll always re-arms it.
|
|
8312
8486
|
*/
|
|
8313
|
-
function isQuotaBenched(acc, family, now) {
|
|
8487
|
+
function isQuotaBenched(acc, family, now, model) {
|
|
8314
8488
|
const quota = acc.quotas?.[family] ?? (family === "argon" ? acc.quotas?.google : void 0);
|
|
8315
8489
|
if (!quota) return false;
|
|
8316
8490
|
const clearedAt = acc.quotaGateClearedAt?.[family];
|
|
8317
|
-
|
|
8491
|
+
if (is5hQuotaBenched(quota, clearedAt, now) || isWeeklyQuotaBenched(quota, clearedAt, now)) return true;
|
|
8492
|
+
if (!model || model.trim() === "" || !Array.isArray(quota.models)) return false;
|
|
8493
|
+
if (clearedAt !== void 0 && (quota.updatedAt ?? 0) <= clearedAt) return false;
|
|
8494
|
+
const matched = quota.models.filter((m) => modelMatches(model, m.modelId));
|
|
8495
|
+
if (matched.length === 0) return false;
|
|
8496
|
+
return matched.every((m) => isWindowBenched(m.remainingFraction, m.resetTime ?? quota.resetTime, LOW_QUOTA_WEEKLY, now));
|
|
8318
8497
|
}
|
|
8319
8498
|
/**
|
|
8320
8499
|
* True when the account cannot operate due to auth expiration, web verification
|
|
@@ -8401,7 +8580,7 @@ function explainIneligibility(acc, family, now, pool, model) {
|
|
|
8401
8580
|
if (model && pool?.isModelSupported && !pool.isModelSupported(acc, model)) return "model-unsupported";
|
|
8402
8581
|
const cd = acc.cooldowns?.[family];
|
|
8403
8582
|
if (cd && cd.cooldownUntil > now) return "cooldown";
|
|
8404
|
-
if (isQuotaBenched(acc, family, now)) return "quota";
|
|
8583
|
+
if (isQuotaBenched(acc, family, now, model)) return "quota";
|
|
8405
8584
|
return null;
|
|
8406
8585
|
}
|
|
8407
8586
|
/**
|
|
@@ -8831,7 +9010,7 @@ var PoolAdmin = class {
|
|
|
8831
9010
|
consecutiveFailures: failures
|
|
8832
9011
|
};
|
|
8833
9012
|
const famPinned = data.activeAccountIds?.[family];
|
|
8834
|
-
if (famPinned === id || famPinned === void 0 && data.lastActiveAccountId === id) this.advanceAccount(id, family);
|
|
9013
|
+
if (famPinned === id || famPinned === void 0 && (data.lastActiveAccountId === id || data.lastActiveAccountId === void 0)) this.advanceAccount(id, family);
|
|
8835
9014
|
this.deps.persist();
|
|
8836
9015
|
}
|
|
8837
9016
|
/**
|
|
@@ -34708,6 +34887,205 @@ function buildAgyEnv(input) {
|
|
|
34708
34887
|
return env;
|
|
34709
34888
|
}
|
|
34710
34889
|
//#endregion
|
|
34890
|
+
//#region src/features/subagent/task-activity.ts
|
|
34891
|
+
const MAX_READ_BYTES_PER_POLL = 131072;
|
|
34892
|
+
const MAX_LINES_PER_POLL = 20;
|
|
34893
|
+
const MAX_LINE_CHARS = 220;
|
|
34894
|
+
const MAX_MESSAGE_CHARS = 240;
|
|
34895
|
+
function noteWarn(log, context, err) {
|
|
34896
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
34897
|
+
log?.(`[task-activity] ${context}: ${detail}`);
|
|
34898
|
+
}
|
|
34899
|
+
function clipSingleLine(text, maxChars) {
|
|
34900
|
+
const single = text.replace(/\r/g, "").replace(/\s+/g, " ").trim();
|
|
34901
|
+
if (single.length <= maxChars) return single;
|
|
34902
|
+
return single.slice(0, Math.max(0, maxChars - 1)) + "…";
|
|
34903
|
+
}
|
|
34904
|
+
function formatTaskLogLine(taskId, line) {
|
|
34905
|
+
return `[后台任务 · ${taskId}] ${clipSingleLine(line, MAX_LINE_CHARS)}`;
|
|
34906
|
+
}
|
|
34907
|
+
function summarizeMessageJson(rawObj, fallbackName = "message") {
|
|
34908
|
+
if (!rawObj || typeof rawObj !== "object") return null;
|
|
34909
|
+
const obj = rawObj;
|
|
34910
|
+
const id = typeof obj.id === "string" && obj.id.trim() !== "" ? obj.id.trim() : "";
|
|
34911
|
+
const rawSender = typeof obj.sender === "string" ? obj.sender.trim() : "";
|
|
34912
|
+
const senderLabel = rawSender.includes("/") ? rawSender.slice(rawSender.lastIndexOf("/") + 1) : rawSender || "system";
|
|
34913
|
+
const details = obj.renderDetails && typeof obj.renderDetails === "object" ? obj.renderDetails : void 0;
|
|
34914
|
+
const title = typeof details?.messageTitle === "string" ? clipSingleLine(details.messageTitle, 120) : "";
|
|
34915
|
+
const content = typeof obj.content === "string" ? clipSingleLine(obj.content, MAX_MESSAGE_CHARS) : "";
|
|
34916
|
+
if (title === "" && content === "") return null;
|
|
34917
|
+
const body = title !== "" && content !== "" && !content.toLowerCase().includes(title.toLowerCase()) ? `${title} — ${content}` : content !== "" ? content : title;
|
|
34918
|
+
return {
|
|
34919
|
+
dedupKey: id !== "" ? id : fallbackName,
|
|
34920
|
+
text: `[后台通知 · ${senderLabel}] ${body}`
|
|
34921
|
+
};
|
|
34922
|
+
}
|
|
34923
|
+
function listFilesSafe(dir, suffix) {
|
|
34924
|
+
if (!existsSync(dir)) return [];
|
|
34925
|
+
try {
|
|
34926
|
+
return readdirSync(dir).filter((f) => f.endsWith(suffix)).sort();
|
|
34927
|
+
} catch {
|
|
34928
|
+
return [];
|
|
34929
|
+
}
|
|
34930
|
+
}
|
|
34931
|
+
var TaskActivityTracker = class {
|
|
34932
|
+
log;
|
|
34933
|
+
logs = /* @__PURE__ */ new Map();
|
|
34934
|
+
seenMessageFiles = /* @__PURE__ */ new Set();
|
|
34935
|
+
seenMessageIds = /* @__PURE__ */ new Set();
|
|
34936
|
+
baselined = false;
|
|
34937
|
+
constructor(log) {
|
|
34938
|
+
this.log = log;
|
|
34939
|
+
}
|
|
34940
|
+
reset() {
|
|
34941
|
+
this.logs.clear();
|
|
34942
|
+
this.seenMessageFiles.clear();
|
|
34943
|
+
this.seenMessageIds.clear();
|
|
34944
|
+
this.baselined = false;
|
|
34945
|
+
}
|
|
34946
|
+
/** Record pre-existing logs and messages at EOF so resumed runs only emit new activity. */
|
|
34947
|
+
baseline(brainDir) {
|
|
34948
|
+
this.baselined = true;
|
|
34949
|
+
this.scanTaskLogs(brainDir, true, () => {});
|
|
34950
|
+
this.scanMessages(brainDir, true, () => {});
|
|
34951
|
+
}
|
|
34952
|
+
poll(brainDir, emit) {
|
|
34953
|
+
try {
|
|
34954
|
+
this.scanTaskLogs(brainDir, false, emit);
|
|
34955
|
+
this.scanMessages(brainDir, false, emit);
|
|
34956
|
+
} catch (err) {
|
|
34957
|
+
noteWarn(this.log, "poll", err);
|
|
34958
|
+
}
|
|
34959
|
+
}
|
|
34960
|
+
scanTaskLogs(brainDir, baselineOnly, emit) {
|
|
34961
|
+
const tasksDir = join(brainDir, ".system_generated", "tasks");
|
|
34962
|
+
for (const file of listFilesSafe(tasksDir, ".log")) {
|
|
34963
|
+
const taskId = file.slice(0, -4);
|
|
34964
|
+
const fullPath = join(tasksDir, file);
|
|
34965
|
+
let tracked = this.logs.get(file);
|
|
34966
|
+
if (!tracked) {
|
|
34967
|
+
let initialOffset = 0;
|
|
34968
|
+
if (baselineOnly || this.baselined) {
|
|
34969
|
+
if (baselineOnly) try {
|
|
34970
|
+
const st = statSync(fullPath);
|
|
34971
|
+
if (st.isFile()) initialOffset = st.size;
|
|
34972
|
+
} catch {
|
|
34973
|
+
initialOffset = 0;
|
|
34974
|
+
}
|
|
34975
|
+
}
|
|
34976
|
+
tracked = {
|
|
34977
|
+
taskId,
|
|
34978
|
+
filePath: fullPath,
|
|
34979
|
+
offset: initialOffset,
|
|
34980
|
+
decoder: new StringDecoder("utf8"),
|
|
34981
|
+
rest: "",
|
|
34982
|
+
seq: 0
|
|
34983
|
+
};
|
|
34984
|
+
this.logs.set(file, tracked);
|
|
34985
|
+
}
|
|
34986
|
+
if (baselineOnly) continue;
|
|
34987
|
+
this.pollSingleTaskLog(tracked, emit);
|
|
34988
|
+
}
|
|
34989
|
+
}
|
|
34990
|
+
pollSingleTaskLog(tracked, emit) {
|
|
34991
|
+
let size;
|
|
34992
|
+
try {
|
|
34993
|
+
const st = statSync(tracked.filePath);
|
|
34994
|
+
if (!st.isFile()) return;
|
|
34995
|
+
size = st.size;
|
|
34996
|
+
} catch {
|
|
34997
|
+
return;
|
|
34998
|
+
}
|
|
34999
|
+
if (size < tracked.offset) {
|
|
35000
|
+
tracked.offset = 0;
|
|
35001
|
+
tracked.decoder = new StringDecoder("utf8");
|
|
35002
|
+
tracked.rest = "";
|
|
35003
|
+
}
|
|
35004
|
+
if (size === tracked.offset) return;
|
|
35005
|
+
const toRead = Math.min(size - tracked.offset, MAX_READ_BYTES_PER_POLL);
|
|
35006
|
+
const buffer = Buffer.allocUnsafe(toRead);
|
|
35007
|
+
let bytesRead = 0;
|
|
35008
|
+
try {
|
|
35009
|
+
const fd = openSync(tracked.filePath, "r");
|
|
35010
|
+
try {
|
|
35011
|
+
bytesRead = readSync(fd, buffer, 0, toRead, tracked.offset);
|
|
35012
|
+
} finally {
|
|
35013
|
+
closeSync(fd);
|
|
35014
|
+
}
|
|
35015
|
+
} catch {
|
|
35016
|
+
return;
|
|
35017
|
+
}
|
|
35018
|
+
if (bytesRead <= 0) return;
|
|
35019
|
+
tracked.offset += bytesRead;
|
|
35020
|
+
const parts = (tracked.rest + tracked.decoder.write(buffer.subarray(0, bytesRead))).split("\n");
|
|
35021
|
+
tracked.rest = size > tracked.offset ? "" : parts.pop() ?? "";
|
|
35022
|
+
if (size > tracked.offset) {
|
|
35023
|
+
tracked.offset = size;
|
|
35024
|
+
tracked.decoder = new StringDecoder("utf8");
|
|
35025
|
+
}
|
|
35026
|
+
const lines = parts.map((l) => l.trim()).filter((l) => l !== "");
|
|
35027
|
+
if (lines.length === 0) return;
|
|
35028
|
+
const skipped = Math.max(0, lines.length - MAX_LINES_PER_POLL);
|
|
35029
|
+
const visible = skipped > 0 ? lines.slice(-20) : lines;
|
|
35030
|
+
if (skipped > 0) {
|
|
35031
|
+
tracked.seq += 1;
|
|
35032
|
+
emit({
|
|
35033
|
+
stepKey: `task-log-${tracked.taskId}-${tracked.seq}`,
|
|
35034
|
+
text: `[后台任务 · ${tracked.taskId}] … 跳过 ${skipped} 行中间输出,最新日志如下\n`,
|
|
35035
|
+
raw: {
|
|
35036
|
+
kind: "task-log",
|
|
35037
|
+
taskId: tracked.taskId,
|
|
35038
|
+
skipped
|
|
35039
|
+
}
|
|
35040
|
+
});
|
|
35041
|
+
}
|
|
35042
|
+
for (const line of visible) {
|
|
35043
|
+
tracked.seq += 1;
|
|
35044
|
+
emit({
|
|
35045
|
+
stepKey: `task-log-${tracked.taskId}-${tracked.seq}`,
|
|
35046
|
+
text: `${formatTaskLogLine(tracked.taskId, line)}\n`,
|
|
35047
|
+
raw: {
|
|
35048
|
+
kind: "task-log",
|
|
35049
|
+
taskId: tracked.taskId,
|
|
35050
|
+
line
|
|
35051
|
+
}
|
|
35052
|
+
});
|
|
35053
|
+
}
|
|
35054
|
+
}
|
|
35055
|
+
scanMessages(brainDir, baselineOnly, emit) {
|
|
35056
|
+
const baseDir = join(brainDir, ".system_generated", "messages");
|
|
35057
|
+
const dirs = [{
|
|
35058
|
+
tag: "delivered",
|
|
35059
|
+
path: baseDir
|
|
35060
|
+
}, {
|
|
35061
|
+
tag: "undelivered",
|
|
35062
|
+
path: join(baseDir, "undelivered")
|
|
35063
|
+
}];
|
|
35064
|
+
for (const dir of dirs) for (const file of listFilesSafe(dir.path, ".json")) {
|
|
35065
|
+
const fileKey = `${dir.tag}:${file}`;
|
|
35066
|
+
if (this.seenMessageFiles.has(fileKey)) continue;
|
|
35067
|
+
this.seenMessageFiles.add(fileKey);
|
|
35068
|
+
const stem = file.slice(0, -5);
|
|
35069
|
+
let parsed;
|
|
35070
|
+
try {
|
|
35071
|
+
parsed = JSON.parse(readFileSync(join(dir.path, file), "utf8"));
|
|
35072
|
+
} catch {
|
|
35073
|
+
continue;
|
|
35074
|
+
}
|
|
35075
|
+
const summary = summarizeMessageJson(parsed, stem);
|
|
35076
|
+
if (!summary) continue;
|
|
35077
|
+
if (this.seenMessageIds.has(summary.dedupKey)) continue;
|
|
35078
|
+
this.seenMessageIds.add(summary.dedupKey);
|
|
35079
|
+
if (baselineOnly) continue;
|
|
35080
|
+
emit({
|
|
35081
|
+
stepKey: `task-msg-${summary.dedupKey}`,
|
|
35082
|
+
text: `${summary.text}\n`,
|
|
35083
|
+
raw: parsed
|
|
35084
|
+
});
|
|
35085
|
+
}
|
|
35086
|
+
}
|
|
35087
|
+
};
|
|
35088
|
+
//#endregion
|
|
34711
35089
|
//#region src/features/subagent/tracker.ts
|
|
34712
35090
|
/**
|
|
34713
35091
|
* Swallow the failures a live transcript produces by design — a row still being
|
|
@@ -34720,6 +35098,62 @@ function noteFailure(log, what, err) {
|
|
|
34720
35098
|
if (code === "ENOENT" || code === "EACCES" || code === "EPERM" || code === "EBUSY") return;
|
|
34721
35099
|
log?.(`subagent tracker: ${what} — ${String(err)}`);
|
|
34722
35100
|
}
|
|
35101
|
+
function createTranscriptCursor(offset = 0) {
|
|
35102
|
+
return {
|
|
35103
|
+
offset,
|
|
35104
|
+
decoder: new StringDecoder("utf8"),
|
|
35105
|
+
rest: ""
|
|
35106
|
+
};
|
|
35107
|
+
}
|
|
35108
|
+
function fileSizeOrZero(logPath) {
|
|
35109
|
+
try {
|
|
35110
|
+
const st = statSync(logPath);
|
|
35111
|
+
return st.isFile() ? st.size : 0;
|
|
35112
|
+
} catch {
|
|
35113
|
+
return 0;
|
|
35114
|
+
}
|
|
35115
|
+
}
|
|
35116
|
+
function readTranscriptDelta(logPath, cursor) {
|
|
35117
|
+
let size;
|
|
35118
|
+
try {
|
|
35119
|
+
const st = statSync(logPath);
|
|
35120
|
+
if (!st.isFile()) return [];
|
|
35121
|
+
size = st.size;
|
|
35122
|
+
} catch {
|
|
35123
|
+
return [];
|
|
35124
|
+
}
|
|
35125
|
+
if (size < cursor.offset) {
|
|
35126
|
+
cursor.offset = 0;
|
|
35127
|
+
cursor.decoder = new StringDecoder("utf8");
|
|
35128
|
+
cursor.rest = "";
|
|
35129
|
+
}
|
|
35130
|
+
if (size === cursor.offset) return [];
|
|
35131
|
+
const length = size - cursor.offset;
|
|
35132
|
+
const buffer = Buffer.allocUnsafe(length);
|
|
35133
|
+
let bytesRead = 0;
|
|
35134
|
+
try {
|
|
35135
|
+
const fd = openSync(logPath, "r");
|
|
35136
|
+
try {
|
|
35137
|
+
bytesRead = readSync(fd, buffer, 0, length, cursor.offset);
|
|
35138
|
+
} finally {
|
|
35139
|
+
closeSync(fd);
|
|
35140
|
+
}
|
|
35141
|
+
} catch {
|
|
35142
|
+
return [];
|
|
35143
|
+
}
|
|
35144
|
+
if (bytesRead <= 0) return [];
|
|
35145
|
+
cursor.offset += bytesRead;
|
|
35146
|
+
const parts = (cursor.rest + cursor.decoder.write(buffer.subarray(0, bytesRead))).split("\n");
|
|
35147
|
+
cursor.rest = parts.pop() ?? "";
|
|
35148
|
+
return parts.filter((line) => line.trim() !== "");
|
|
35149
|
+
}
|
|
35150
|
+
function extractAuthoritativeSubagentQuotaError(text, fromParentSystemMessage) {
|
|
35151
|
+
if (!text) return false;
|
|
35152
|
+
if (!looksLikeHardRateLimit(text) || isTransientBurstRateLimit(text) || looksLikeTosViolation(text)) return false;
|
|
35153
|
+
if (fromParentSystemMessage) return /The subagent\b[\s\S]*?\bencountered an error/i.test(text);
|
|
35154
|
+
if (/Command exited with code\b/i.test(text)) return false;
|
|
35155
|
+
return /image failed to generate|failed to generate content|cloudcode-pa\.googleapis\.com|RESOURCE_EXHAUSTED|QUOTA_EXHAUSTED|Individual quota reached|agent executor error/i.test(text);
|
|
35156
|
+
}
|
|
34723
35157
|
function isActiveSubagentState(state) {
|
|
34724
35158
|
return state === void 0 || state === "" || state === "SUBAGENT_STATE_ALIVE";
|
|
34725
35159
|
}
|
|
@@ -34790,15 +35224,19 @@ var SubagentTracker = class {
|
|
|
34790
35224
|
brainDir = null;
|
|
34791
35225
|
trackedSubagents = /* @__PURE__ */ new Map();
|
|
34792
35226
|
seenStepKeys = /* @__PURE__ */ new Set();
|
|
35227
|
+
taskActivity;
|
|
34793
35228
|
timer = null;
|
|
34794
35229
|
active = false;
|
|
34795
35230
|
polling = false;
|
|
34796
35231
|
pollIntervalMs;
|
|
35232
|
+
parentCursor = createTranscriptCursor();
|
|
35233
|
+
parentLines = 0;
|
|
34797
35234
|
constructor(opts) {
|
|
34798
35235
|
this.opts = opts;
|
|
34799
35236
|
this.conversationId = opts.conversationId ?? "";
|
|
34800
35237
|
this.accountHome = opts.accountHome;
|
|
34801
35238
|
this.pollIntervalMs = opts.pollIntervalMs ?? 500;
|
|
35239
|
+
this.taskActivity = new TaskActivityTracker(opts.log);
|
|
34802
35240
|
}
|
|
34803
35241
|
/** Follow the conversation agy reported in its stream-json init/result envelope. */
|
|
34804
35242
|
setConversationId(cid) {
|
|
@@ -34806,6 +35244,9 @@ var SubagentTracker = class {
|
|
|
34806
35244
|
this.conversationId = cid;
|
|
34807
35245
|
this.brainDir = null;
|
|
34808
35246
|
this.baselined = false;
|
|
35247
|
+
this.parentCursor = createTranscriptCursor();
|
|
35248
|
+
this.parentLines = 0;
|
|
35249
|
+
this.taskActivity.reset();
|
|
34809
35250
|
if (this.active) {
|
|
34810
35251
|
if (this.opts.resumed === true) {
|
|
34811
35252
|
this.baselined = true;
|
|
@@ -34830,6 +35271,9 @@ var SubagentTracker = class {
|
|
|
34830
35271
|
try {
|
|
34831
35272
|
if (!this.brainDir) this.brainDir = findBrainDir(this.conversationId, this.accountHome);
|
|
34832
35273
|
if (!this.brainDir) return;
|
|
35274
|
+
const parentLog = join(this.brainDir, ".system_generated", "logs", "transcript.jsonl");
|
|
35275
|
+
this.parentCursor = createTranscriptCursor(fileSizeOrZero(parentLog));
|
|
35276
|
+
this.taskActivity.baseline(this.brainDir);
|
|
34833
35277
|
this.scanSubagentsForConv(this.conversationId, true);
|
|
34834
35278
|
for (const cid of Array.from(this.trackedSubagents.keys())) this.scanSubagentsForConv(cid, true);
|
|
34835
35279
|
} catch (err) {
|
|
@@ -34837,20 +35281,32 @@ var SubagentTracker = class {
|
|
|
34837
35281
|
}
|
|
34838
35282
|
}
|
|
34839
35283
|
stop() {
|
|
34840
|
-
|
|
34841
|
-
|
|
34842
|
-
|
|
34843
|
-
this.
|
|
35284
|
+
try {
|
|
35285
|
+
if (this.active && !this.polling && this.conversationId !== "") this.poll();
|
|
35286
|
+
} finally {
|
|
35287
|
+
this.active = false;
|
|
35288
|
+
if (this.timer) {
|
|
35289
|
+
clearInterval(this.timer);
|
|
35290
|
+
this.timer = null;
|
|
35291
|
+
}
|
|
34844
35292
|
}
|
|
34845
35293
|
}
|
|
34846
|
-
/**
|
|
34847
|
-
|
|
34848
|
-
|
|
35294
|
+
/** Perform one final synchronous scan even after `stop()` halted the interval. */
|
|
35295
|
+
flush() {
|
|
35296
|
+
this.poll(true);
|
|
35297
|
+
}
|
|
35298
|
+
/** Run one scan across parent transcript, known conversations, and subagents. */
|
|
35299
|
+
poll(force = false) {
|
|
35300
|
+
if (this.polling || !this.active && !force) return;
|
|
34849
35301
|
if (!this.conversationId) return;
|
|
34850
35302
|
this.polling = true;
|
|
34851
35303
|
try {
|
|
34852
35304
|
if (!this.brainDir) this.brainDir = findBrainDir(this.conversationId, this.accountHome);
|
|
34853
35305
|
if (!this.brainDir) return;
|
|
35306
|
+
this.pollParentTranscript();
|
|
35307
|
+
this.taskActivity.poll(this.brainDir, (ev) => {
|
|
35308
|
+
this.emitSubagentEvent(ev.stepKey, ev.text, ev.raw);
|
|
35309
|
+
});
|
|
34854
35310
|
const convsToScan = [this.conversationId, ...Array.from(this.trackedSubagents.keys())];
|
|
34855
35311
|
for (const cid of convsToScan) this.scanSubagentsForConv(cid);
|
|
34856
35312
|
for (const sub of this.trackedSubagents.values()) if (!sub.terminal) this.pollSubagentTranscript(sub);
|
|
@@ -34860,6 +35316,30 @@ var SubagentTracker = class {
|
|
|
34860
35316
|
this.polling = false;
|
|
34861
35317
|
}
|
|
34862
35318
|
}
|
|
35319
|
+
pollParentTranscript() {
|
|
35320
|
+
if (!this.brainDir) return;
|
|
35321
|
+
const logPath = join(this.brainDir, ".system_generated", "logs", "transcript.jsonl");
|
|
35322
|
+
for (const line of readTranscriptDelta(logPath, this.parentCursor)) try {
|
|
35323
|
+
const o = JSON.parse(line);
|
|
35324
|
+
this.parentLines += 1;
|
|
35325
|
+
if (o.source !== "SYSTEM" || o.type !== "SYSTEM_MESSAGE" || typeof o.content !== "string") continue;
|
|
35326
|
+
if (!extractAuthoritativeSubagentQuotaError(o.content, true)) continue;
|
|
35327
|
+
const qKey = `parent-quota-${typeof o.step_index === "number" ? o.step_index : this.parentLines}`;
|
|
35328
|
+
if (this.seenStepKeys.has(qKey)) continue;
|
|
35329
|
+
const role = o.content.match(/The subagent\s+(.+?)\s+(?:\([^)]+\)\s+)?(?:with id|encountered an error)/i)?.[1]?.trim();
|
|
35330
|
+
const model = extractQuotaErrorModel(o.content);
|
|
35331
|
+
const resetMs = parseResetDurationMs(o.content);
|
|
35332
|
+
const prefix = role ? `[子代理 · ${role}]` : "[子代理]";
|
|
35333
|
+
this.emitSubagentEvent(qKey, `${prefix} 额度已用尽,正在准备切换账号...\n`, o);
|
|
35334
|
+
this.opts.onQuotaExhausted?.({
|
|
35335
|
+
source: "parent-system-message",
|
|
35336
|
+
errorText: o.content,
|
|
35337
|
+
...role ? { role } : {},
|
|
35338
|
+
...model ? { model } : {},
|
|
35339
|
+
...resetMs !== void 0 ? { resetMs } : {}
|
|
35340
|
+
});
|
|
35341
|
+
} catch {}
|
|
35342
|
+
}
|
|
34863
35343
|
emitSubagentEvent(stepKey, text, raw) {
|
|
34864
35344
|
if (this.seenStepKeys.has(stepKey)) return;
|
|
34865
35345
|
this.seenStepKeys.add(stepKey);
|
|
@@ -34909,14 +35389,8 @@ var SubagentTracker = class {
|
|
|
34909
35389
|
const typeName = typeof desc?.typeName === "string" && desc.typeName !== "" ? desc.typeName : "Subagent";
|
|
34910
35390
|
const subBrainDir = findBrainDir(subId, this.accountHome) ?? join(bDir, "..", subId);
|
|
34911
35391
|
const active = isActiveSubagentState(rawState);
|
|
34912
|
-
|
|
34913
|
-
|
|
34914
|
-
const logPath = join(subBrainDir, ".system_generated", "logs", "transcript.jsonl");
|
|
34915
|
-
const st = statSync(logPath);
|
|
34916
|
-
if (st.isFile()) initialOffset = st.size;
|
|
34917
|
-
} catch {
|
|
34918
|
-
initialOffset = 0;
|
|
34919
|
-
}
|
|
35392
|
+
const logPath = join(subBrainDir, ".system_generated", "logs", "transcript.jsonl");
|
|
35393
|
+
const initialOffset = baselineOnly ? fileSizeOrZero(logPath) : 0;
|
|
34920
35394
|
const tracked = {
|
|
34921
35395
|
conversationId: subId,
|
|
34922
35396
|
role,
|
|
@@ -34924,9 +35398,7 @@ var SubagentTracker = class {
|
|
|
34924
35398
|
brainDir: subBrainDir,
|
|
34925
35399
|
state: rawState,
|
|
34926
35400
|
terminal: !active,
|
|
34927
|
-
|
|
34928
|
-
decoder: new StringDecoder("utf8"),
|
|
34929
|
-
rest: "",
|
|
35401
|
+
cursor: createTranscriptCursor(initialOffset),
|
|
34930
35402
|
lines: 0
|
|
34931
35403
|
};
|
|
34932
35404
|
this.trackedSubagents.set(subId, tracked);
|
|
@@ -34935,6 +35407,7 @@ var SubagentTracker = class {
|
|
|
34935
35407
|
else if (active) this.emitSubagentEvent(spawnKey, `[子代理 · ${role}] 已启动协同专员 (${typeName})\n`, obj);
|
|
34936
35408
|
else {
|
|
34937
35409
|
this.seenStepKeys.add(spawnKey);
|
|
35410
|
+
this.pollSubagentTranscript(tracked, true);
|
|
34938
35411
|
if (rawState !== void 0) this.emitSubagentEvent(`subagent-state-${subId}-${rawState}`, `[子代理 · ${role}] 协同专员未保持运行 (${rawState})\n`, obj);
|
|
34939
35412
|
}
|
|
34940
35413
|
} catch (err) {
|
|
@@ -34942,70 +35415,161 @@ var SubagentTracker = class {
|
|
|
34942
35415
|
}
|
|
34943
35416
|
}
|
|
34944
35417
|
}
|
|
34945
|
-
|
|
34946
|
-
* Read the transcript lines appended since the last poll.
|
|
34947
|
-
*
|
|
34948
|
-
* agy rewrites nothing and only appends, so the tracker keeps a byte offset
|
|
34949
|
-
* instead of re-reading (and re-splitting) a transcript that grows to hundreds
|
|
34950
|
-
* of kilobytes: each poll costs one bounded read of the new bytes.
|
|
34951
|
-
*/
|
|
34952
|
-
readNewLines(sub) {
|
|
35418
|
+
pollSubagentTranscript(sub, quotaOnly = false) {
|
|
34953
35419
|
const logPath = join(sub.brainDir, ".system_generated", "logs", "transcript.jsonl");
|
|
34954
|
-
|
|
34955
|
-
try {
|
|
34956
|
-
const st = statSync(logPath);
|
|
34957
|
-
if (!st.isFile()) return [];
|
|
34958
|
-
size = st.size;
|
|
34959
|
-
} catch {
|
|
34960
|
-
return [];
|
|
34961
|
-
}
|
|
34962
|
-
if (size < sub.offset) {
|
|
34963
|
-
sub.offset = 0;
|
|
34964
|
-
sub.decoder = new StringDecoder("utf8");
|
|
34965
|
-
sub.rest = "";
|
|
34966
|
-
}
|
|
34967
|
-
if (size === sub.offset) return [];
|
|
34968
|
-
const length = size - sub.offset;
|
|
34969
|
-
const buffer = Buffer.allocUnsafe(length);
|
|
34970
|
-
let bytesRead = 0;
|
|
34971
|
-
try {
|
|
34972
|
-
const fd = openSync(logPath, "r");
|
|
34973
|
-
try {
|
|
34974
|
-
bytesRead = readSync(fd, buffer, 0, length, sub.offset);
|
|
34975
|
-
} finally {
|
|
34976
|
-
closeSync(fd);
|
|
34977
|
-
}
|
|
34978
|
-
} catch {
|
|
34979
|
-
return [];
|
|
34980
|
-
}
|
|
34981
|
-
if (bytesRead <= 0) return [];
|
|
34982
|
-
sub.offset += bytesRead;
|
|
34983
|
-
const parts = (sub.rest + sub.decoder.write(buffer.subarray(0, bytesRead))).split("\n");
|
|
34984
|
-
sub.rest = parts.pop() ?? "";
|
|
34985
|
-
return parts.filter((line) => line.trim() !== "");
|
|
34986
|
-
}
|
|
34987
|
-
pollSubagentTranscript(sub) {
|
|
34988
|
-
for (const line of this.readNewLines(sub)) try {
|
|
35420
|
+
for (const line of readTranscriptDelta(logPath, sub.cursor)) try {
|
|
34989
35421
|
const o = JSON.parse(line);
|
|
34990
35422
|
sub.lines += 1;
|
|
34991
35423
|
const stepIdx = typeof o.step_index === "number" ? o.step_index : sub.lines;
|
|
34992
35424
|
const baseKey = `subagent-${sub.conversationId}-${stepIdx}`;
|
|
34993
|
-
if (
|
|
34994
|
-
|
|
34995
|
-
|
|
34996
|
-
|
|
34997
|
-
|
|
34998
|
-
|
|
34999
|
-
|
|
35000
|
-
|
|
35001
|
-
|
|
35002
|
-
|
|
35003
|
-
|
|
35004
|
-
|
|
35425
|
+
if (!quotaOnly) {
|
|
35426
|
+
if (Array.isArray(o.tool_calls) && o.tool_calls.length > 0) for (let tcIdx = 0; tcIdx < o.tool_calls.length; tcIdx++) {
|
|
35427
|
+
const tc = o.tool_calls[tcIdx];
|
|
35428
|
+
const toolName = typeof tc?.name === "string" ? tc.name : "";
|
|
35429
|
+
if (toolName === "") continue;
|
|
35430
|
+
const msg = formatSubagentAction(sub.role, toolName, tc.args);
|
|
35431
|
+
this.emitSubagentEvent(`${baseKey}-tc-${tcIdx}`, `${msg}\n`, o);
|
|
35432
|
+
}
|
|
35433
|
+
else if (o.type === "PLANNER_RESPONSE" && typeof o.content === "string" && o.content.trim() !== "") {
|
|
35434
|
+
const firstLine = o.content.trim().split("\n")[0].replace(/^#+\s*/, "").slice(0, 100);
|
|
35435
|
+
if (firstLine !== "") this.emitSubagentEvent(`${baseKey}-content`, `[子代理 · ${sub.role}] 阶段进展: ${firstLine}\n`, o);
|
|
35436
|
+
}
|
|
35437
|
+
}
|
|
35438
|
+
if (isTerminalTranscriptStatus(o.status)) {
|
|
35439
|
+
if (!quotaOnly) this.emitSubagentEvent(`${baseKey}-status`, `[子代理 · ${sub.role}] 执行异常/中止 (${o.status})\n`, o);
|
|
35440
|
+
if (/^(ERROR|FAILED)$/i.test(o.status.trim()) && typeof o.content === "string" && extractAuthoritativeSubagentQuotaError(o.content, false)) {
|
|
35441
|
+
const qKey = `subagent-quota-${sub.conversationId}-${stepIdx}`;
|
|
35442
|
+
if (!this.seenStepKeys.has(qKey)) {
|
|
35443
|
+
const model = extractQuotaErrorModel(o.content);
|
|
35444
|
+
const resetMs = parseResetDurationMs(o.content);
|
|
35445
|
+
this.emitSubagentEvent(qKey, `[子代理 · ${sub.role}] 额度已用尽,正在准备切换账号...\n`, o);
|
|
35446
|
+
this.opts.onQuotaExhausted?.({
|
|
35447
|
+
source: "subagent-transcript",
|
|
35448
|
+
errorText: o.content,
|
|
35449
|
+
role: sub.role,
|
|
35450
|
+
conversationId: sub.conversationId,
|
|
35451
|
+
...model ? { model } : {},
|
|
35452
|
+
...resetMs !== void 0 ? { resetMs } : {}
|
|
35453
|
+
});
|
|
35454
|
+
}
|
|
35455
|
+
}
|
|
35456
|
+
}
|
|
35005
35457
|
} catch {}
|
|
35006
35458
|
}
|
|
35007
35459
|
};
|
|
35008
35460
|
//#endregion
|
|
35461
|
+
//#region src/provider/quota-failover.ts
|
|
35462
|
+
const DIRECT_QUOTA_TOOLS = /* @__PURE__ */ new Set([
|
|
35463
|
+
"generate_image",
|
|
35464
|
+
"invoke_subagent",
|
|
35465
|
+
"swarm",
|
|
35466
|
+
"run_workflow"
|
|
35467
|
+
]);
|
|
35468
|
+
/**
|
|
35469
|
+
* Inspect one completed/failed tool step for an authoritative hard quota error
|
|
35470
|
+
* emitted directly by agy's image-generation or subagent delegation tool.
|
|
35471
|
+
*/
|
|
35472
|
+
function isDirectToolQuotaExhausted(ev) {
|
|
35473
|
+
if (ev.kind !== "step" || ev.stepKind !== "tool" || !ev.tool) return null;
|
|
35474
|
+
if (ev.state !== "ERROR" && !ev.tool.error) return null;
|
|
35475
|
+
const unwrapped = unwrapMcpTool(ev.tool.name, ev.tool.args);
|
|
35476
|
+
if (!DIRECT_QUOTA_TOOLS.has(unwrapped.tool)) return null;
|
|
35477
|
+
const err = [ev.tool.error, typeof ev.tool.output === "string" ? ev.tool.output : ""].filter(Boolean).join(" ").trim();
|
|
35478
|
+
if (err === "") return null;
|
|
35479
|
+
if (!looksLikeHardRateLimit(err) || isTransientBurstRateLimit(err) || looksLikeTosViolation(err)) return null;
|
|
35480
|
+
return {
|
|
35481
|
+
toolName: unwrapped.tool,
|
|
35482
|
+
errorText: err
|
|
35483
|
+
};
|
|
35484
|
+
}
|
|
35485
|
+
/**
|
|
35486
|
+
* Find an alternative healthy account that can serve BOTH the current turn's
|
|
35487
|
+
* `(inputFamily, activeModel)` and (if different) the exhausted `exhaustedFamily`.
|
|
35488
|
+
*/
|
|
35489
|
+
function findAlternativeAccount(pool, currentAccountId, inputFamily, activeModel, exhaustedFamily) {
|
|
35490
|
+
const now = Date.now();
|
|
35491
|
+
return pool.getAccounts().find((alt) => {
|
|
35492
|
+
if (alt.id === currentAccountId) return false;
|
|
35493
|
+
if (!(typeof pool.isAccountEligibleForModel === "function" ? pool.isAccountEligibleForModel(alt, inputFamily, activeModel, now) : pool.isAccountEligible(alt, inputFamily, now))) return false;
|
|
35494
|
+
if (exhaustedFamily !== inputFamily && !pool.isAccountEligible(alt, exhaustedFamily, now)) return false;
|
|
35495
|
+
return true;
|
|
35496
|
+
});
|
|
35497
|
+
}
|
|
35498
|
+
function createLiveQuotaController(ctx) {
|
|
35499
|
+
const cooled = /* @__PURE__ */ new Set();
|
|
35500
|
+
let state = null;
|
|
35501
|
+
let triggeredKill = false;
|
|
35502
|
+
const handleQuotaExhausted = (info) => {
|
|
35503
|
+
const raw = info.errorText.trim();
|
|
35504
|
+
if (raw === "" || isTransientBurstRateLimit(raw) || looksLikeTosViolation(raw)) return;
|
|
35505
|
+
const detectedModel = info.model ?? extractQuotaErrorModel(raw);
|
|
35506
|
+
const modelFam = detectedModel ? modelFamilyOf(detectedModel) : void 0;
|
|
35507
|
+
const targetFamily = modelFam && modelFam !== "unknown" ? modelFam : ctx.inputFamily;
|
|
35508
|
+
const resetMs = info.resetMs ?? parseResetDurationMs(raw);
|
|
35509
|
+
const resetTime = resetMs && resetMs > 0 ? new Date(Date.now() + resetMs).toISOString() : void 0;
|
|
35510
|
+
let cooledNow = false;
|
|
35511
|
+
if (ctx.account && ctx.pool && !cooled.has(targetFamily)) {
|
|
35512
|
+
cooled.add(targetFamily);
|
|
35513
|
+
ctx.pool.recordFailure(ctx.account.id, targetFamily, raw, resetTime);
|
|
35514
|
+
cooledNow = true;
|
|
35515
|
+
}
|
|
35516
|
+
const alt = ctx.account && ctx.pool ? findAlternativeAccount(ctx.pool, ctx.account.id, ctx.inputFamily, ctx.activeModel, targetFamily) : void 0;
|
|
35517
|
+
const subject = info.role ? `子代理 (${info.role})` : "子代理/工具";
|
|
35518
|
+
if (alt && !triggeredKill) {
|
|
35519
|
+
triggeredKill = true;
|
|
35520
|
+
state = {
|
|
35521
|
+
family: targetFamily,
|
|
35522
|
+
crossFamily: targetFamily !== ctx.inputFamily,
|
|
35523
|
+
errorText: raw.slice(0, 300),
|
|
35524
|
+
rawErrorText: raw,
|
|
35525
|
+
...resetTime === void 0 ? {} : { resetTime },
|
|
35526
|
+
cooledInLive: cooledNow || cooled.has(targetFamily)
|
|
35527
|
+
};
|
|
35528
|
+
ctx.emitNotice(`live-quota-switch-${ctx.account?.id ?? "none"}-${targetFamily}`, `[agy] ${subject}额度耗尽,已自动切换至可用账号 ${alt.alias || alt.id}...\n`);
|
|
35529
|
+
ctx.onTriggerKill();
|
|
35530
|
+
} else if (!alt) ctx.emitNotice(`live-quota-none-${ctx.account?.id ?? "none"}-${targetFamily}`, `[agy] ${subject}额度耗尽,当前无其他可用账号可切换\n`);
|
|
35531
|
+
};
|
|
35532
|
+
return {
|
|
35533
|
+
handleQuotaExhausted,
|
|
35534
|
+
failoverState: () => state,
|
|
35535
|
+
cooledFamilies: () => cooled
|
|
35536
|
+
};
|
|
35537
|
+
}
|
|
35538
|
+
function reportDirectToolQuota(quotaCtrl, ev) {
|
|
35539
|
+
if (!quotaCtrl) return;
|
|
35540
|
+
const direct = isDirectToolQuotaExhausted(ev);
|
|
35541
|
+
if (!direct) return;
|
|
35542
|
+
const m = extractQuotaErrorModel(direct.errorText);
|
|
35543
|
+
const r = parseResetDurationMs(direct.errorText);
|
|
35544
|
+
quotaCtrl.handleQuotaExhausted({
|
|
35545
|
+
source: "tool-step",
|
|
35546
|
+
errorText: direct.errorText,
|
|
35547
|
+
...m ? { model: m } : {},
|
|
35548
|
+
...r !== void 0 ? { resetMs: r } : {}
|
|
35549
|
+
});
|
|
35550
|
+
}
|
|
35551
|
+
function poolQuotaHopsFor(pool) {
|
|
35552
|
+
if (!pool) return 0;
|
|
35553
|
+
return Math.min(Math.max(0, pool.getAccounts().filter((a) => a.enabled).length - 1), 16);
|
|
35554
|
+
}
|
|
35555
|
+
function createPoolQuotaHooks(pool) {
|
|
35556
|
+
return {
|
|
35557
|
+
poolQuotaHops: () => poolQuotaHopsFor(pool),
|
|
35558
|
+
onQuotaExhausted: (accountId, model, errorText) => {
|
|
35559
|
+
const errFam = modelFamilyOf(extractQuotaErrorModel(errorText) || model);
|
|
35560
|
+
const mainFam = modelFamilyOf(model);
|
|
35561
|
+
const targetFam = errFam !== "unknown" ? errFam : mainFam;
|
|
35562
|
+
const resetMs = parseResetDurationMs(errorText);
|
|
35563
|
+
const resetTime = resetMs && resetMs > 0 ? new Date(Date.now() + resetMs).toISOString() : void 0;
|
|
35564
|
+
if (accountId) {
|
|
35565
|
+
pool.recordFailure(accountId, targetFam, errorText, resetTime);
|
|
35566
|
+
if (targetFam !== mainFam) pool.advanceAccount(accountId, mainFam, model);
|
|
35567
|
+
}
|
|
35568
|
+
return pool.selectAccount(mainFam, model) !== null;
|
|
35569
|
+
}
|
|
35570
|
+
};
|
|
35571
|
+
}
|
|
35572
|
+
//#endregion
|
|
35009
35573
|
//#region src/provider/attempt.ts
|
|
35010
35574
|
function brief$1(s) {
|
|
35011
35575
|
const flat = s.trim().replace(/\s+/g, " ");
|
|
@@ -35096,7 +35660,7 @@ function buildSpawnArgs(input, printTimeoutMinutes, log) {
|
|
|
35096
35660
|
};
|
|
35097
35661
|
}
|
|
35098
35662
|
/** Create the recording, parser, HOME snapshot and tool watchdog for one run. */
|
|
35099
|
-
function createAttemptRecorder(deps, cfg, account, initialConversationId, trackSubagents = true) {
|
|
35663
|
+
function createAttemptRecorder(deps, cfg, account, initialConversationId, trackSubagents = true, quotaCtx) {
|
|
35100
35664
|
const rec = deps.runs.create();
|
|
35101
35665
|
rec.accountHome = account && account.dir ? account.dir : void 0;
|
|
35102
35666
|
const conversationsDir = conversationsDirFor(rec.accountHome);
|
|
@@ -35104,17 +35668,33 @@ function createAttemptRecorder(deps, cfg, account, initialConversationId, trackS
|
|
|
35104
35668
|
const parser = new StreamJsonParser();
|
|
35105
35669
|
deps.onParser?.(parser);
|
|
35106
35670
|
const watchdog = createToolWatchdog(deps.log, cfg);
|
|
35671
|
+
let attachedProc = null;
|
|
35672
|
+
let pendingQuotaKill = false;
|
|
35673
|
+
const quotaCtrl = quotaCtx ? createLiveQuotaController({
|
|
35674
|
+
pool: deps.pool,
|
|
35675
|
+
account,
|
|
35676
|
+
inputFamily: quotaCtx.family,
|
|
35677
|
+
activeModel: quotaCtx.activeModel,
|
|
35678
|
+
emitNotice: (stepKey, text) => rec.append({
|
|
35679
|
+
kind: "step",
|
|
35680
|
+
stepKey,
|
|
35681
|
+
stepKind: "subagent",
|
|
35682
|
+
text,
|
|
35683
|
+
raw: {}
|
|
35684
|
+
}),
|
|
35685
|
+
onTriggerKill: () => {
|
|
35686
|
+
if (attachedProc) attachedProc.kill("quota");
|
|
35687
|
+
else pendingQuotaKill = true;
|
|
35688
|
+
}
|
|
35689
|
+
}) : null;
|
|
35107
35690
|
const tracker = trackSubagents ? new SubagentTracker({
|
|
35108
35691
|
conversationId: initialConversationId,
|
|
35109
35692
|
resumed: Boolean(initialConversationId),
|
|
35110
35693
|
accountHome: rec.accountHome,
|
|
35111
35694
|
log: deps.log,
|
|
35112
|
-
onEvent: (ev) =>
|
|
35113
|
-
|
|
35114
|
-
|
|
35115
|
-
noteActivity: () => {
|
|
35116
|
-
watchdog.markEvent();
|
|
35117
|
-
}
|
|
35695
|
+
onEvent: (ev) => rec.append(ev),
|
|
35696
|
+
...quotaCtrl ? { onQuotaExhausted: (info) => quotaCtrl.handleQuotaExhausted(info) } : {},
|
|
35697
|
+
noteActivity: () => watchdog.markEvent()
|
|
35118
35698
|
}) : void 0;
|
|
35119
35699
|
const state = {
|
|
35120
35700
|
rec,
|
|
@@ -35124,10 +35704,15 @@ function createAttemptRecorder(deps, cfg, account, initialConversationId, trackS
|
|
|
35124
35704
|
streamCid: initialConversationId ?? null,
|
|
35125
35705
|
printTimeoutMinutes: watchdog.printTimeoutMinutes,
|
|
35126
35706
|
attachProcess: (proc) => {
|
|
35707
|
+
attachedProc = proc;
|
|
35127
35708
|
watchdog.attach(proc);
|
|
35128
35709
|
tracker?.start();
|
|
35710
|
+
if (pendingQuotaKill) proc.kill("quota");
|
|
35129
35711
|
proc.outcome.finally(() => tracker?.stop());
|
|
35130
35712
|
},
|
|
35713
|
+
startTracker: () => {
|
|
35714
|
+
tracker?.start();
|
|
35715
|
+
},
|
|
35131
35716
|
feedLine: (line, onEvent, onFirstEvent) => {
|
|
35132
35717
|
watchdog.markEvent();
|
|
35133
35718
|
onFirstEvent?.();
|
|
@@ -35142,6 +35727,7 @@ function createAttemptRecorder(deps, cfg, account, initialConversationId, trackS
|
|
|
35142
35727
|
}
|
|
35143
35728
|
watchdog.trackStep(ev);
|
|
35144
35729
|
rec.append(ev);
|
|
35730
|
+
reportDirectToolQuota(quotaCtrl, ev);
|
|
35145
35731
|
if (typeof onEvent === "function") try {
|
|
35146
35732
|
onEvent(ev);
|
|
35147
35733
|
} catch {}
|
|
@@ -35159,10 +35745,15 @@ function createAttemptRecorder(deps, cfg, account, initialConversationId, trackS
|
|
|
35159
35745
|
} catch {}
|
|
35160
35746
|
}
|
|
35161
35747
|
},
|
|
35748
|
+
flushTracker: () => {
|
|
35749
|
+
tracker?.flush();
|
|
35750
|
+
},
|
|
35162
35751
|
stop: () => {
|
|
35163
35752
|
watchdog.stop();
|
|
35164
35753
|
tracker?.stop();
|
|
35165
35754
|
},
|
|
35755
|
+
quotaFailover: () => quotaCtrl?.failoverState() ?? null,
|
|
35756
|
+
cooledInLiveFamilies: () => quotaCtrl?.cooledFamilies() ?? /* @__PURE__ */ new Set(),
|
|
35166
35757
|
conversationId: () => state.streamCid ?? diffConversations(before, conversationsDir).conversationId
|
|
35167
35758
|
};
|
|
35168
35759
|
return state;
|
|
@@ -35732,19 +36323,24 @@ async function* dispatchCredentialSlot(deps, input) {
|
|
|
35732
36323
|
}
|
|
35733
36324
|
/** Evidence + failure for one settled run (pure classification inputs). */
|
|
35734
36325
|
function classifyRun(input) {
|
|
35735
|
-
const { cfg, outcome, parser, rec, runLogHome, runLogsBefore } = input;
|
|
36326
|
+
const { cfg, outcome, parser, rec, runLogHome, runLogsBefore, quotaFailover } = input;
|
|
35736
36327
|
const r = rec.getResultEvent();
|
|
35737
36328
|
const u = rec.lastUsage ?? rec.finalUsage({});
|
|
35738
36329
|
const zeroUsageTos = r !== null && (u?.input_tokens ?? 0) === 0 && (u?.output_tokens ?? 0) === 0 && looksLikeTosViolation(r.response);
|
|
35739
36330
|
const errorResponseTos = r !== null && !r.ok && looksLikeTosViolation(r.response);
|
|
35740
|
-
const
|
|
36331
|
+
const errorResponseHardQuota = r !== null && isHardQuotaResultError(r.ok, r.error, r.response);
|
|
36332
|
+
const liveQuota = quotaFailover !== null;
|
|
36333
|
+
const consumable = !liveQuota && r !== null && (r.ok || r.response !== "") && !zeroUsageTos && !errorResponseTos && !errorResponseHardQuota;
|
|
36334
|
+
const runLogText = readAgyRunLogText(runLogHome, runLogsBefore);
|
|
36335
|
+
const logQuotaHint = !consumable && !liveQuota ? runLogText.split(/\r?\n/).filter((l) => /(?:errorreport\.go|session\.go).*?(?:RESOURCE_EXHAUSTED|Individual quota reached|QUOTA_EXHAUSTED)/i.test(l)).slice(-2).join(" ") : "";
|
|
35741
36336
|
const rawErrText = [
|
|
35742
36337
|
outcome.stderrTail,
|
|
35743
36338
|
parser.stats.lastResultError,
|
|
35744
|
-
!consumable && r && (!r.ok || zeroUsageTos) ? r.response : ""
|
|
36339
|
+
!consumable && r && (!r.ok || zeroUsageTos) ? r.error ?? r.response : "",
|
|
36340
|
+
liveQuota ? quotaFailover.rawErrorText : "",
|
|
36341
|
+
logQuotaHint
|
|
35745
36342
|
].filter(Boolean).join(" ");
|
|
35746
|
-
const isRateLimit = looksLikeRateLimit(rawErrText);
|
|
35747
|
-
const runLogText = readAgyRunLogText(runLogHome, runLogsBefore);
|
|
36343
|
+
const isRateLimit = liveQuota || looksLikeRateLimit(rawErrText);
|
|
35748
36344
|
const keyringJitter = looksLikeAuthVerificationJitter(runLogText);
|
|
35749
36345
|
const transportFailure = isTransportFailure({
|
|
35750
36346
|
rawErrText,
|
|
@@ -35753,20 +36349,27 @@ function classifyRun(input) {
|
|
|
35753
36349
|
stdoutHead: outcome.stdout,
|
|
35754
36350
|
runLogText
|
|
35755
36351
|
});
|
|
35756
|
-
const silentTimeout = outcome.timedOut === true && outcome.stdout.trim() === "";
|
|
36352
|
+
const silentTimeout = !liveQuota && outcome.timedOut === true && outcome.stdout.trim() === "";
|
|
36353
|
+
const failure = classifyFailure({
|
|
36354
|
+
cfg,
|
|
36355
|
+
outcome: liveQuota ? {
|
|
36356
|
+
...outcome,
|
|
36357
|
+
capped: false,
|
|
36358
|
+
timedOut: false,
|
|
36359
|
+
idleGraceTerminated: false
|
|
36360
|
+
} : outcome,
|
|
36361
|
+
parser,
|
|
36362
|
+
sawAuthFailure: liveQuota ? false : sawAuthFailure(parser, outcome),
|
|
36363
|
+
consumable,
|
|
36364
|
+
transportFailure: liveQuota ? false : transportFailure,
|
|
36365
|
+
keyringJitter: liveQuota ? false : keyringJitter,
|
|
36366
|
+
rawErrText,
|
|
36367
|
+
isRateLimit,
|
|
36368
|
+
silentTimeout
|
|
36369
|
+
});
|
|
36370
|
+
if (failure && liveQuota && failure.code === Err.RATE_LIMIT && parser.stats.lastResultError === void 0 && outcome.stderrTail === "") failure.message = "Google Antigravity quota / rate limit reached (subagent/tool): " + quotaFailover.errorText;
|
|
35757
36371
|
return {
|
|
35758
|
-
failure
|
|
35759
|
-
cfg,
|
|
35760
|
-
outcome,
|
|
35761
|
-
parser,
|
|
35762
|
-
sawAuthFailure: sawAuthFailure(parser, outcome),
|
|
35763
|
-
consumable,
|
|
35764
|
-
transportFailure,
|
|
35765
|
-
keyringJitter,
|
|
35766
|
-
rawErrText,
|
|
35767
|
-
isRateLimit,
|
|
35768
|
-
silentTimeout
|
|
35769
|
-
}),
|
|
36372
|
+
failure,
|
|
35770
36373
|
rawErrText,
|
|
35771
36374
|
runLogText,
|
|
35772
36375
|
silentTimeout
|
|
@@ -35789,12 +36392,15 @@ async function handleAttemptOutcome(deps, input, ctx) {
|
|
|
35789
36392
|
const { cfg, sessionKey } = input;
|
|
35790
36393
|
const outcome = await ctx.proc.outcome;
|
|
35791
36394
|
slot.releaseOnce();
|
|
36395
|
+
recorder.flushTracker();
|
|
35792
36396
|
recorder.stop();
|
|
35793
36397
|
deps.unregisterActiveRun(sessionKey, rec);
|
|
35794
36398
|
deps.clearInFlightPrompt(sessionKey, spawn.prompt);
|
|
35795
36399
|
recorder.drain(input.agyOnEvent);
|
|
36400
|
+
const quotaFailover = recorder.quotaFailover();
|
|
36401
|
+
const cooledInLiveFamilies = recorder.cooledInLiveFamilies();
|
|
35796
36402
|
const conversationId = recorder.conversationId();
|
|
35797
|
-
if (!rec.hasConsumableResult && outcome.idleGraceTerminated) {
|
|
36403
|
+
if (!rec.hasConsumableResult && outcome.idleGraceTerminated && quotaFailover === null) {
|
|
35798
36404
|
const recovered = conversationId === null ? "" : recoverFinalReply(conversationId, rec.accountHome);
|
|
35799
36405
|
const synthResult = {
|
|
35800
36406
|
kind: "result",
|
|
@@ -35813,41 +36419,47 @@ async function handleAttemptOutcome(deps, input, ctx) {
|
|
|
35813
36419
|
parser,
|
|
35814
36420
|
rec,
|
|
35815
36421
|
runLogHome,
|
|
35816
|
-
runLogsBefore
|
|
36422
|
+
runLogsBefore,
|
|
36423
|
+
quotaFailover
|
|
35817
36424
|
});
|
|
35818
36425
|
if (failure !== null) {
|
|
35819
36426
|
if (failure.code === Err.NETWORK) failure.retryAfterMs = networkRetryDelayMs(deps.noteNetworkFailure(sessionKey));
|
|
35820
36427
|
else deps.clearNetworkStreak(sessionKey);
|
|
35821
36428
|
}
|
|
35822
|
-
rec.settle(failure);
|
|
35823
36429
|
const superseded = deps.isSuperseded(sessionKey, rec);
|
|
35824
|
-
|
|
35825
|
-
|
|
35826
|
-
|
|
35827
|
-
|
|
35828
|
-
|
|
35829
|
-
|
|
35830
|
-
|
|
35831
|
-
|
|
35832
|
-
|
|
35833
|
-
|
|
35834
|
-
|
|
35835
|
-
|
|
35836
|
-
|
|
35837
|
-
|
|
35838
|
-
|
|
35839
|
-
|
|
35840
|
-
|
|
35841
|
-
|
|
35842
|
-
|
|
35843
|
-
|
|
35844
|
-
|
|
35845
|
-
|
|
35846
|
-
|
|
35847
|
-
|
|
35848
|
-
|
|
35849
|
-
|
|
35850
|
-
|
|
36430
|
+
try {
|
|
36431
|
+
await finalizeAttempt({
|
|
36432
|
+
rec,
|
|
36433
|
+
outcome,
|
|
36434
|
+
failure,
|
|
36435
|
+
store: deps.store,
|
|
36436
|
+
...deps.pool === void 0 ? {} : { pool: deps.pool },
|
|
36437
|
+
...input.account === void 0 ? {} : { account: input.account },
|
|
36438
|
+
family: input.family,
|
|
36439
|
+
persistBinding: input.persistBinding,
|
|
36440
|
+
sessionAccountKey: input.sessionAccountKey,
|
|
36441
|
+
...input.binding === void 0 ? {} : { binding: input.binding },
|
|
36442
|
+
conversationId,
|
|
36443
|
+
messages: input.messages,
|
|
36444
|
+
model: input.model,
|
|
36445
|
+
activeModel: input.activeModel,
|
|
36446
|
+
superseded,
|
|
36447
|
+
newlySentImageIds: input.newlySentImageIds,
|
|
36448
|
+
rawErrText,
|
|
36449
|
+
runLogText,
|
|
36450
|
+
silentTimeout,
|
|
36451
|
+
sessionKey,
|
|
36452
|
+
quotaFailover,
|
|
36453
|
+
cooledInLiveFamilies,
|
|
36454
|
+
clearNetworkStreak: (key) => deps.clearNetworkStreak(key),
|
|
36455
|
+
dropFailoverBudget: (key) => deps.dropFailoverBudget(key),
|
|
36456
|
+
clearRateLimitFailover: (key) => deps.clearRateLimitFailover(key),
|
|
36457
|
+
sharesSingleEgress: () => deps.sharesSingleEgress(),
|
|
36458
|
+
...deps.onRun === void 0 ? {} : { onRun: deps.onRun }
|
|
36459
|
+
});
|
|
36460
|
+
} finally {
|
|
36461
|
+
rec.settle(failure);
|
|
36462
|
+
}
|
|
35851
36463
|
}
|
|
35852
36464
|
/**
|
|
35853
36465
|
* Run one attempt: recorder → slot dispatch → spawn → concurrent outcome.
|
|
@@ -35855,7 +36467,10 @@ async function handleAttemptOutcome(deps, input, ctx) {
|
|
|
35855
36467
|
* queue notices; the outcome closure runs detached exactly as before.
|
|
35856
36468
|
*/
|
|
35857
36469
|
async function* runAgyAttempt(deps, input) {
|
|
35858
|
-
const recorder = createAttemptRecorder(deps, input.cfg, input.account, input.binding?.conversationId
|
|
36470
|
+
const recorder = createAttemptRecorder(deps, input.cfg, input.account, input.binding?.conversationId, true, {
|
|
36471
|
+
family: input.family,
|
|
36472
|
+
activeModel: input.activeModel
|
|
36473
|
+
});
|
|
35859
36474
|
const spawn = buildSpawnArgs(input, recorder.printTimeoutMinutes, deps.log);
|
|
35860
36475
|
const slot = yield* dispatchCredentialSlot(deps, {
|
|
35861
36476
|
...input,
|
|
@@ -36022,21 +36637,14 @@ var AgyAdapter = class extends LlmAdapter {
|
|
|
36022
36637
|
buildArgs(opts) {
|
|
36023
36638
|
return buildAgyArgs(opts);
|
|
36024
36639
|
}
|
|
36025
|
-
/** Effort and
|
|
36640
|
+
/** Effort, model, and aux-quota self-healing wrapper around streamAttempt (issue #67, PR #100). */
|
|
36026
36641
|
async *stream(options) {
|
|
36027
36642
|
yield* streamWithEffortRetry({
|
|
36028
36643
|
streamAttempt: (opts, suppressEffort) => this.streamAttempt(opts, suppressEffort),
|
|
36029
36644
|
learnEffortUnsupported: (model) => this.learnEffortUnsupported(model),
|
|
36030
36645
|
catalogAdvertisesEffort: (model) => this.deps.catalog.advertisedEfforts(model).length > 0,
|
|
36031
|
-
|
|
36032
|
-
|
|
36033
|
-
const pool = this.deps.pool;
|
|
36034
|
-
if (pool) {
|
|
36035
|
-
const currentId = pool.getActiveAccountId(fam);
|
|
36036
|
-
if (currentId) pool.markModelUnsupported(currentId, unrecog);
|
|
36037
|
-
pool.selectAccount(fam, unrecog);
|
|
36038
|
-
}
|
|
36039
|
-
}
|
|
36646
|
+
pool: this.deps.pool,
|
|
36647
|
+
defaultModel: () => this.deps.getConfig().defaultModel
|
|
36040
36648
|
}, options);
|
|
36041
36649
|
}
|
|
36042
36650
|
async *streamAttempt(options, suppressEffort = false) {
|
|
@@ -36192,22 +36800,25 @@ var AgyAdapter = class extends LlmAdapter {
|
|
|
36192
36800
|
}
|
|
36193
36801
|
if (samePrompt && prevPrompt !== void 0 && prevPrompt.inFlight && liveRun === void 0 && now - prevPrompt.startedAt < 2e3) throw new LlmError("Duplicate request ignored: an identical request is already running for this session.", Err.BUSY);
|
|
36194
36802
|
if (runLive && !samePrompt) {
|
|
36195
|
-
const liveCid = (liveRun.accountHome ?? "") === (account?.dir ?? "") ? liveRun.conversationId : null;
|
|
36196
|
-
if (liveCid !== null && binding === void 0 && !isAux && sessionAccountKey !== "") binding = {
|
|
36197
|
-
conversationId: liveCid,
|
|
36198
|
-
lastMessageCount: Math.max(0, lastAssistantIdx + 1),
|
|
36199
|
-
updatedAt: Date.now(),
|
|
36200
|
-
model: currentModel,
|
|
36201
|
-
interrupted: true
|
|
36202
|
-
};
|
|
36203
|
-
else if (binding !== void 0) binding.interrupted = true;
|
|
36204
36803
|
if (liveRun.hasResult) {
|
|
36205
36804
|
if (!await waitForRunSettle(liveRun, 1500)) this.deps.log?.("previous agy run did not exit within 1.5s after producing its result — preempting it");
|
|
36206
36805
|
}
|
|
36806
|
+
const wasConsumable = liveRun.hasConsumableResult;
|
|
36207
36807
|
if (!liveRun.isSettled) {
|
|
36808
|
+
const liveCid = (liveRun.accountHome ?? "") === (account?.dir ?? "") ? liveRun.conversationId : null;
|
|
36809
|
+
if (!wasConsumable) {
|
|
36810
|
+
if (liveCid !== null && binding === void 0 && !isAux && sessionAccountKey !== "") binding = {
|
|
36811
|
+
conversationId: liveCid,
|
|
36812
|
+
lastMessageCount: Math.max(0, lastAssistantIdx + 1),
|
|
36813
|
+
updatedAt: Date.now(),
|
|
36814
|
+
model: currentModel,
|
|
36815
|
+
interrupted: true
|
|
36816
|
+
};
|
|
36817
|
+
else if (binding !== void 0) binding.interrupted = true;
|
|
36818
|
+
}
|
|
36208
36819
|
liveRun.requestAbort?.();
|
|
36209
36820
|
if (!await waitForRunSettle(liveRun, 3500)) this.warnOnce("settle-timeout:" + sessionKey, "the preempted agy run did not exit within 3.5s — starting the replacement anyway; its late write-back is ignored because the newer run owns the session");
|
|
36210
|
-
if (!prompt.includes("the previous turn was interrupted")) prompt = "[the previous turn was interrupted before it finished — do not continue or answer that unfinished work; respond only to the newest request below]\n\n" + prompt;
|
|
36821
|
+
if (!wasConsumable && !prompt.includes("the previous turn was interrupted")) prompt = "[the previous turn was interrupted before it finished — do not continue or answer that unfinished work; respond only to the newest request below]\n\n" + prompt;
|
|
36211
36822
|
}
|
|
36212
36823
|
}
|
|
36213
36824
|
this.activeSessionPrompts.set(sessionKey, {
|
|
@@ -36449,12 +37060,12 @@ function createImageCatalog(cfg) {
|
|
|
36449
37060
|
}, cfg.fallbackModels, 3e5, void 0, { cacheFile: join(stateDir(), "model-catalog.json") });
|
|
36450
37061
|
}
|
|
36451
37062
|
/** Build the real adapter (same deps as the smoke scripts / host). */
|
|
36452
|
-
function createImageAdapter(cfg, log, catalog = createImageCatalog(cfg)) {
|
|
37063
|
+
function createImageAdapter(cfg, log, catalog = createImageCatalog(cfg), pool = new AccountPoolManager()) {
|
|
36453
37064
|
return new AgyAdapter({
|
|
36454
37065
|
getConfig: () => cfg,
|
|
36455
37066
|
catalog,
|
|
36456
37067
|
store: new SessionStore(join(stateDir(), "cli-image-sessions.json")),
|
|
36457
|
-
pool
|
|
37068
|
+
pool,
|
|
36458
37069
|
bin: () => resolveAgyBin(cfg),
|
|
36459
37070
|
acquire: () => Promise.resolve(() => void 0),
|
|
36460
37071
|
runs: new RunRegistry(),
|
|
@@ -36467,12 +37078,14 @@ function createDefaultImageService(args, _req, writeErr) {
|
|
|
36467
37078
|
if (cfg.workspaceRoot === "") cfg.workspaceRoot = process.cwd();
|
|
36468
37079
|
const bin = resolveAgyBin(cfg);
|
|
36469
37080
|
const catalog = createImageCatalog(cfg);
|
|
36470
|
-
const
|
|
37081
|
+
const pool = new AccountPoolManager();
|
|
37082
|
+
const adapter = createImageAdapter(cfg, args.quiet ? void 0 : (m) => writeErr("[agy] " + m + "\n"), catalog, pool);
|
|
36471
37083
|
return new ImageService({
|
|
36472
37084
|
cfg: () => cfg,
|
|
36473
37085
|
bin: () => bin,
|
|
36474
37086
|
run: (call) => runViaAdapter(adapter, call),
|
|
36475
37087
|
catalog: () => catalog.get(),
|
|
37088
|
+
...createPoolQuotaHooks(pool),
|
|
36476
37089
|
...args.quiet ? {} : { log: (m) => writeErr("[image] " + m + "\n") }
|
|
36477
37090
|
});
|
|
36478
37091
|
}
|