@tekmidian/pai 0.66.48 → 0.66.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auto-route-CMARQbQ4.mjs +8 -0
- package/dist/{auto-route-DrncbMaz.mjs → auto-route-Dpm6nZ6X.mjs} +2 -2
- package/dist/{auto-route-DrncbMaz.mjs.map → auto-route-Dpm6nZ6X.mjs.map} +1 -1
- package/dist/{chain-Bba4Y8A8.mjs → chain-xSc3-u0X.mjs} +5 -5
- package/dist/{chain-Bba4Y8A8.mjs.map → chain-xSc3-u0X.mjs.map} +1 -1
- package/dist/cli/index.mjs +14 -14
- package/dist/cli/program.mjs +14 -14
- package/dist/{config-2pR0RqHM.mjs → config-CYjjGSWq.mjs} +41 -4
- package/dist/config-CYjjGSWq.mjs.map +1 -0
- package/dist/config-IuOG1W92.mjs +4 -0
- package/dist/daemon/index.mjs +10 -10
- package/dist/{daemon-C25MKGD8.mjs → daemon-BIG7XcqE.mjs} +10 -10
- package/dist/{daemon-C6K3kikA.mjs → daemon-CiSmbylJ.mjs} +7 -7
- package/dist/{daemon-C6K3kikA.mjs.map → daemon-CiSmbylJ.mjs.map} +1 -1
- package/dist/daemon-mcp/index.mjs +8 -8
- package/dist/{fallback-DyhAZAOe.mjs → fallback-BgAukP8r.mjs} +6 -5
- package/dist/fallback-BgAukP8r.mjs.map +1 -0
- package/dist/hooks/block-sleep-poll.mjs.map +2 -2
- package/dist/hooks/load-project-context.mjs +26 -2
- package/dist/hooks/load-project-context.mjs.map +2 -2
- package/dist/hooks/route-agents-to-worker.mjs +26 -2
- package/dist/hooks/route-agents-to-worker.mjs.map +2 -2
- package/dist/hooks/security-validator.mjs +26 -2
- package/dist/hooks/security-validator.mjs.map +2 -2
- package/dist/hooks/worker-guard.mjs +26 -2
- package/dist/hooks/worker-guard.mjs.map +2 -2
- package/dist/hooks/worker-pool-host.mjs +242 -14
- package/dist/hooks/worker-pool-host.mjs.map +4 -4
- package/dist/hooks/worker-proxy.mjs +26 -2
- package/dist/hooks/worker-proxy.mjs.map +2 -2
- package/dist/hooks/worker-status-line.mjs +26 -2
- package/dist/hooks/worker-status-line.mjs.map +2 -2
- package/dist/hooks/worker-supervision.mjs +26 -2
- package/dist/hooks/worker-supervision.mjs.map +2 -2
- package/dist/index.mjs +4 -4
- package/dist/{local-EmOjw5jq.mjs → local-Bt-7L0oB.mjs} +3 -3
- package/dist/{local-EmOjw5jq.mjs.map → local-Bt-7L0oB.mjs.map} +1 -1
- package/dist/local-oqPQEmhb.mjs +7 -0
- package/dist/{local-tiers-BIINbhkY.mjs → local-tiers-C0QSQcP1.mjs} +2 -2
- package/dist/{local-tiers-8fmjiySw.mjs → local-tiers-_ZOTbjeP.mjs} +2 -2
- package/dist/{local-tiers-8fmjiySw.mjs.map → local-tiers-_ZOTbjeP.mjs.map} +1 -1
- package/dist/{main-resolver-DBXvnYKp.mjs → main-resolver-BwjexuSA.mjs} +3 -3
- package/dist/{main-resolver-B10-lsU0.mjs → main-resolver-CVo0ab7j.mjs} +3 -3
- package/dist/{main-resolver-B10-lsU0.mjs.map → main-resolver-CVo0ab7j.mjs.map} +1 -1
- package/dist/{pai-marker-CDwQOpQ8.mjs → pai-marker-CJ6WMt05.mjs} +4 -4
- package/dist/{pai-marker-CDwQOpQ8.mjs.map → pai-marker-CJ6WMt05.mjs.map} +1 -1
- package/dist/{planner-dO-3PLjD.mjs → planner-ZKXZf9JS.mjs} +8 -8
- package/dist/{planner-dO-3PLjD.mjs.map → planner-ZKXZf9JS.mjs.map} +1 -1
- package/dist/{pool-DLhqOY3P.mjs → pool-CgsnP3La.mjs} +3 -3
- package/dist/{pool-DLhqOY3P.mjs.map → pool-CgsnP3La.mjs.map} +1 -1
- package/dist/{pool-cli-DerrWWpP.mjs → pool-cli-kGGuZCzJ.mjs} +4 -4
- package/dist/{pool-cli-DerrWWpP.mjs.map → pool-cli-kGGuZCzJ.mjs.map} +1 -1
- package/dist/{program-B7gVKc-u.mjs → program-DgcwJN6Z.mjs} +25 -463
- package/dist/program-DgcwJN6Z.mjs.map +1 -0
- package/dist/{run-a_JzcVwv.mjs → run-CsrBbtj_.mjs} +686 -21
- package/dist/run-CsrBbtj_.mjs.map +1 -0
- package/dist/{run-env-DyFZ8C1s.mjs → run-env-Cyw2iwBh.mjs} +15 -5
- package/dist/{run-env-DyFZ8C1s.mjs.map → run-env-Cyw2iwBh.mjs.map} +1 -1
- package/dist/{status-C3fBJKU_.mjs → status-B0br9zV0.mjs} +2 -2
- package/dist/status-B0br9zV0.mjs.map +1 -0
- package/dist/{supervisor-B_2JKYxe.mjs → supervisor-BZA0KICt.mjs} +7 -7
- package/dist/{supervisor-B_2JKYxe.mjs.map → supervisor-BZA0KICt.mjs.map} +1 -1
- package/dist/workers-config-BifEz8ns.mjs +4 -0
- package/dist/{worktree-lnmAOFQv.mjs → worktree-B6d-1uIn.mjs} +2 -2
- package/dist/{worktree-lnmAOFQv.mjs.map → worktree-B6d-1uIn.mjs.map} +1 -1
- package/docs/worker.md +22 -0
- package/package.json +1 -1
- package/dist/auto-route-DYuVfXwc.mjs +0 -8
- package/dist/config-2pR0RqHM.mjs.map +0 -1
- package/dist/config-CQWpEdXb.mjs +0 -4
- package/dist/fallback-DyhAZAOe.mjs.map +0 -1
- package/dist/local-B5SekWOC.mjs +0 -7
- package/dist/program-B7gVKc-u.mjs.map +0 -1
- package/dist/run-a_JzcVwv.mjs.map +0 -1
- package/dist/status-C3fBJKU_.mjs.map +0 -1
- package/dist/workers-config-nXSiW07z.mjs +0 -4
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
import { u as paiHomePath } from "./runtime-paths-DkZoxofH.mjs";
|
|
2
|
-
import {
|
|
3
|
-
import { B as parseRunnerArgs, F as preambleBlock, G as ledgerPath, H as stripPromptValues, I as preambleLine, J as statusPath, K as panesDir, P as loadPreamble, R as appendSystemPromptArgs, U as ensureNoMcpConfig, V as shortText, W as eventsPath, Y as workersLogDir, _ as parseSleepSecs, d as isLive, f as loadStatus, h as nowStamp, m as newWorkerId, p as loadStatuses, q as routingStatePath, s as describeTool, t as UNLABELED, v as saveStatus } from "./status-
|
|
4
|
-
import { a as gcWorktreesThrottled, b as appendLedger, c as markResumed, d as recordWorktree, f as salvageOnExit, h as worktreeWanted, m as worktreeSystemPrompt, n as adoptTree, s as inPlaceSystemPrompt, t as addWorktree } from "./worktree-
|
|
2
|
+
import { B as resolveModelCapability, E as isModelCapability, I as providerCostTier, L as providerKeyPath, O as maskKey, R as readWorkersSection, V as resolveProviderKey, a as DEFAULT_BUDGET_WALLS, b as assertProviderRunnable, g as NATIVE_ANTHROPIC_MODELS, h as MODEL_CAPABILITIES, i as DEFAULT_BUDGET_WALL, it as certainContextWindow, k as nativeAnthropicProvider, r as DEFAULT_BUDGET_PRIORITY, t as ANTHROPIC_NATIVE, v as WORKER_CLASSES, w as getProviderOrNative, x as classModelCapability, y as WorkersConfigError, z as resolveCapability } from "./config-CYjjGSWq.mjs";
|
|
3
|
+
import { B as parseRunnerArgs, F as preambleBlock, G as ledgerPath, H as stripPromptValues, I as preambleLine, J as statusPath, K as panesDir, P as loadPreamble, R as appendSystemPromptArgs, U as ensureNoMcpConfig, V as shortText, W as eventsPath, Y as workersLogDir, _ as parseSleepSecs, d as isLive, f as loadStatus, h as nowStamp, m as newWorkerId, p as loadStatuses, q as routingStatePath, s as describeTool, t as UNLABELED, v as saveStatus } from "./status-B0br9zV0.mjs";
|
|
4
|
+
import { a as gcWorktreesThrottled, b as appendLedger, c as markResumed, d as recordWorktree, f as salvageOnExit, h as worktreeWanted, m as worktreeSystemPrompt, n as adoptTree, s as inPlaceSystemPrompt, t as addWorktree } from "./worktree-B6d-1uIn.mjs";
|
|
5
5
|
import { i as getRegistryBackend } from "./factory-DRAEDNKV.mjs";
|
|
6
|
-
import { _ as readMcpServers, c as poolSettings, g as mcpServersFromToolGrants, h as grantsChrome, i as poolIneligibleReason, l as poolSlotCwd, m as expandMcpNames, u as postTaskToPool, v as writeMcpConfig } from "./pool-
|
|
7
|
-
import { n as claudeCommand, r as pinnedSettingsArgs, t as buildRunEnv } from "./run-env-
|
|
6
|
+
import { _ as readMcpServers, c as poolSettings, g as mcpServersFromToolGrants, h as grantsChrome, i as poolIneligibleReason, l as poolSlotCwd, m as expandMcpNames, u as postTaskToPool, v as writeMcpConfig } from "./pool-CgsnP3La.mjs";
|
|
7
|
+
import { n as claudeCommand, r as pinnedSettingsArgs, t as buildRunEnv } from "./run-env-Cyw2iwBh.mjs";
|
|
8
8
|
import { appendFileSync, closeSync, existsSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
|
|
9
9
|
import { homedir, tmpdir } from "node:os";
|
|
10
10
|
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
|
|
11
|
-
import { execFileSync, spawn } from "node:child_process";
|
|
11
|
+
import { execFile, execFileSync, spawn, spawnSync } from "node:child_process";
|
|
12
12
|
import { createHash } from "node:crypto";
|
|
13
13
|
import { fileURLToPath } from "node:url";
|
|
14
|
-
import { connect, createServer } from "node:net";
|
|
15
|
-
import { createServer as createServer$1 } from "node:http";
|
|
14
|
+
import net, { connect, createServer } from "node:net";
|
|
15
|
+
import http, { createServer as createServer$1 } from "node:http";
|
|
16
|
+
import https from "node:https";
|
|
16
17
|
import { createInterface } from "node:readline";
|
|
17
18
|
|
|
18
19
|
//#region src/workers/scope.ts
|
|
@@ -1275,6 +1276,595 @@ function pinNotices(launch, claudeJson) {
|
|
|
1275
1276
|
return lines;
|
|
1276
1277
|
}
|
|
1277
1278
|
|
|
1279
|
+
//#endregion
|
|
1280
|
+
//#region src/router/limits.ts
|
|
1281
|
+
/**
|
|
1282
|
+
* limits.ts — per-provider plan limits for the cockpit. Polled at most once a
|
|
1283
|
+
* minute and cached; keys and tokens stay in this process and never reach the
|
|
1284
|
+
* page or the JSON it is served.
|
|
1285
|
+
*
|
|
1286
|
+
* anthropic Max plan 5h/7d via the OAuth usage endpoint (token from the
|
|
1287
|
+
* keychain, as the statusline does); on a failed poll (http 429)
|
|
1288
|
+
* the last good reading is served with its age and the poll backs
|
|
1289
|
+
* off, advisor-mode.json only when no last good reading exists
|
|
1290
|
+
* others the provider's workers.yaml `usage` block (jq expressions)
|
|
1291
|
+
* local up/down (TCP), RSS of the listener, tokens/s from accounting
|
|
1292
|
+
* rest counted requests vs a configured limit, labelled "estimated"
|
|
1293
|
+
*/
|
|
1294
|
+
const ESTIMATE_FILE = join(homedir(), ".claude", "pai", "router-limits.json");
|
|
1295
|
+
const ADVISOR_FILE = join(homedir(), ".claude", "pai", "advisor-mode.json");
|
|
1296
|
+
/** Last good plan reading per provider, so a rate-limited poll shows an age, not "reset unknown". */
|
|
1297
|
+
const LASTGOOD_FILE = join(homedir(), ".claude", "pai", "logs", "router-limits-last.json");
|
|
1298
|
+
/** A last good reading older than this is too stale to serve when the endpoint fails. */
|
|
1299
|
+
const PLAN_LASTGOOD_MAX_AGE = 2 * 36e5;
|
|
1300
|
+
/** Upper bound for the poll delay after repeated 429s. */
|
|
1301
|
+
const BACKOFF_MAX = 30 * 6e4;
|
|
1302
|
+
function readLastGood(file = LASTGOOD_FILE) {
|
|
1303
|
+
try {
|
|
1304
|
+
const j = JSON.parse(readFileSync(file, "utf8"));
|
|
1305
|
+
const out = {};
|
|
1306
|
+
for (const [k, v] of Object.entries(j)) if (v && Array.isArray(v.windows) && typeof v.fetchedAt === "number") out[k] = {
|
|
1307
|
+
windows: v.windows,
|
|
1308
|
+
fetchedAt: v.fetchedAt
|
|
1309
|
+
};
|
|
1310
|
+
return out;
|
|
1311
|
+
} catch {
|
|
1312
|
+
return {};
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
function writeLastGood(file, entries) {
|
|
1316
|
+
try {
|
|
1317
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
1318
|
+
writeFileSync(file, JSON.stringify(entries, null, 1));
|
|
1319
|
+
} catch {}
|
|
1320
|
+
}
|
|
1321
|
+
/**
|
|
1322
|
+
* The reading to show when a poll failed: the last good one, with its age in
|
|
1323
|
+
* the error field, while it is fresh enough to trust; undefined otherwise.
|
|
1324
|
+
*/
|
|
1325
|
+
function staleReading(provider, entry, cause, now, maxAge = PLAN_LASTGOOD_MAX_AGE) {
|
|
1326
|
+
if (!entry || !entry.windows.length || now - entry.fetchedAt >= maxAge) return void 0;
|
|
1327
|
+
const min = Math.round((now - entry.fetchedAt) / 6e4);
|
|
1328
|
+
return {
|
|
1329
|
+
provider,
|
|
1330
|
+
kind: "plan",
|
|
1331
|
+
windows: entry.windows,
|
|
1332
|
+
fetchedAt: entry.fetchedAt,
|
|
1333
|
+
error: `stale ${min}min (${cause})`
|
|
1334
|
+
};
|
|
1335
|
+
}
|
|
1336
|
+
/** Per-provider poll backoff after a http 429: the interval doubles per strike, resets on success. */
|
|
1337
|
+
var Backoff = class {
|
|
1338
|
+
mult = /* @__PURE__ */ new Map();
|
|
1339
|
+
until = /* @__PURE__ */ new Map();
|
|
1340
|
+
constructor(baseMs, maxMs = BACKOFF_MAX) {
|
|
1341
|
+
this.baseMs = baseMs;
|
|
1342
|
+
this.maxMs = maxMs;
|
|
1343
|
+
}
|
|
1344
|
+
/** Record a 429 seen for `provider` at `now`; returns the delay before its next poll. */
|
|
1345
|
+
penalize(provider, now) {
|
|
1346
|
+
const m = Math.min((this.mult.get(provider) ?? 1) * 2, this.maxMs / this.baseMs);
|
|
1347
|
+
this.mult.set(provider, m);
|
|
1348
|
+
this.until.set(provider, now + m * this.baseMs);
|
|
1349
|
+
return m * this.baseMs;
|
|
1350
|
+
}
|
|
1351
|
+
/** ms until `provider` is due again; 0 when it may be polled now. */
|
|
1352
|
+
remaining(provider, now) {
|
|
1353
|
+
return Math.max(0, (this.until.get(provider) ?? 0) - now);
|
|
1354
|
+
}
|
|
1355
|
+
/** A poll succeeded: forget the strikes. */
|
|
1356
|
+
reset(provider) {
|
|
1357
|
+
this.mult.delete(provider);
|
|
1358
|
+
this.until.delete(provider);
|
|
1359
|
+
}
|
|
1360
|
+
};
|
|
1361
|
+
function toMs(v, unit) {
|
|
1362
|
+
if (v === null || v === void 0 || v === "") return void 0;
|
|
1363
|
+
if (unit === "iso" || unit === void 0 && typeof v === "string" && !/^\d+(\.\d+)?$/.test(v)) {
|
|
1364
|
+
const t = Date.parse(String(v));
|
|
1365
|
+
return Number.isFinite(t) ? t : void 0;
|
|
1366
|
+
}
|
|
1367
|
+
const n = Number(v);
|
|
1368
|
+
if (!Number.isFinite(n)) return void 0;
|
|
1369
|
+
return unit === "s" ? n * 1e3 : n;
|
|
1370
|
+
}
|
|
1371
|
+
function jq(expr, input) {
|
|
1372
|
+
const r = spawnSync("jq", ["-r", expr], {
|
|
1373
|
+
input,
|
|
1374
|
+
encoding: "utf8",
|
|
1375
|
+
timeout: 3e3
|
|
1376
|
+
});
|
|
1377
|
+
if (r.status !== 0) return void 0;
|
|
1378
|
+
const out = r.stdout.trim().split("\n")[0];
|
|
1379
|
+
return out === void 0 || out === "" || out === "null" ? void 0 : out;
|
|
1380
|
+
}
|
|
1381
|
+
/** Windows of a provider usage response, using its configured jq expressions. */
|
|
1382
|
+
function windowsFromJson(body, cfg) {
|
|
1383
|
+
const out = [];
|
|
1384
|
+
for (const w of cfg.windows) {
|
|
1385
|
+
const pct = Number(jq(w.percent, body));
|
|
1386
|
+
if (!Number.isFinite(pct) || jq(w.percent, body) === void 0) continue;
|
|
1387
|
+
const reset = w.resetAt ? toMs(jq(w.resetAt, body), w.resetUnit) : void 0;
|
|
1388
|
+
out.push({
|
|
1389
|
+
name: w.name,
|
|
1390
|
+
percent: Math.round(pct * 10) / 10,
|
|
1391
|
+
...reset ? { resetAt: reset } : {}
|
|
1392
|
+
});
|
|
1393
|
+
}
|
|
1394
|
+
return out;
|
|
1395
|
+
}
|
|
1396
|
+
/** Anthropic OAuth usage response: five_hour / seven_day {utilization, resets_at ISO}. */
|
|
1397
|
+
function parseAnthropicUsage(body) {
|
|
1398
|
+
let j;
|
|
1399
|
+
try {
|
|
1400
|
+
j = JSON.parse(body);
|
|
1401
|
+
} catch {
|
|
1402
|
+
return [];
|
|
1403
|
+
}
|
|
1404
|
+
const out = [];
|
|
1405
|
+
for (const [key, name] of [["five_hour", "5h"], ["seven_day", "7d"]]) {
|
|
1406
|
+
const w = j[key];
|
|
1407
|
+
if (!w || typeof w.utilization !== "number") continue;
|
|
1408
|
+
const reset = toMs(w.resets_at, "iso");
|
|
1409
|
+
out.push({
|
|
1410
|
+
name,
|
|
1411
|
+
percent: w.utilization,
|
|
1412
|
+
...reset ? { resetAt: reset } : {}
|
|
1413
|
+
});
|
|
1414
|
+
}
|
|
1415
|
+
return out;
|
|
1416
|
+
}
|
|
1417
|
+
/** Counted requests inside each window vs a configured cap: labelled estimated. */
|
|
1418
|
+
function estimateWindows(rows, provider, caps, now) {
|
|
1419
|
+
const spans = {
|
|
1420
|
+
"5h": 5 * 36e5,
|
|
1421
|
+
"7d": 7 * 864e5
|
|
1422
|
+
};
|
|
1423
|
+
const out = [];
|
|
1424
|
+
for (const [name, span] of Object.entries(spans)) {
|
|
1425
|
+
const cap = caps[name];
|
|
1426
|
+
if (!cap) continue;
|
|
1427
|
+
const inWin = rows.filter((r) => r.provider === provider && Date.parse(r.ts) >= now - span);
|
|
1428
|
+
const first = inWin.reduce((m, r) => Math.min(m, Date.parse(r.ts)), Infinity);
|
|
1429
|
+
out.push({
|
|
1430
|
+
name,
|
|
1431
|
+
percent: Math.round(inWin.length / cap * 1e3) / 10,
|
|
1432
|
+
...Number.isFinite(first) ? { resetAt: first + span } : {},
|
|
1433
|
+
estimated: true
|
|
1434
|
+
});
|
|
1435
|
+
}
|
|
1436
|
+
return out;
|
|
1437
|
+
}
|
|
1438
|
+
function readEstimateCaps(file = ESTIMATE_FILE) {
|
|
1439
|
+
try {
|
|
1440
|
+
return existsSync(file) ? JSON.parse(readFileSync(file, "utf8")) : {};
|
|
1441
|
+
} catch {
|
|
1442
|
+
return {};
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
/** The OAuth usage endpoint the anthropic plan readings come from. */
|
|
1446
|
+
const ANTHROPIC_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
1447
|
+
function getJson(url, headers) {
|
|
1448
|
+
return new Promise((resolve, reject) => {
|
|
1449
|
+
const rq = (url.protocol === "https:" ? https : http).get(url, {
|
|
1450
|
+
headers,
|
|
1451
|
+
timeout: 5e3
|
|
1452
|
+
}, (res) => {
|
|
1453
|
+
let t = "";
|
|
1454
|
+
res.on("data", (c) => t += c);
|
|
1455
|
+
res.on("end", () => res.statusCode === 200 ? resolve(t) : reject(/* @__PURE__ */ new Error(`http ${res.statusCode}`)));
|
|
1456
|
+
});
|
|
1457
|
+
rq.on("timeout", () => rq.destroy(/* @__PURE__ */ new Error("timeout")));
|
|
1458
|
+
rq.on("error", reject);
|
|
1459
|
+
});
|
|
1460
|
+
}
|
|
1461
|
+
function sh(cmd, args) {
|
|
1462
|
+
return new Promise((resolve) => execFile(cmd, args, { timeout: 4e3 }, (_e, out) => resolve(String(out ?? "").trim())));
|
|
1463
|
+
}
|
|
1464
|
+
async function anthropicToken() {
|
|
1465
|
+
for (const args of [["-a", process.env.USER ?? ""], []]) {
|
|
1466
|
+
const raw = await sh("security", [
|
|
1467
|
+
"find-generic-password",
|
|
1468
|
+
"-s",
|
|
1469
|
+
"Claude Code-credentials",
|
|
1470
|
+
...args,
|
|
1471
|
+
"-w"
|
|
1472
|
+
]);
|
|
1473
|
+
try {
|
|
1474
|
+
const t = JSON.parse(raw).claudeAiOauth?.accessToken;
|
|
1475
|
+
if (t) return t;
|
|
1476
|
+
} catch {}
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1479
|
+
async function anthropicLimits(now, advisorFile, lastGood) {
|
|
1480
|
+
const base = {
|
|
1481
|
+
provider: "anthropic",
|
|
1482
|
+
kind: "plan",
|
|
1483
|
+
fetchedAt: now
|
|
1484
|
+
};
|
|
1485
|
+
try {
|
|
1486
|
+
const token = await anthropicToken();
|
|
1487
|
+
if (!token) throw new Error("no OAuth token in keychain");
|
|
1488
|
+
const windows = parseAnthropicUsage(await getJson(new URL(ANTHROPIC_USAGE_URL), {
|
|
1489
|
+
authorization: `Bearer ${token}`,
|
|
1490
|
+
"anthropic-beta": "oauth-2025-04-20"
|
|
1491
|
+
}));
|
|
1492
|
+
if (windows.length) return {
|
|
1493
|
+
...base,
|
|
1494
|
+
windows
|
|
1495
|
+
};
|
|
1496
|
+
throw new Error("no usage windows in response");
|
|
1497
|
+
} catch (e) {
|
|
1498
|
+
const stale = staleReading("anthropic", lastGood, e.message, now);
|
|
1499
|
+
if (stale) return stale;
|
|
1500
|
+
try {
|
|
1501
|
+
const a = JSON.parse(readFileSync(advisorFile, "utf8"));
|
|
1502
|
+
if (typeof a.weeklyBudgetPercent === "number" && a.asOf && now / 1e3 - a.asOf < 6 * 3600) return {
|
|
1503
|
+
...base,
|
|
1504
|
+
windows: [{
|
|
1505
|
+
name: "7d",
|
|
1506
|
+
percent: a.weeklyBudgetPercent
|
|
1507
|
+
}],
|
|
1508
|
+
error: `${e.message}; advisor-mode fallback`
|
|
1509
|
+
};
|
|
1510
|
+
} catch {}
|
|
1511
|
+
return {
|
|
1512
|
+
...base,
|
|
1513
|
+
windows: [],
|
|
1514
|
+
error: e.message
|
|
1515
|
+
};
|
|
1516
|
+
}
|
|
1517
|
+
}
|
|
1518
|
+
async function providerPlanLimits(name, p, now, lastGood) {
|
|
1519
|
+
const base = {
|
|
1520
|
+
provider: name,
|
|
1521
|
+
kind: "plan",
|
|
1522
|
+
fetchedAt: now
|
|
1523
|
+
};
|
|
1524
|
+
const u = p.usage;
|
|
1525
|
+
try {
|
|
1526
|
+
const key = resolveProviderKey(p);
|
|
1527
|
+
const [hn, ...pre] = (u.authHeader ?? "Authorization: Bearer").split(/:\s*/);
|
|
1528
|
+
const prefix = pre.join(": ");
|
|
1529
|
+
const headers = { [hn.toLowerCase()]: prefix ? `${prefix} ${key}` : String(key) };
|
|
1530
|
+
const windows = windowsFromJson(await getJson(new URL(u.url), headers), u);
|
|
1531
|
+
return windows.length ? {
|
|
1532
|
+
...base,
|
|
1533
|
+
windows
|
|
1534
|
+
} : {
|
|
1535
|
+
...base,
|
|
1536
|
+
windows,
|
|
1537
|
+
error: "no windows parsed"
|
|
1538
|
+
};
|
|
1539
|
+
} catch (e) {
|
|
1540
|
+
const msg = e.message.replace(/Bearer\s+\S+/g, "Bearer ***");
|
|
1541
|
+
return staleReading(name, lastGood, msg, now) ?? {
|
|
1542
|
+
...base,
|
|
1543
|
+
windows: [],
|
|
1544
|
+
error: msg
|
|
1545
|
+
};
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
function probeTcp(port, host) {
|
|
1549
|
+
return new Promise((resolve) => {
|
|
1550
|
+
const s = net.connect({
|
|
1551
|
+
port,
|
|
1552
|
+
host,
|
|
1553
|
+
timeout: 800
|
|
1554
|
+
});
|
|
1555
|
+
s.on("connect", () => (s.destroy(), resolve(true)));
|
|
1556
|
+
s.on("timeout", () => (s.destroy(), resolve(false)));
|
|
1557
|
+
s.on("error", () => resolve(false));
|
|
1558
|
+
});
|
|
1559
|
+
}
|
|
1560
|
+
async function localLimits(name, p, rows, now) {
|
|
1561
|
+
const u = new URL(p.baseUrl);
|
|
1562
|
+
const port = Number(u.port) || 80;
|
|
1563
|
+
const listening = await probeTcp(port, u.hostname);
|
|
1564
|
+
const served = listening ? await servedModels(u) : void 0;
|
|
1565
|
+
const up = listening && modelServed(served, Object.values(p.models).filter((m) => typeof m === "string"));
|
|
1566
|
+
let rssMb;
|
|
1567
|
+
if (up) {
|
|
1568
|
+
const pid = (await sh("lsof", [
|
|
1569
|
+
"-ti",
|
|
1570
|
+
`tcp:${port}`,
|
|
1571
|
+
"-sTCP:LISTEN"
|
|
1572
|
+
])).split("\n")[0];
|
|
1573
|
+
const rss = pid ? Number(await sh("ps", [
|
|
1574
|
+
"-o",
|
|
1575
|
+
"rss=",
|
|
1576
|
+
"-p",
|
|
1577
|
+
pid
|
|
1578
|
+
])) : NaN;
|
|
1579
|
+
if (Number.isFinite(rss) && rss > 0) rssMb = Math.round(rss / 1024);
|
|
1580
|
+
}
|
|
1581
|
+
return {
|
|
1582
|
+
provider: name,
|
|
1583
|
+
kind: "local",
|
|
1584
|
+
windows: [],
|
|
1585
|
+
up,
|
|
1586
|
+
...rssMb ? { rssMb } : {},
|
|
1587
|
+
...served ? { served } : {},
|
|
1588
|
+
...tokensPerSec(rows, name, now),
|
|
1589
|
+
fetchedAt: now
|
|
1590
|
+
};
|
|
1591
|
+
}
|
|
1592
|
+
/** Model ids the server at this base URL lists on GET /v1/models; undefined when it cannot say. */
|
|
1593
|
+
async function servedModels(base) {
|
|
1594
|
+
try {
|
|
1595
|
+
const ids = (JSON.parse(await getJson(new URL("/v1/models", base), {})).data ?? []).map((m) => m.id).filter((i) => typeof i === "string");
|
|
1596
|
+
return ids.length ? ids : void 0;
|
|
1597
|
+
} catch {
|
|
1598
|
+
return;
|
|
1599
|
+
}
|
|
1600
|
+
}
|
|
1601
|
+
/** Several providers can share one port: the provider is up only if its model is the one served. Unknown list = port decides. */
|
|
1602
|
+
function modelServed(served, wanted) {
|
|
1603
|
+
if (!served) return true;
|
|
1604
|
+
return wanted.some((w) => served.includes(w));
|
|
1605
|
+
}
|
|
1606
|
+
/** output tokens / generation seconds over the last hour (slow-path requests only). */
|
|
1607
|
+
function tokensPerSec(rows, provider, now) {
|
|
1608
|
+
let out = 0;
|
|
1609
|
+
let ms = 0;
|
|
1610
|
+
for (const r of rows) {
|
|
1611
|
+
if (r.provider !== provider || Date.parse(r.ts) < now - 36e5 || r.status !== 200 || !r.output_tokens) continue;
|
|
1612
|
+
out += r.output_tokens;
|
|
1613
|
+
ms += r.ms;
|
|
1614
|
+
}
|
|
1615
|
+
return ms ? { tokensPerSec: Math.round(out / (ms / 1e3) * 10) / 10 } : {};
|
|
1616
|
+
}
|
|
1617
|
+
function isLocalUrl(u) {
|
|
1618
|
+
try {
|
|
1619
|
+
const h = new URL(u).hostname;
|
|
1620
|
+
return h === "127.0.0.1" || h === "localhost";
|
|
1621
|
+
} catch {
|
|
1622
|
+
return false;
|
|
1623
|
+
}
|
|
1624
|
+
}
|
|
1625
|
+
/**
|
|
1626
|
+
* Polls all providers at most once per `ttlMs`; concurrent callers share one
|
|
1627
|
+
* poll. A provider whose poll answered http 429 is not fetched again until its
|
|
1628
|
+
* backoff (the interval doubled per strike, capped) has run out — its last
|
|
1629
|
+
* good reading is served with the current age instead.
|
|
1630
|
+
*/
|
|
1631
|
+
var LimitsCache = class {
|
|
1632
|
+
value = [];
|
|
1633
|
+
at = 0;
|
|
1634
|
+
inflight;
|
|
1635
|
+
backoff;
|
|
1636
|
+
constructor(deps, ttlMs = 6e4) {
|
|
1637
|
+
this.deps = deps;
|
|
1638
|
+
this.ttlMs = ttlMs;
|
|
1639
|
+
this.backoff = new Backoff(ttlMs);
|
|
1640
|
+
}
|
|
1641
|
+
get(now = Date.now()) {
|
|
1642
|
+
if (this.value.length && now - this.at < this.ttlMs) return Promise.resolve(this.value);
|
|
1643
|
+
this.inflight ??= this.poll(now).then((v) => {
|
|
1644
|
+
this.value = v;
|
|
1645
|
+
this.at = now;
|
|
1646
|
+
this.inflight = void 0;
|
|
1647
|
+
return v;
|
|
1648
|
+
});
|
|
1649
|
+
return this.inflight;
|
|
1650
|
+
}
|
|
1651
|
+
async poll(now) {
|
|
1652
|
+
const rows = this.deps.rows();
|
|
1653
|
+
let providers = {};
|
|
1654
|
+
try {
|
|
1655
|
+
providers = this.deps.providers();
|
|
1656
|
+
} catch {}
|
|
1657
|
+
const caps = readEstimateCaps(this.deps.estimateFile);
|
|
1658
|
+
const lastGoodFile = this.deps.lastGoodFile ?? LASTGOOD_FILE;
|
|
1659
|
+
const lastGood = readLastGood(lastGoodFile);
|
|
1660
|
+
const previous = new Map(this.value.map((v) => [v.provider, v]));
|
|
1661
|
+
const jobs = [];
|
|
1662
|
+
const skipped = [];
|
|
1663
|
+
/** Queue a plan poll unless the provider is backing off from a 429. */
|
|
1664
|
+
const plan = (name, fn) => {
|
|
1665
|
+
if (this.backoff.remaining(name, now) > 0) {
|
|
1666
|
+
skipped.push(name);
|
|
1667
|
+
return;
|
|
1668
|
+
}
|
|
1669
|
+
jobs.push(fn(lastGood[name]));
|
|
1670
|
+
};
|
|
1671
|
+
plan("anthropic", (lg) => anthropicLimits(now, this.deps.advisorFile ?? ADVISOR_FILE, lg));
|
|
1672
|
+
for (const [name, p] of Object.entries(providers)) {
|
|
1673
|
+
if (p.native || name === "anthropic" || p.enabled === false || !p.baseUrl) continue;
|
|
1674
|
+
if (isLocalUrl(p.baseUrl)) jobs.push(localLimits(name, p, rows, now));
|
|
1675
|
+
else if (p.usage) plan(name, (lg) => providerPlanLimits(name, p, now, lg));
|
|
1676
|
+
else if (caps[name]) jobs.push(Promise.resolve({
|
|
1677
|
+
provider: name,
|
|
1678
|
+
kind: "estimated",
|
|
1679
|
+
windows: estimateWindows(rows, name, caps[name], now),
|
|
1680
|
+
fetchedAt: now
|
|
1681
|
+
}));
|
|
1682
|
+
}
|
|
1683
|
+
const results = await Promise.all(jobs);
|
|
1684
|
+
const store = { ...lastGood };
|
|
1685
|
+
let dirty = false;
|
|
1686
|
+
for (const r of results) if (r.error) {
|
|
1687
|
+
if (r.error.includes("http 429")) this.backoff.penalize(r.provider, now);
|
|
1688
|
+
} else {
|
|
1689
|
+
this.backoff.reset(r.provider);
|
|
1690
|
+
if (r.windows.length) {
|
|
1691
|
+
store[r.provider] = {
|
|
1692
|
+
windows: r.windows,
|
|
1693
|
+
fetchedAt: r.fetchedAt
|
|
1694
|
+
};
|
|
1695
|
+
dirty = true;
|
|
1696
|
+
}
|
|
1697
|
+
}
|
|
1698
|
+
if (dirty) writeLastGood(lastGoodFile, store);
|
|
1699
|
+
for (const name of skipped) {
|
|
1700
|
+
const prev = previous.get(name);
|
|
1701
|
+
const cause = prev?.error?.includes("http 429") ? "http 429" : prev?.error ?? "http 429";
|
|
1702
|
+
results.push(prev ? {
|
|
1703
|
+
...prev,
|
|
1704
|
+
error: `stale ${Math.max(0, Math.round((now - prev.fetchedAt) / 6e4))}min (${cause})`
|
|
1705
|
+
} : {
|
|
1706
|
+
provider: name,
|
|
1707
|
+
kind: "plan",
|
|
1708
|
+
windows: [],
|
|
1709
|
+
error: cause,
|
|
1710
|
+
fetchedAt: now
|
|
1711
|
+
});
|
|
1712
|
+
}
|
|
1713
|
+
return results;
|
|
1714
|
+
}
|
|
1715
|
+
};
|
|
1716
|
+
|
|
1717
|
+
//#endregion
|
|
1718
|
+
//#region src/workers/budget.ts
|
|
1719
|
+
/**
|
|
1720
|
+
* budget.ts — live budget probes for budget-aware routing (policy "budget").
|
|
1721
|
+
*
|
|
1722
|
+
* One reading per provider: {percent, resetAt} of its most-constrained usage
|
|
1723
|
+
* window, from the same endpoints the cockpit limits card polls — anthropic
|
|
1724
|
+
* via the OAuth usage endpoint (keychain token), every other provider via its
|
|
1725
|
+
* workers.yaml `usage` block. Readings are cached for BUDGET_TTL_MS so a
|
|
1726
|
+
* burst of spawns costs one poll, and a failed probe reads as "unknown",
|
|
1727
|
+
* never as "full": routing must not break because an endpoint did.
|
|
1728
|
+
*
|
|
1729
|
+
* The routing decision itself lives in routing.ts (wall skip, cooldown until
|
|
1730
|
+
* reset); this module only fetches and caches. Keys never leave this process
|
|
1731
|
+
* and never appear in a reading or an error.
|
|
1732
|
+
*/
|
|
1733
|
+
/** How long a reading is trusted before the next spawn re-probes. */
|
|
1734
|
+
const BUDGET_TTL_MS = 5 * 6e4;
|
|
1735
|
+
/**
|
|
1736
|
+
* The most-constrained window of a usage response: highest percent, with its
|
|
1737
|
+
* reset. A provider is as full as its fullest window (a 96% 5h window is the
|
|
1738
|
+
* wall even when the 7d window reads 20%).
|
|
1739
|
+
*/
|
|
1740
|
+
function pickBudgetWindow(windows, now) {
|
|
1741
|
+
let best = null;
|
|
1742
|
+
for (const w of windows) if (!best || w.percent > best.percent) best = w;
|
|
1743
|
+
if (!best) return {
|
|
1744
|
+
percent: null,
|
|
1745
|
+
resetAt: null,
|
|
1746
|
+
fetchedAt: now
|
|
1747
|
+
};
|
|
1748
|
+
return {
|
|
1749
|
+
percent: best.percent,
|
|
1750
|
+
resetAt: best.resetAt ?? null,
|
|
1751
|
+
fetchedAt: now
|
|
1752
|
+
};
|
|
1753
|
+
}
|
|
1754
|
+
/**
|
|
1755
|
+
* Fetch one provider's budget reading. anthropic (native) reads the OAuth
|
|
1756
|
+
* usage endpoint with the keychain token; every other provider its
|
|
1757
|
+
* workers.yaml `usage` block. Any failure — no key, no network, bad JSON —
|
|
1758
|
+
* is "unknown", and unknown never walls a provider.
|
|
1759
|
+
*/
|
|
1760
|
+
async function fetchBudgetReading(name, p) {
|
|
1761
|
+
const now = Date.now();
|
|
1762
|
+
try {
|
|
1763
|
+
if (p.native || name === "anthropic") {
|
|
1764
|
+
const token = await anthropicToken();
|
|
1765
|
+
if (!token) throw new Error("no OAuth token in keychain");
|
|
1766
|
+
return pickBudgetWindow(parseAnthropicUsage(await getJson(new URL(ANTHROPIC_USAGE_URL), {
|
|
1767
|
+
authorization: `Bearer ${token}`,
|
|
1768
|
+
"anthropic-beta": "oauth-2025-04-20"
|
|
1769
|
+
})), now);
|
|
1770
|
+
}
|
|
1771
|
+
if (!p.usage) return {
|
|
1772
|
+
percent: null,
|
|
1773
|
+
resetAt: null,
|
|
1774
|
+
fetchedAt: now
|
|
1775
|
+
};
|
|
1776
|
+
const key = resolveProviderKey(p);
|
|
1777
|
+
const [hn, ...pre] = (p.usage.authHeader ?? "Authorization: Bearer").split(/:\s*/);
|
|
1778
|
+
const prefix = pre.join(": ");
|
|
1779
|
+
return pickBudgetWindow(windowsFromJson(await getJson(new URL(p.usage.url), { [hn.toLowerCase()]: prefix ? `${prefix} ${key}` : String(key) }), p.usage), now);
|
|
1780
|
+
} catch {
|
|
1781
|
+
return {
|
|
1782
|
+
percent: null,
|
|
1783
|
+
resetAt: null,
|
|
1784
|
+
fetchedAt: now
|
|
1785
|
+
};
|
|
1786
|
+
}
|
|
1787
|
+
}
|
|
1788
|
+
/**
|
|
1789
|
+
* Budget readings for spawn-time routing, at most one poll per TTL;
|
|
1790
|
+
* concurrent warmers share the inflight poll. `seed` lets tests (and a
|
|
1791
|
+
* caller with fresh readings of its own) put values in without a poll.
|
|
1792
|
+
*/
|
|
1793
|
+
var BudgetCache = class {
|
|
1794
|
+
value = {};
|
|
1795
|
+
at = 0;
|
|
1796
|
+
inflight;
|
|
1797
|
+
constructor(ttlMs = BUDGET_TTL_MS, fetch = fetchBudgetReading) {
|
|
1798
|
+
this.ttlMs = ttlMs;
|
|
1799
|
+
this.fetch = fetch;
|
|
1800
|
+
}
|
|
1801
|
+
/** Refresh every stale provider of `config`; resolves to the full map. */
|
|
1802
|
+
async warm(config, now = Date.now()) {
|
|
1803
|
+
if (this.at && now - this.at < this.ttlMs) return this.value;
|
|
1804
|
+
this.inflight ??= this.poll(config).then((v) => {
|
|
1805
|
+
this.value = v;
|
|
1806
|
+
this.at = now;
|
|
1807
|
+
this.inflight = void 0;
|
|
1808
|
+
return v;
|
|
1809
|
+
});
|
|
1810
|
+
return this.inflight;
|
|
1811
|
+
}
|
|
1812
|
+
async poll(config) {
|
|
1813
|
+
const out = { ...this.value };
|
|
1814
|
+
const jobs = [this.fetch("anthropic", nativeAnthropicProvider(config.nativeModels)).then((r) => {
|
|
1815
|
+
out.anthropic = r;
|
|
1816
|
+
})];
|
|
1817
|
+
for (const [name, p] of Object.entries(config.providers)) jobs.push(this.fetch(name, p).then((r) => {
|
|
1818
|
+
out[name] = r;
|
|
1819
|
+
}));
|
|
1820
|
+
await Promise.all(jobs);
|
|
1821
|
+
return out;
|
|
1822
|
+
}
|
|
1823
|
+
/** Synchronous read for routing (possibly absent — absence is "unknown"). */
|
|
1824
|
+
get(name) {
|
|
1825
|
+
return this.value[name];
|
|
1826
|
+
}
|
|
1827
|
+
snapshot() {
|
|
1828
|
+
return { ...this.value };
|
|
1829
|
+
}
|
|
1830
|
+
seed(map, at = Date.now()) {
|
|
1831
|
+
this.value = { ...map };
|
|
1832
|
+
this.at = at;
|
|
1833
|
+
}
|
|
1834
|
+
clear() {
|
|
1835
|
+
this.value = {};
|
|
1836
|
+
this.at = 0;
|
|
1837
|
+
this.inflight = void 0;
|
|
1838
|
+
}
|
|
1839
|
+
};
|
|
1840
|
+
/** The process-wide cache routing reads; run.ts warms it before resolving. */
|
|
1841
|
+
const budgetCache = new BudgetCache();
|
|
1842
|
+
/** A provider's wall: routing.budgetWalls[name], else the built-in default for it, else 95. */
|
|
1843
|
+
function budgetWall(routing, name) {
|
|
1844
|
+
return routing.budgetWalls[name] ?? DEFAULT_BUDGET_WALLS[name] ?? DEFAULT_BUDGET_WALL;
|
|
1845
|
+
}
|
|
1846
|
+
/** At or above its wall? Unknown (absent/null percent) is never full. */
|
|
1847
|
+
function budgetExceeded(r, wall) {
|
|
1848
|
+
return !!r && r.percent !== null && r.percent >= wall;
|
|
1849
|
+
}
|
|
1850
|
+
/** Is this reading fresh enough to act on (wall cooldowns) at `now`? */
|
|
1851
|
+
function budgetFresh(r, now, ttlMs = BUDGET_TTL_MS) {
|
|
1852
|
+
return now - r.fetchedAt < ttlMs;
|
|
1853
|
+
}
|
|
1854
|
+
/** "08:02" for a reading's reset, else now + 10 min — the reason line's clock. */
|
|
1855
|
+
function budgetUntilLabel(r, now = /* @__PURE__ */ new Date()) {
|
|
1856
|
+
const t = r.resetAt && r.resetAt > now.getTime() ? r.resetAt : now.getTime() + 10 * 6e4;
|
|
1857
|
+
const d = new Date(t);
|
|
1858
|
+
return `${String(d.getHours()).padStart(2, "0")}:${String(d.getMinutes()).padStart(2, "0")}`;
|
|
1859
|
+
}
|
|
1860
|
+
/**
|
|
1861
|
+
* Why `chosen` is not the walled provider skipped ahead of it:
|
|
1862
|
+
* "anthropic 96% until 08:02" — the one-line switch reason.
|
|
1863
|
+
*/
|
|
1864
|
+
function budgetSkipReason(name, r, now = /* @__PURE__ */ new Date()) {
|
|
1865
|
+
return `${name} ${r.percent}% until ${budgetUntilLabel(r, now)}`;
|
|
1866
|
+
}
|
|
1867
|
+
|
|
1278
1868
|
//#endregion
|
|
1279
1869
|
//#region src/workers/routing.ts
|
|
1280
1870
|
/**
|
|
@@ -1289,11 +1879,24 @@ function pinNotices(launch, claudeJson) {
|
|
|
1289
1879
|
* retryOnQuota is set, the runner restarts the same task on the next provider
|
|
1290
1880
|
* (ledger: WORKER-REROUTE).
|
|
1291
1881
|
*
|
|
1882
|
+
* Policy "budget" (the default) orders candidates by routing.priority —
|
|
1883
|
+
* falling back to routing.order, else DEFAULT_BUDGET_PRIORITY — sorted by
|
|
1884
|
+
* percent ascending within equal priority, and additionally skips every
|
|
1885
|
+
* provider whose live budget reading (budget.ts) reads at or above its wall
|
|
1886
|
+
* (routing.budgetWalls). A walled provider gets a cooldown until
|
|
1887
|
+
* max(now + BUDGET_COOLDOWN_MIN, resetAt), so it stays skipped until its
|
|
1888
|
+
* window resets and re-enters on its own. The chosen target carries a
|
|
1889
|
+
* `switchReason` ("glm: anthropic 96% until 08:02") when budget skipped a
|
|
1890
|
+
* higher-priority provider. A run's provider is resolved once at spawn; a
|
|
1891
|
+
* running worker is never switched mid-flight.
|
|
1892
|
+
*
|
|
1292
1893
|
* An explicit --provider or a class mapping that pins a provider always
|
|
1293
1894
|
* bypasses all of this.
|
|
1294
1895
|
*/
|
|
1295
1896
|
const QUOTA_SKIP_DEFAULT = 95;
|
|
1296
1897
|
const PROBE_TIMEOUT_MS = 1e4;
|
|
1898
|
+
/** Minimum minutes a budget-walled provider stays in cooldown, even when its reset is closer. */
|
|
1899
|
+
const BUDGET_COOLDOWN_MIN = 10;
|
|
1297
1900
|
function readRoutingState(logDir) {
|
|
1298
1901
|
const path = routingStatePath(logDir);
|
|
1299
1902
|
if (!existsSync(path)) return { cooldowns: {} };
|
|
@@ -1329,6 +1932,18 @@ function clearCooldown(logDir, provider) {
|
|
|
1329
1932
|
writeRoutingState(logDir, state);
|
|
1330
1933
|
}
|
|
1331
1934
|
/**
|
|
1935
|
+
* The candidate order for auto-routing: the class's own order, else — policy
|
|
1936
|
+
* "budget" — routing.priority, falling back to routing.order, else
|
|
1937
|
+
* DEFAULT_BUDGET_PRIORITY. Policy "order" uses routing.order as before.
|
|
1938
|
+
*/
|
|
1939
|
+
function autoOrder(config, cls) {
|
|
1940
|
+
if (cls?.order?.length) return cls.order;
|
|
1941
|
+
if (config.routing.policy === "order") return config.routing.order;
|
|
1942
|
+
if (config.routing.priority.length) return config.routing.priority;
|
|
1943
|
+
if (config.routing.order.length) return config.routing.order;
|
|
1944
|
+
return [...DEFAULT_BUDGET_PRIORITY];
|
|
1945
|
+
}
|
|
1946
|
+
/**
|
|
1332
1947
|
* Run a provider's quotaProbe and return its percentage (0–100), or null when
|
|
1333
1948
|
* there is no probe or it printed nothing usable. A probe must never break
|
|
1334
1949
|
* routing: failures read as "unknown", not as "full".
|
|
@@ -1359,6 +1974,28 @@ function quotaExceeded(provider) {
|
|
|
1359
1974
|
const used = probeQuota(provider);
|
|
1360
1975
|
return used !== null && used >= quotaSkipThreshold(provider);
|
|
1361
1976
|
}
|
|
1977
|
+
/**
|
|
1978
|
+
* Put every provider whose fresh budget reading is at or above its wall into
|
|
1979
|
+
* a cooldown until max(now + BUDGET_COOLDOWN_MIN, resetAt), so the walk below
|
|
1980
|
+
* skips it until its window resets and it re-enters on its own. Never
|
|
1981
|
+
* shortens an existing cooldown (a quota-failure cooldown outlasting the
|
|
1982
|
+
* reset stays). Fresh readings only: a stale cache must not re-wall anyone.
|
|
1983
|
+
*/
|
|
1984
|
+
function recordBudgetCooldowns(config, logDir, now = /* @__PURE__ */ new Date()) {
|
|
1985
|
+
if (config.routing.policy === "order") return;
|
|
1986
|
+
const state = readRoutingState(logDir);
|
|
1987
|
+
let dirty = false;
|
|
1988
|
+
for (const [name, r] of Object.entries(budgetCache.snapshot())) {
|
|
1989
|
+
if (r.percent === null || !budgetFresh(r, now.getTime())) continue;
|
|
1990
|
+
if (!budgetExceeded(r, budgetWall(config.routing, name))) continue;
|
|
1991
|
+
const until = Math.max(now.getTime() + BUDGET_COOLDOWN_MIN * 6e4, r.resetAt ?? 0);
|
|
1992
|
+
const prev = state.cooldowns[name];
|
|
1993
|
+
if (prev && Date.parse(prev) >= until) continue;
|
|
1994
|
+
state.cooldowns[name] = new Date(until).toISOString();
|
|
1995
|
+
dirty = true;
|
|
1996
|
+
}
|
|
1997
|
+
if (dirty) writeRoutingState(logDir, state);
|
|
1998
|
+
}
|
|
1362
1999
|
var NoProviderError = class extends WorkersConfigError {};
|
|
1363
2000
|
function mustExist(config, name) {
|
|
1364
2001
|
const p = getProviderOrNative(config, name);
|
|
@@ -1370,12 +2007,16 @@ function mustBeRunnable(name, p) {
|
|
|
1370
2007
|
return p;
|
|
1371
2008
|
}
|
|
1372
2009
|
/** Why one provider of a routing order did not qualify (message fragment). */
|
|
1373
|
-
function exclusionReason(config, state, name, cls) {
|
|
2010
|
+
function exclusionReason(config, state, name, cls, budgets) {
|
|
1374
2011
|
const p = getProviderOrNative(config, name);
|
|
1375
2012
|
if (!p) return "not configured";
|
|
1376
2013
|
if (!p.enabled) return "disabled";
|
|
1377
2014
|
if (cooldownRemaining(state, name) > 0) return "cooldown";
|
|
1378
2015
|
if (quotaExceeded(p)) return "quota";
|
|
2016
|
+
if (budgets) {
|
|
2017
|
+
const r = budgets[name];
|
|
2018
|
+
if (r && r.percent !== null && budgetExceeded(r, budgetWall(config.routing, name))) return `budget ${r.percent}% ≥ wall ${budgetWall(config.routing, name)}`;
|
|
2019
|
+
}
|
|
1379
2020
|
const tier = providerCostTier(p);
|
|
1380
2021
|
if (cls?.maxCostTier !== void 0 && tier > cls.maxCostTier) return `cost tier ${tier} > max ${cls.maxCostTier}`;
|
|
1381
2022
|
if (cls?.requireTags?.length) {
|
|
@@ -1435,13 +2076,22 @@ function resolveTarget(config, logDir, opts = {}) {
|
|
|
1435
2076
|
};
|
|
1436
2077
|
}
|
|
1437
2078
|
const state = readRoutingState(logDir);
|
|
1438
|
-
const
|
|
2079
|
+
const useBudget = config.routing.policy !== "order";
|
|
2080
|
+
const order = autoOrder(config, cls);
|
|
2081
|
+
const budgets = useBudget ? budgetCache.snapshot() : void 0;
|
|
2082
|
+
const candidates = useBudget ? order.map((name, idx) => ({
|
|
2083
|
+
name,
|
|
2084
|
+
idx,
|
|
2085
|
+
percent: budgets?.[name]?.percent
|
|
2086
|
+
})).sort((a, b) => a.idx - b.idx || (a.percent ?? 0) - (b.percent ?? 0)).map((c) => c.name) : order;
|
|
1439
2087
|
const excluded = [];
|
|
1440
|
-
|
|
2088
|
+
let budgetSkipped = null;
|
|
2089
|
+
for (const name of candidates) {
|
|
1441
2090
|
const p = getProviderOrNative(config, name);
|
|
1442
|
-
const why = exclusionReason(config, state, name, cls);
|
|
2091
|
+
const why = exclusionReason(config, state, name, cls, budgets);
|
|
1443
2092
|
if (why) {
|
|
1444
2093
|
excluded.push(`${name}: ${why}`);
|
|
2094
|
+
if (useBudget && budgetSkipped === null && why.startsWith("budget") && p) budgetSkipped = budgetSkipReason(name, budgets[name]);
|
|
1445
2095
|
continue;
|
|
1446
2096
|
}
|
|
1447
2097
|
return {
|
|
@@ -1449,18 +2099,23 @@ function resolveTarget(config, logDir, opts = {}) {
|
|
|
1449
2099
|
provider: p,
|
|
1450
2100
|
modelAlias: null,
|
|
1451
2101
|
classMcp: cls?.mcp ?? null,
|
|
1452
|
-
via: "auto"
|
|
2102
|
+
via: "auto",
|
|
2103
|
+
...budgetSkipped ? { switchReason: `${name}: ${budgetSkipped}` } : {}
|
|
1453
2104
|
};
|
|
1454
2105
|
}
|
|
1455
2106
|
throw new NoProviderError(`auto-routing found no usable provider${opts.className ? ` for class "${opts.className}"` : ""} (order: [${order.join(", ")}]).${excluded.length ? `\nExcluded: ${excluded.join("; ")}.` : ""}\nClear a cooldown with: pai worker providers enable <name>; widen the class with: pai worker classes set ${opts.className ?? "<class>"}=<target>`);
|
|
1456
2107
|
}
|
|
1457
2108
|
/**
|
|
1458
|
-
* Next provider after `from` in auto order, applying the same filters
|
|
1459
|
-
*
|
|
2109
|
+
* Next provider after `from` in auto order, applying the same filters —
|
|
2110
|
+
* including the budget wall under policy "budget", so a reroute never moves
|
|
2111
|
+
* a run onto a provider with no budget left. Used by rerouting; null when
|
|
2112
|
+
* the order is exhausted.
|
|
1460
2113
|
*/
|
|
1461
2114
|
function nextAutoProvider(config, logDir, from, now = /* @__PURE__ */ new Date()) {
|
|
1462
2115
|
const state = readRoutingState(logDir);
|
|
1463
|
-
const
|
|
2116
|
+
const useBudget = config.routing.policy !== "order";
|
|
2117
|
+
const order = autoOrder(config);
|
|
2118
|
+
const budgets = useBudget ? budgetCache.snapshot() : void 0;
|
|
1464
2119
|
const start = order.indexOf(from);
|
|
1465
2120
|
for (let i = start + 1; i < order.length; i++) {
|
|
1466
2121
|
const name = order[i];
|
|
@@ -1469,6 +2124,10 @@ function nextAutoProvider(config, logDir, from, now = /* @__PURE__ */ new Date()
|
|
|
1469
2124
|
const end = state.cooldowns[name];
|
|
1470
2125
|
if (end && Date.parse(end) > now.getTime()) continue;
|
|
1471
2126
|
if (quotaExceeded(p)) continue;
|
|
2127
|
+
if (budgets) {
|
|
2128
|
+
const r = budgets[name];
|
|
2129
|
+
if (r && budgetExceeded(r, budgetWall(config.routing, name))) continue;
|
|
2130
|
+
}
|
|
1472
2131
|
return name;
|
|
1473
2132
|
}
|
|
1474
2133
|
return null;
|
|
@@ -4459,7 +5118,7 @@ async function runWorker(opts) {
|
|
|
4459
5118
|
if (gc?.length) process.stderr.write(`pai worker: archived ${gc.length} leftover worktree(s) (pai worker gc)\n`);
|
|
4460
5119
|
} catch {}
|
|
4461
5120
|
if (opts.className === "plan" && !opts._planner) {
|
|
4462
|
-
const { runPlanner } = await import("./planner-
|
|
5121
|
+
const { runPlanner } = await import("./planner-ZKXZf9JS.mjs");
|
|
4463
5122
|
return runPlanner(opts);
|
|
4464
5123
|
}
|
|
4465
5124
|
const parent = launchParent(opts.parent);
|
|
@@ -4478,6 +5137,10 @@ async function runWorker(opts) {
|
|
|
4478
5137
|
providerFlag = routed.providerFlag ?? void 0;
|
|
4479
5138
|
}
|
|
4480
5139
|
}
|
|
5140
|
+
if (!providerFlag && !capabilityResolution && config.routing.policy !== "order") try {
|
|
5141
|
+
await budgetCache.warm(config);
|
|
5142
|
+
recordBudgetCooldowns(config, logDir);
|
|
5143
|
+
} catch {}
|
|
4481
5144
|
const target = capabilityResolution ? {
|
|
4482
5145
|
providerName: capabilityResolution.providerName,
|
|
4483
5146
|
provider: capabilityResolution.provider,
|
|
@@ -5203,6 +5866,7 @@ async function executeRun(a) {
|
|
|
5203
5866
|
origin: headless ? "spawn" : "chat",
|
|
5204
5867
|
outputFormat: parsed.outputFormat,
|
|
5205
5868
|
...a.route ? { route: a.route } : {},
|
|
5869
|
+
...target.via === "auto" && target.switchReason ? { switchReason: target.switchReason } : {},
|
|
5206
5870
|
...a.fallThroughOf ? { fallThroughOf: a.fallThroughOf } : {},
|
|
5207
5871
|
...headless ? { reportFormat: a.reportFormat } : {},
|
|
5208
5872
|
...trailerApplied ? { promptTrailer: true } : {},
|
|
@@ -5245,7 +5909,8 @@ async function executeRun(a) {
|
|
|
5245
5909
|
cwd,
|
|
5246
5910
|
label,
|
|
5247
5911
|
...a.capability ? { capability: a.capability } : {},
|
|
5248
|
-
...a.specPath ? { spec: a.specPath } : {}
|
|
5912
|
+
...a.specPath ? { spec: a.specPath } : {},
|
|
5913
|
+
...target.switchReason ? { switch: target.switchReason } : {}
|
|
5249
5914
|
});
|
|
5250
5915
|
if (paneVerified) status.pane = { opened: true };
|
|
5251
5916
|
else if (headless) announcePane(logDir, config, wid, term, noPane, ledger, status);
|
|
@@ -5599,7 +6264,7 @@ async function executeRun(a) {
|
|
|
5599
6264
|
if (!ok && headless && a.target.via === "auto" && config.routing.retryOnQuota && status.turns <= 1 && status.tools === 0 && isQuotaFailure(resultText)) {
|
|
5600
6265
|
setCooldown(logDir, target.providerName, config.routing.cooldownMinutes);
|
|
5601
6266
|
const next = nextAutoProvider(config, logDir, target.providerName);
|
|
5602
|
-
if (next && a.reroutes < config.
|
|
6267
|
+
if (next && a.reroutes < autoOrder(config).length) {
|
|
5603
6268
|
appendLedger(ledger, "WORKER-REROUTE", {
|
|
5604
6269
|
from: target.providerName,
|
|
5605
6270
|
to: next,
|
|
@@ -6026,5 +6691,5 @@ function resultFromOutput(out) {
|
|
|
6026
6691
|
}
|
|
6027
6692
|
|
|
6028
6693
|
//#endregion
|
|
6029
|
-
export { ensureProxyRunning as $, routeRun as A, detectCurrentProviderName as B, workerDepth as C,
|
|
6030
|
-
//# sourceMappingURL=run-
|
|
6694
|
+
export { ensureProxyRunning as $, routeRun as A, detectCurrentProviderName as B, workerDepth as C, sessionTag as Ct, onBattery as D, findLocal as E, endLabel as F, detectToolErrors as G, maskLaunchEnv as H, requestKill as I, parseSince as J, errorsTable as K, buildLaunchPlan as L, clickrControlsArgv as M, markControlsHeld as N, parseDuration as O, callerDescription as P, DEFAULT_PROXY_PORT as Q, buildLaunchRows as R, sayToWorker as S, sessionMapPath as St, clearPolicy as T, SAME_WORKER_WINDOW_MS as U, launchableProviderNames as V, detectRunningWorkerErrors as W, toolErrorSigTag as X, readToolErrors as Y, toolErrorText as Z, waitOnCallLine as _, currentTabKey as _t, deadlineLabel as a, LimitsCache as at, handoffFromInside as b, ownerSessionName as bt, parseDenialText as c, assertPaneNotDisabled as ct, realDeps as d, closeOwnPane as dt, stopProxy as et, statusMarker as f, headlessAllowed as ft, realWaitDeps as g, resolveHostSession as gt, parseWaitOn as h, paneBackend as ht, usageContextTokens as i, resolveTarget as it, writePolicy as j, readPolicy as k, parseWorkerReport as l, checkPaneForWorker as lt, detachedLine as m, openPaneForWorker as mt, runWorker as n, probeQuota as nt, denialLines as o, pinNotices as ot, waitOn as p, openFollowPane as pt, groupToolErrors as q, testProvider as r, quotaSkipThreshold as rt, denialMark as s, projectLaunchConfig as st, printResult as t, clearCooldown as tt, renderReport as u, closeEndedPanes as ut, waitOnResultLine as v, itermForClaudeSession as vt, POLICY_MODES as w, workerInScope as wt, readInbox as x, resolveSession as xt, waitingLine as y, itermUuid as yt, detectCurrentModel as z };
|
|
6695
|
+
//# sourceMappingURL=run-CsrBbtj_.mjs.map
|