@drakon-systems/multi-clawd 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -4
- package/dist/direct-report.js +20 -7
- package/dist/direct-route.js +13 -7
- package/dist/direct-sync.js +22 -11
- package/dist/explain-core.js +48 -7
- package/dist/health.js +23 -3
- package/dist/index.js +68 -6
- package/dist/session-affinity.js +108 -0
- package/dist/shim.js +48 -6
- package/dist/sticky.js +11 -1
- package/openclaw.plugin.json +21 -2
- package/package.json +1 -1
- package/scripts/account-config.mjs +115 -0
- package/scripts/cli.mjs +42 -6
- package/scripts/doctor.mjs +20 -8
package/README.md
CHANGED
|
@@ -30,6 +30,7 @@ multi-clawd update # install (or update) the OpenClaw plugin — right flags,
|
|
|
30
30
|
multi-clawd setup # guided wizard: accounts, isolated second login, pool, watchdog
|
|
31
31
|
multi-clawd login claw2 # launch the right Claude sign-in for an account (or re-auth it)
|
|
32
32
|
multi-clawd explain # your whole setup in plain English — accounts, chain, live health
|
|
33
|
+
multi-clawd account disable claw2 --dry-run # preview benching an account
|
|
33
34
|
multi-clawd chain # audit your model routing — what actually serves each turn
|
|
34
35
|
multi-clawd direct # the direct anthropic/* route: status, or `direct sync` (v1.9, optional)
|
|
35
36
|
multi-clawd doctor # health check (add --probe for a live end-to-end turn)
|
|
@@ -386,7 +387,7 @@ openclaw plugins install (Get-Location).Path
|
|
|
386
387
|
**Or let your agent install it.** Running an OpenClaw assistant or Claude
|
|
387
388
|
Code on the target machine already? Paste it this and go make coffee:
|
|
388
389
|
|
|
389
|
-
> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v2.
|
|
390
|
+
> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v2.4.0/SETUP-AGENT.md
|
|
390
391
|
> and follow it to set up multi-clawd on this machine. I own a second
|
|
391
392
|
> Claude account — ask me when you need me to log in.
|
|
392
393
|
|
|
@@ -546,6 +547,7 @@ error.
|
|
|
546
547
|
"accounts": [
|
|
547
548
|
{ "id": "claw1", "label": "Main Claude", "native": true },
|
|
548
549
|
{ "id": "claw2", "label": "Second Max", "configDir": "~/.claw2",
|
|
550
|
+
// "disabled": true, // optional; default false: bench without removing the pool member
|
|
549
551
|
// v0.3 preferred: resolve via your gateway's secret providers —
|
|
550
552
|
// no plaintext token file on disk
|
|
551
553
|
"oauthTokenRef": { "source": "exec", "provider": "onepassword",
|
|
@@ -553,10 +555,13 @@ error.
|
|
|
553
555
|
],
|
|
554
556
|
"pool": {
|
|
555
557
|
"id": "clawd",
|
|
556
|
-
"accounts": ["claw1", "claw2"], // preference order; first = home
|
|
558
|
+
"accounts": ["claw1", "claw2"], // preference order; first = home for ordered strategy
|
|
559
|
+
"strategy": "ordered", // optional: "reset-aware" drains healthy weekly quota that resets soonest
|
|
557
560
|
"utilizationThreshold": 0.85, // default hand-over; minute/hour and overage windows use this
|
|
558
561
|
"weeklyUtilizationThreshold": 0.97, // optional: day/week/month quota windows at 97%
|
|
559
562
|
"minDwellMs": 600000, // v0.3: anti-flap dwell before returning home
|
|
563
|
+
"sessionAffinity": true, // optional; default false: keep healthy CLI sessions on their serving account
|
|
564
|
+
"sessionAffinityTtlMs": 2592000000, // optional; default 30 days
|
|
560
565
|
"degrade": { // v0.3.5: last step before provider drop
|
|
561
566
|
"ladder": ["claude-opus-4-8"], // whole pool exhausted → same account, lower tier
|
|
562
567
|
"pins": [ // contractual lanes never degrade
|
|
@@ -577,6 +582,48 @@ error.
|
|
|
577
582
|
} }
|
|
578
583
|
```
|
|
579
584
|
|
|
585
|
+
Set `accounts[].disabled: true` to bench a member while keeping it in
|
|
586
|
+
`pool.accounts` and visible in `explain`, `doctor`, and `usage` as "disabled by
|
|
587
|
+
operator". It is excluded from pool selection, in-turn retries, session
|
|
588
|
+
affinity, and the managed direct Anthropic order. Transcript lookup still
|
|
589
|
+
searches its session directory. To edit `~/.openclaw/openclaw.json` with a
|
|
590
|
+
timestamped backup and without reformatting unrelated config, run
|
|
591
|
+
`multi-clawd account disable <id>` or `multi-clawd account enable <id>`.
|
|
592
|
+
Add `--dry-run` to preview without writing anything. If every pool member is
|
|
593
|
+
disabled, pooled launches fail closed with an error naming
|
|
594
|
+
`pool.accounts[].disabled`; they never fall back to a disabled member.
|
|
595
|
+
On the direct `anthropic/*` route, a disabled member is left out of the order
|
|
596
|
+
multi-clawd writes. If every direct member is disabled, multi-clawd stops
|
|
597
|
+
managing that order and logs a warning naming `pool.accounts[].disabled`, but
|
|
598
|
+
it cannot empty an order OpenClaw has already stored, so clear or edit that
|
|
599
|
+
stored order yourself if the direct route must not use those profiles.
|
|
600
|
+
Changes take effect when the plugin next reads its config, for example after
|
|
601
|
+
a gateway reload. A turn already admitted or running continues on its selected
|
|
602
|
+
account; the command does not interrupt it.
|
|
603
|
+
|
|
604
|
+
`pool.strategy` defaults to `"ordered"`, which retains the configured account
|
|
605
|
+
order. Set it to `"reset-aware"` to use healthy weekly quota before its
|
|
606
|
+
`seven_day` window resets: among `ok` accounts with live seven-day reset data,
|
|
607
|
+
the soonest reset wins. Accounts without that data follow in pool order;
|
|
608
|
+
near-limit or exhausted accounts never outrank a healthy one. Equal reset
|
|
609
|
+
times keep pool order. The existing `minDwellMs` delay still holds an account
|
|
610
|
+
that remains usable. Both CLI launches and the direct Anthropic route use the
|
|
611
|
+
same preference. Other values are rejected by the plugin config schema.
|
|
612
|
+
|
|
613
|
+
`pool.sessionAffinity: true` remembers the account that successfully served each
|
|
614
|
+
Claude CLI session (`--session-id` or `--resume`) and prefers it on later turns
|
|
615
|
+
while its health is `ok` or `no_data`. A near-limit or otherwise unhealthy
|
|
616
|
+
account loses its pin; the successful replacement becomes the new pin. New
|
|
617
|
+
sessions and launches without a session id keep normal pool selection. Pins
|
|
618
|
+
live in `~/.openclaw/state/multi-clawd/pool-<id>.sessions.json`, expire after
|
|
619
|
+
30 days by default (`sessionAffinityTtlMs`), and are capped at 1000 sessions.
|
|
620
|
+
This applies to shimmed CLI launches; direct Anthropic and side-question
|
|
621
|
+
launches keep their existing routing. The shim can switch to native,
|
|
622
|
+
config-dir, and token-file accounts. On verified OpenClaw launches, the
|
|
623
|
+
gateway also resolves a pinned secret-reference account at execution time.
|
|
624
|
+
Legacy shim launches fall back to normal selection when a secret-reference
|
|
625
|
+
pin needs a different account.
|
|
626
|
+
|
|
580
627
|
How it decides, per launch (all data from each account's live
|
|
581
628
|
`rate_limit_event` stream, captured by the shim):
|
|
582
629
|
|
|
@@ -868,7 +915,7 @@ callsite (the ones static scanners flag) and why each exists.
|
|
|
868
915
|
| Network | The gateway plugin polls `https://api.anthropic.com/api/oauth/usage` for eligible pooled accounts on an interval (default 120000 ms / 2 minutes); `pool.usagePoll.enabled: false` disables that poll. `multi-clawd usage` requests the same endpoint on demand. `doctor` runs `npm view` when its registry cache is stale; install and update commands contact the package registry. Claude Code subprocesses make their own provider requests. No plugin telemetry or analytics endpoint. |
|
|
869
916
|
| Credentials | Reads declared setup-token files or gateway-resolved `oauthTokenRef` values for Claude child launches; reads each eligible account's `.credentials.json` access token for the usage poll; reads Claude login credentials from files or the macOS Keychain via the `security` CLI for login checks and runtime auth; reads account identity for file-backed logins from `.claude.json` (`~/.claude.json` for a native login, or the account config dir). Setup tokens are passed to Claude through the child environment. |
|
|
870
917
|
| Files read | `~/.openclaw/openclaw.json`, declared account config dirs and token files, Claude `.credentials.json` and `.claude.json` files as described above, and local state files. |
|
|
871
|
-
| Files written | Under `~/.openclaw/state/multi-clawd/` by default: `<id>.json`, `<id>.usage.json`, `pool-<id>.sticky.json`, `direct-order.sticky.json`, `known-models.json`, `alerts-spool.jsonl`, `watchdog.json`, `registry-check.json`, `auth-profiles-<agent>.json`, `runtime/<hash>/` (copied runtime files), and `watchdog-launcher.mjs`. The watchdog state file can be redirected with `MULTI_CLAWD_WATCHDOG_STATE`. Setup writes `~/.openclaw/openclaw.json` and a dated backup. On session handover, transcript `.jsonl` files are copied between account config dirs. The optional watchdog installs `~/.config/systemd/user/multi-clawd-eviction-watchdog.service` and `.timer` on Linux, or `~/Library/LaunchAgents/com.multi-clawd.eviction-watchdog.plist` on macOS; the unit points at the state-dir launcher. |
|
|
918
|
+
| Files written | Under `~/.openclaw/state/multi-clawd/` by default: `<id>.json`, `<id>.usage.json`, `pool-<id>.sticky.json`, `pool-<id>.sessions.json` (when affinity is enabled), `direct-order.sticky.json`, `known-models.json`, `alerts-spool.jsonl`, `watchdog.json`, `registry-check.json`, `auth-profiles-<agent>.json`, `runtime/<hash>/` (copied runtime files), and `watchdog-launcher.mjs`. The watchdog state file can be redirected with `MULTI_CLAWD_WATCHDOG_STATE`. Setup writes `~/.openclaw/openclaw.json` and a dated backup. On session handover, transcript `.jsonl` files are copied between account config dirs. The optional watchdog installs `~/.config/systemd/user/multi-clawd-eviction-watchdog.service` and `.timer` on Linux, or `~/Library/LaunchAgents/com.multi-clawd.eviction-watchdog.plist` on macOS; the unit points at the state-dir launcher. |
|
|
872
919
|
| Processes | Launches Claude Code and the node + shim transport; CLI, setup and doctor run npm, OpenClaw, Claude and OS scheduler commands; the optional watchdog checks OpenClaw logs and may restart the gateway; Hermes integration runs its Python bridge. See [SECURITY.md](SECURITY.md) for the process-call table. |
|
|
873
920
|
| Consent | The wizard asks before every write, merges non-destructively, and never overwrites an existing account entry. `--dry-run` previews everything. |
|
|
874
921
|
|
|
@@ -967,8 +1014,10 @@ Early but real — built for and dogfooded in production.
|
|
|
967
1014
|
- **v1.9** — one pool, both transports: accounts can also serve OpenClaw's
|
|
968
1015
|
direct `anthropic/*` route; the gateway keeps the `anthropic` auth order in
|
|
969
1016
|
pool-health order; `multi-clawd direct` / `direct sync` ✅
|
|
1017
|
+
- **Unreleased** — optional per-session affinity for shimmed Claude CLI
|
|
1018
|
+
launches; health overrides the pin, with bounded expiring state ✅
|
|
970
1019
|
- **Next** — standalone localhost proxy (OpenAI-compatible) so Hermes and
|
|
971
|
-
custom runtimes can share the pool;
|
|
1020
|
+
custom runtimes can share the pool; local
|
|
972
1021
|
five-hour-window signal (turn counting); per-account lock for the shim
|
|
973
1022
|
persistence race
|
|
974
1023
|
|
package/dist/direct-report.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import { classifyAccountHealth } from "./health.js";
|
|
1
|
+
import { classifyAccountHealth, sevenDayReset } from "./health.js";
|
|
2
2
|
import { DIRECT_PROVIDER, collectDirectMembers, } from "./direct-route.js";
|
|
3
3
|
import { parseUnusableProfiles, planAgentOrder, readDirectRouteSnapshot, safeCliError, } from "./direct-sync.js";
|
|
4
4
|
export function describeDirectSource(member) {
|
|
5
|
+
if (member.disabled)
|
|
6
|
+
return "disabled by operator";
|
|
5
7
|
const s = member.source;
|
|
6
8
|
switch (s.kind) {
|
|
7
9
|
case "ref":
|
|
@@ -19,11 +21,18 @@ export async function gatherDirectStatus(params) {
|
|
|
19
21
|
if (members.length === 0 && problems.length === 0)
|
|
20
22
|
return undefined;
|
|
21
23
|
const errors = [];
|
|
22
|
-
const verdicts = members.map((m) =>
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
24
|
+
const verdicts = members.map((m) => {
|
|
25
|
+
const state = params.readHealth(m.accountId);
|
|
26
|
+
return {
|
|
27
|
+
accountId: m.accountId,
|
|
28
|
+
profileId: m.profileId,
|
|
29
|
+
disabled: m.disabled,
|
|
30
|
+
verdict: classifyAccountHealth(state, params.healthOptions, params.nowMs).verdict,
|
|
31
|
+
sevenDayResetsAt: params.strategy === "reset-aware"
|
|
32
|
+
? sevenDayReset(state, params.healthOptions, params.nowMs)
|
|
33
|
+
: undefined,
|
|
34
|
+
};
|
|
35
|
+
});
|
|
27
36
|
const read = members.length > 0 ? await readDirectRouteSnapshot(params.runner, params.agentId) : {};
|
|
28
37
|
if ("error" in read && read.error)
|
|
29
38
|
errors.push(read.error);
|
|
@@ -38,7 +47,7 @@ export async function gatherDirectStatus(params) {
|
|
|
38
47
|
errors.push(`models status ${safeCliError(status)}`);
|
|
39
48
|
}
|
|
40
49
|
let pendingOrder;
|
|
41
|
-
if (snapshot) {
|
|
50
|
+
if (snapshot && members.some((m) => !m.disabled)) {
|
|
42
51
|
pendingOrder = planAgentOrder({
|
|
43
52
|
members: verdicts,
|
|
44
53
|
snapshot,
|
|
@@ -46,8 +55,12 @@ export async function gatherDirectStatus(params) {
|
|
|
46
55
|
sticky: params.sticky,
|
|
47
56
|
nowMs: params.nowMs,
|
|
48
57
|
minDwellMs: params.minDwellMs,
|
|
58
|
+
strategy: params.strategy,
|
|
49
59
|
}).order;
|
|
50
60
|
}
|
|
61
|
+
else if (members.length > 0 && members.every((m) => m.disabled)) {
|
|
62
|
+
errors.push("pool.accounts[].disabled: every direct member is disabled");
|
|
63
|
+
}
|
|
51
64
|
const byProfile = new Map(unusable.map((u) => [u.profileId, u]));
|
|
52
65
|
const order = snapshot?.storedOrder ?? (params.configOrder?.length ? [...params.configOrder] : snapshot ? [] : undefined);
|
|
53
66
|
const orderSource = snapshot?.storedOrder
|
package/dist/direct-route.js
CHANGED
|
@@ -115,7 +115,7 @@ export function collectDirectMembers(accounts, poolOrder = []) {
|
|
|
115
115
|
continue;
|
|
116
116
|
}
|
|
117
117
|
seenProfiles.add(profileId);
|
|
118
|
-
members.push({ accountId: account.id, profileId, source });
|
|
118
|
+
members.push({ accountId: account.id, profileId, source, disabled: account.disabled === true });
|
|
119
119
|
}
|
|
120
120
|
return { members, problems };
|
|
121
121
|
}
|
|
@@ -128,22 +128,28 @@ const VERDICT_RANK = {
|
|
|
128
128
|
credential_failed: 4,
|
|
129
129
|
};
|
|
130
130
|
export function planDirectOrder(params) {
|
|
131
|
-
const
|
|
132
|
-
if (members.length === 0)
|
|
131
|
+
const members = params.members.filter((m) => !m.disabled);
|
|
132
|
+
if (params.members.length === 0)
|
|
133
133
|
return undefined;
|
|
134
|
+
if (members.length === 0)
|
|
135
|
+
throw new Error("pool.accounts[].disabled: every direct member is disabled");
|
|
134
136
|
const decision = decideStickySelection({
|
|
135
|
-
verdicts: members.map((m) => ({ id: m.accountId, verdict: m.verdict })),
|
|
137
|
+
verdicts: members.map((m) => ({ id: m.accountId, verdict: m.verdict, sevenDayResetsAt: m.sevenDayResetsAt })),
|
|
136
138
|
sticky: params.sticky,
|
|
137
139
|
nowMs: params.nowMs,
|
|
138
140
|
minDwellMs: params.minDwellMs,
|
|
141
|
+
strategy: params.strategy,
|
|
139
142
|
});
|
|
140
143
|
const first = members.find((m) => m.accountId === decision.account) ?? members[0];
|
|
141
144
|
const rest = members
|
|
142
145
|
.filter((m) => m !== first)
|
|
143
146
|
.map((m, index) => ({ m, index }))
|
|
144
|
-
.sort((a, b) => VERDICT_RANK[a.m.verdict] - VERDICT_RANK[b.m.verdict] ||
|
|
147
|
+
.sort((a, b) => VERDICT_RANK[a.m.verdict] - VERDICT_RANK[b.m.verdict] ||
|
|
148
|
+
(params.strategy === "reset-aware" && a.m.verdict === "ok"
|
|
149
|
+
? (a.m.sevenDayResetsAt ?? Infinity) - (b.m.sevenDayResetsAt ?? Infinity)
|
|
150
|
+
: 0) || a.index - b.index)
|
|
145
151
|
.map(({ m }) => m.profileId);
|
|
146
|
-
const managed = new Set(members.map((m) => m.profileId));
|
|
152
|
+
const managed = new Set(params.members.map((m) => m.profileId));
|
|
147
153
|
const unmanaged = (params.currentOrder ?? []).filter((id, i, all) => typeof id === "string" && !managed.has(id) && all.indexOf(id) === i);
|
|
148
154
|
const order = [first.profileId, ...rest, ...unmanaged];
|
|
149
155
|
const current = params.currentOrder ?? [];
|
|
@@ -153,5 +159,5 @@ export function planDirectOrder(params) {
|
|
|
153
159
|
export function directRoutePools(accounts, directRoute) {
|
|
154
160
|
if (directRoute?.manageOrder === false)
|
|
155
161
|
return false;
|
|
156
|
-
return collectDirectMembers(accounts).members.length >= 2;
|
|
162
|
+
return collectDirectMembers(accounts).members.filter((m) => !m.disabled).length >= 2;
|
|
157
163
|
}
|
package/dist/direct-sync.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { classifyAccountHealth } from "./health.js";
|
|
1
|
+
import { classifyAccountHealth, sevenDayReset } from "./health.js";
|
|
2
2
|
import { DIRECT_PROVIDER, planDirectOrder } from "./direct-route.js";
|
|
3
3
|
import { parseClaudeSetupToken } from "./hermes-core.js";
|
|
4
4
|
export function safeCliError(result) {
|
|
@@ -130,6 +130,8 @@ export async function syncDirectProfiles(params) {
|
|
|
130
130
|
const stored = new Set(listed.map((p) => p.id));
|
|
131
131
|
const results = [];
|
|
132
132
|
for (const member of params.members) {
|
|
133
|
+
if (member.disabled)
|
|
134
|
+
continue;
|
|
133
135
|
const base = { accountId: member.accountId, profileId: member.profileId };
|
|
134
136
|
const { source } = member;
|
|
135
137
|
if (source.kind === "existing") {
|
|
@@ -212,18 +214,19 @@ function sameOrder(a, b) {
|
|
|
212
214
|
}
|
|
213
215
|
export function planAgentOrder(params) {
|
|
214
216
|
const { snapshot } = params;
|
|
215
|
-
const present = params.members.filter((m) => snapshot.stored.has(m.profileId));
|
|
216
|
-
const missing = params.members.filter((m) => !snapshot.stored.has(m.profileId)).map((m) => m.profileId);
|
|
217
|
-
if (present.length === 0)
|
|
217
|
+
const present = params.members.filter((m) => !m.disabled && snapshot.stored.has(m.profileId));
|
|
218
|
+
const missing = params.members.filter((m) => !m.disabled && !snapshot.stored.has(m.profileId)).map((m) => m.profileId);
|
|
219
|
+
if (present.length === 0 && params.members.some((m) => !m.disabled))
|
|
218
220
|
return { missing };
|
|
219
221
|
const explicit = snapshot.storedOrder ?? (params.configOrder?.length ? [...params.configOrder] : null);
|
|
220
222
|
const base = (explicit ?? [...snapshot.stored]).filter((id) => snapshot.stored.has(id));
|
|
221
223
|
const plan = planDirectOrder({
|
|
222
|
-
members:
|
|
224
|
+
members: params.members.map((m) => ({ ...m, disabled: m.disabled || !snapshot.stored.has(m.profileId) })),
|
|
223
225
|
currentOrder: base,
|
|
224
226
|
sticky: params.sticky,
|
|
225
227
|
nowMs: params.nowMs,
|
|
226
228
|
minDwellMs: params.minDwellMs,
|
|
229
|
+
strategy: params.strategy,
|
|
227
230
|
});
|
|
228
231
|
if (!plan)
|
|
229
232
|
return { missing };
|
|
@@ -244,13 +247,20 @@ export function createDirectOrderController(deps) {
|
|
|
244
247
|
return { managedOrder: [], agents: [] };
|
|
245
248
|
running = true;
|
|
246
249
|
try {
|
|
247
|
-
const verdicts = deps.members.map((m) =>
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
250
|
+
const verdicts = deps.members.map((m) => {
|
|
251
|
+
const state = deps.readHealth(m.accountId);
|
|
252
|
+
return {
|
|
253
|
+
accountId: m.accountId,
|
|
254
|
+
profileId: m.profileId,
|
|
255
|
+
disabled: m.disabled,
|
|
256
|
+
verdict: classifyAccountHealth(state, deps.healthOptions, nowMs).verdict,
|
|
257
|
+
sevenDayResetsAt: deps.strategy === "reset-aware"
|
|
258
|
+
? sevenDayReset(state, deps.healthOptions, nowMs)
|
|
259
|
+
: undefined,
|
|
260
|
+
};
|
|
261
|
+
});
|
|
252
262
|
const sticky = deps.readSticky();
|
|
253
|
-
const managed = planDirectOrder({ members: verdicts, sticky, nowMs, minDwellMs: deps.minDwellMs });
|
|
263
|
+
const managed = planDirectOrder({ members: verdicts, sticky, nowMs, minDwellMs: deps.minDwellMs, strategy: deps.strategy });
|
|
254
264
|
if (!managed)
|
|
255
265
|
return { managedOrder: [], agents: [] };
|
|
256
266
|
if (JSON.stringify(managed.sticky) !== JSON.stringify(sticky))
|
|
@@ -285,6 +295,7 @@ export function createDirectOrderController(deps) {
|
|
|
285
295
|
sticky,
|
|
286
296
|
nowMs,
|
|
287
297
|
minDwellMs: deps.minDwellMs,
|
|
298
|
+
strategy: deps.strategy,
|
|
288
299
|
});
|
|
289
300
|
for (const id of plan.missing) {
|
|
290
301
|
const k = `${agentId}:${id}`;
|
package/dist/explain-core.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { DEFAULT_UTILIZATION_THRESHOLD } from "./health.js";
|
|
1
|
+
import { DEFAULT_UTILIZATION_THRESHOLD, choosePoolAccount, poolStrategy } from "./health.js";
|
|
2
|
+
import { decideStickySelection } from "./sticky.js";
|
|
2
3
|
const WINDOW_LABELS = {
|
|
3
4
|
five_hour: "5-hour",
|
|
4
5
|
seven_day: "weekly",
|
|
@@ -80,7 +81,7 @@ export function renderExplanation(model) {
|
|
|
80
81
|
const lines = [];
|
|
81
82
|
lines.push("ACCOUNTS");
|
|
82
83
|
for (const acc of model.accounts) {
|
|
83
|
-
lines.push(` ${acc.id}${acc.label ? ` "${acc.label}"` : ""}`);
|
|
84
|
+
lines.push(` ${acc.id}${acc.label ? ` "${acc.label}"` : ""}${acc.disabled ? " — disabled by operator" : ""}`);
|
|
84
85
|
lines.push(` → ${describeAccount(acc)}`);
|
|
85
86
|
}
|
|
86
87
|
lines.push("");
|
|
@@ -88,9 +89,15 @@ export function renderExplanation(model) {
|
|
|
88
89
|
const generalThreshold = model.pool.utilizationThreshold ?? DEFAULT_UTILIZATION_THRESHOLD;
|
|
89
90
|
const weeklyThreshold = model.pool.weeklyUtilizationThreshold ?? generalThreshold;
|
|
90
91
|
lines.push(`POOL ${model.pool.id} (${model.pool.accounts.join(" → ")})`);
|
|
91
|
-
lines.push(`
|
|
92
|
-
|
|
93
|
-
|
|
92
|
+
lines.push(` Strategy: ${poolStrategy(model.pool.strategy)}.`);
|
|
93
|
+
if (poolStrategy(model.pool.strategy) === "reset-aware") {
|
|
94
|
+
lines.push(" Healthy accounts with a live seven-day reset are preferred by soonest reset.");
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
lines.push(" Every Claude launch runs on the first account below its threshold;");
|
|
98
|
+
lines.push(" the home account reclaims work");
|
|
99
|
+
lines.push(" automatically once its window resets.");
|
|
100
|
+
}
|
|
94
101
|
lines.push(` Thresholds: general ${Math.round(generalThreshold * 100)}%, weekly ${Math.round(weeklyThreshold * 100)}%${model.pool.weeklyUtilizationThreshold === undefined ? " (follows general)" : ""}.`);
|
|
95
102
|
lines.push(` The weekly threshold applies to day/week/month quota windows; overage uses the general threshold.`);
|
|
96
103
|
const ladder = model.pool.degrade?.ladder ?? [];
|
|
@@ -126,12 +133,46 @@ export function renderExplanation(model) {
|
|
|
126
133
|
lines.push("RIGHT NOW");
|
|
127
134
|
for (const h of model.health) {
|
|
128
135
|
const word = VERDICT_WORDS[h.verdict] ?? h.verdict;
|
|
129
|
-
lines.push(` ${h.id}: ${word}${h.detail ? ` — ${h.detail}` : ""}`);
|
|
136
|
+
lines.push(` ${h.id}: ${model.accounts.find((a) => a.id === h.id)?.disabled ? "disabled by operator" : word}${h.detail ? ` — ${h.detail}` : ""}`);
|
|
130
137
|
lines.push(` usage: ${h.usage?.length ? renderUsageLine(h.usage, nowMs) : "no live telemetry"}`);
|
|
131
138
|
}
|
|
132
139
|
if (model.pool) {
|
|
140
|
+
const strategy = poolStrategy(model.pool.strategy);
|
|
141
|
+
const verdicts = model.pool.accounts.filter((id) => !model.accounts.find((a) => a.id === id)?.disabled).map((id) => {
|
|
142
|
+
const h = model.health.find((row) => row.id === id);
|
|
143
|
+
return {
|
|
144
|
+
id,
|
|
145
|
+
verdict: h?.verdict ?? "no_data",
|
|
146
|
+
sevenDayResetsAt: (h?.usage?.find((u) => u.window === "usage:seven_day" && u.resetsAt !== undefined && u.resetsAt > nowMs)
|
|
147
|
+
?? h?.usage?.find((u) => u.window === "seven_day" && u.resetsAt !== undefined && u.resetsAt > nowMs))?.resetsAt,
|
|
148
|
+
};
|
|
149
|
+
});
|
|
150
|
+
if (verdicts.length) {
|
|
151
|
+
const preferred = choosePoolAccount(verdicts, strategy);
|
|
152
|
+
const decision = decideStickySelection({ verdicts, sticky: model.stickyEntry, nowMs, minDwellMs: model.pool.minDwellMs, strategy });
|
|
153
|
+
const picked = verdicts.find((v) => v.id === decision.account);
|
|
154
|
+
let reason;
|
|
155
|
+
if (preferred && decision.account !== preferred)
|
|
156
|
+
reason = "sticky dwell keeps this usable account";
|
|
157
|
+
else if (strategy === "ordered")
|
|
158
|
+
reason = picked?.verdict === "near_limit" ? "first near-limit member in pool order" : "first healthy member in pool order";
|
|
159
|
+
else if (picked?.verdict === "ok" && picked.sevenDayResetsAt !== undefined)
|
|
160
|
+
reason = `seven-day window resets soonest (in ${relativeUntil(picked.sevenDayResetsAt, nowMs)})`;
|
|
161
|
+
else if (picked?.verdict === "ok")
|
|
162
|
+
reason = "first healthy member without live seven-day reset data";
|
|
163
|
+
else if (picked?.verdict === "no_data")
|
|
164
|
+
reason = "first member without live health data";
|
|
165
|
+
else
|
|
166
|
+
reason = "no healthy member; using the existing fallback";
|
|
167
|
+
lines.push(` strategy: ${strategy} — picked ${decision.account}: ${reason}`);
|
|
168
|
+
}
|
|
169
|
+
else {
|
|
170
|
+
lines.push(` strategy: ${strategy} — pool.accounts[].disabled: every pool member is disabled`);
|
|
171
|
+
}
|
|
133
172
|
lines.push(model.stickyAccount
|
|
134
|
-
?
|
|
173
|
+
? strategy === "reset-aware"
|
|
174
|
+
? ` pool is on ${model.stickyAccount} (dwell applies while it remains usable)`
|
|
175
|
+
: ` pool is rotated onto ${model.stickyAccount} (returns home when the home window resets)`
|
|
135
176
|
: ` pool is on its home account (no rotation active)`);
|
|
136
177
|
}
|
|
137
178
|
}
|
package/dist/health.js
CHANGED
|
@@ -222,7 +222,27 @@ export function summarizeWindowUsage(state, options, nowMs) {
|
|
|
222
222
|
}
|
|
223
223
|
return usage.sort((a, b) => (b.resetsAt ?? 0) - (a.resetsAt ?? 0));
|
|
224
224
|
}
|
|
225
|
-
export function
|
|
225
|
+
export function poolStrategy(value) {
|
|
226
|
+
return value === "reset-aware" ? "reset-aware" : "ordered";
|
|
227
|
+
}
|
|
228
|
+
export function sevenDayReset(state, options, nowMs) {
|
|
229
|
+
const usage = summarizeWindowUsage(state, options, nowMs);
|
|
230
|
+
return (usage.find((w) => w.window === "usage:seven_day" && w.resetsAt !== undefined)
|
|
231
|
+
?? usage.find((w) => w.window === "seven_day" && w.resetsAt !== undefined))?.resetsAt;
|
|
232
|
+
}
|
|
233
|
+
export function choosePoolAccount(pool, strategy = "ordered") {
|
|
234
|
+
if (strategy === "reset-aware") {
|
|
235
|
+
let best;
|
|
236
|
+
for (const member of pool) {
|
|
237
|
+
if (member.verdict !== "ok")
|
|
238
|
+
continue;
|
|
239
|
+
if (!best || (member.sevenDayResetsAt !== undefined &&
|
|
240
|
+
(best.sevenDayResetsAt === undefined || member.sevenDayResetsAt < best.sevenDayResetsAt)))
|
|
241
|
+
best = member;
|
|
242
|
+
}
|
|
243
|
+
if (best?.sevenDayResetsAt !== undefined)
|
|
244
|
+
return best.id;
|
|
245
|
+
}
|
|
226
246
|
const usable = pool.find((a) => a.verdict === "ok" || a.verdict === "no_data");
|
|
227
247
|
if (usable)
|
|
228
248
|
return usable.id;
|
|
@@ -252,6 +272,6 @@ export function fallbackPoolAccount(pool, nowMs) {
|
|
|
252
272
|
return retest.id;
|
|
253
273
|
return (pool.find((a) => a.verdict !== "credential_failed") ?? pool[0]).id;
|
|
254
274
|
}
|
|
255
|
-
export function pickPoolAccountForLaunch(pool, nowMs) {
|
|
256
|
-
return choosePoolAccount(pool) ?? fallbackPoolAccount(pool, nowMs);
|
|
275
|
+
export function pickPoolAccountForLaunch(pool, nowMs, strategy = "ordered") {
|
|
276
|
+
return choosePoolAccount(pool, strategy) ?? fallbackPoolAccount(pool, nowMs);
|
|
257
277
|
}
|
package/dist/index.js
CHANGED
|
@@ -7,8 +7,9 @@ import { MODEL_ALIASES, buildCatalogEntries, canonicalModelId, isModernClaudeMod
|
|
|
7
7
|
import { decideDegradation, matchesPin } from "./degrade.js";
|
|
8
8
|
import { resolveExecMode, permissionModeArgs } from "./exec-policy.js";
|
|
9
9
|
import { resolveBaseModelIds } from "./catalog-source.js";
|
|
10
|
-
import { allCredentialFailed, classifyAccountHealth, credentialFailureFor, lastAuthSuccessAt, overdueProbeAccount, } from "./health.js";
|
|
10
|
+
import { allCredentialFailed, classifyAccountHealth, credentialFailureFor, lastAuthSuccessAt, overdueProbeAccount, sevenDayReset, } from "./health.js";
|
|
11
11
|
import { decideStickySelection } from "./sticky.js";
|
|
12
|
+
import { DEFAULT_SESSION_AFFINITY_TTL_MS, launchSessionId, readSessionPin, SESSION_AFFINITY_ENV, } from "./session-affinity.js";
|
|
12
13
|
import { mergeHealthStates, parseStoredState, recordCredentialFailure, } from "./shim-core.js";
|
|
13
14
|
import { createTokenRefResolver, isSecretRefShape, } from "./token-resolution.js";
|
|
14
15
|
import { resolveSecretRefValues } from "openclaw/plugin-sdk/secret-ref-runtime";
|
|
@@ -436,6 +437,7 @@ export function resolveBackendLaunch(params) {
|
|
|
436
437
|
const SHIM_ONLY_ENV = [
|
|
437
438
|
RETRY_ROSTER_ENV,
|
|
438
439
|
SESSION_DIRS_ENV,
|
|
440
|
+
SESSION_AFFINITY_ENV,
|
|
439
441
|
"MULTI_CLAWD_MODEL_OVERRIDE",
|
|
440
442
|
CLAUDE_BIN_ENV,
|
|
441
443
|
CLAUDE_ARGV0_ENV,
|
|
@@ -572,6 +574,8 @@ const launchTransports = new WeakMap();
|
|
|
572
574
|
export function buildRetryRoster(members, launchedId) {
|
|
573
575
|
const roster = [];
|
|
574
576
|
for (const member of members) {
|
|
577
|
+
if (member.disabled === true)
|
|
578
|
+
continue;
|
|
575
579
|
if (member.id === launchedId)
|
|
576
580
|
continue;
|
|
577
581
|
const tokenFile = !member.native && member.oauthTokenFile ? expandHome(member.oauthTokenFile) : undefined;
|
|
@@ -785,9 +789,10 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
785
789
|
return;
|
|
786
790
|
const poolId = pool.id?.trim() || "clawd";
|
|
787
791
|
const memberIds = (pool.accounts ?? []).filter((id) => registeredIds.has(id));
|
|
788
|
-
const
|
|
792
|
+
const configuredMembers = memberIds
|
|
789
793
|
.map((id) => accounts.find((a) => a.id.trim() === id))
|
|
790
794
|
.filter((a) => a !== undefined);
|
|
795
|
+
const members = configuredMembers.filter((a) => a.disabled !== true);
|
|
791
796
|
const ladder = (pool.degrade?.ladder ?? []).filter((m) => {
|
|
792
797
|
if (isModernClaudeModelId(m))
|
|
793
798
|
return true;
|
|
@@ -795,7 +800,10 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
795
800
|
return false;
|
|
796
801
|
});
|
|
797
802
|
const pins = pool.degrade?.pins ?? [];
|
|
798
|
-
if (members.length
|
|
803
|
+
if (members.length === 0 && memberIds.length > 0) {
|
|
804
|
+
logger.error(`[multi-clawd] pool "${poolId}": every member has pool.accounts[].disabled: true`);
|
|
805
|
+
}
|
|
806
|
+
if (members.length < 2 && !(members.length === 0 && memberIds.length > 0) && !(members.length === 1 && (ladder.length > 0 || memberIds.length > 1))) {
|
|
799
807
|
logger.warn(`[multi-clawd] pool "${poolId}" has ${members.length} registered account(s) — need at least 2 (or 1 with a degrade ladder); pool not registered`);
|
|
800
808
|
return;
|
|
801
809
|
}
|
|
@@ -820,6 +828,8 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
820
828
|
const backend = buildBackend(poolAccount, execMode, deps?.launch);
|
|
821
829
|
const { launch, transport } = launchTransports.get(backend);
|
|
822
830
|
backend.prepareExecution = async (ctx) => {
|
|
831
|
+
if (members.length === 0)
|
|
832
|
+
throw new Error(`[multi-clawd] pool "${poolId}": every member has pool.accounts[].disabled: true`);
|
|
823
833
|
const now = Date.now();
|
|
824
834
|
const requestedModel = canonicalModelId(ctx.modelId) ?? ctx.modelId;
|
|
825
835
|
const states = members.map((a) => ({ id: a.id, state: readHealthState(a.id) }));
|
|
@@ -843,12 +853,16 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
843
853
|
id: v.id,
|
|
844
854
|
verdict: v.health.verdict,
|
|
845
855
|
observedAt: v.health.observedAt,
|
|
856
|
+
sevenDayResetsAt: pool.strategy === "reset-aware"
|
|
857
|
+
? sevenDayReset(states.find((s) => s.id === v.id)?.state, options, now)
|
|
858
|
+
: undefined,
|
|
846
859
|
}));
|
|
847
860
|
const decision = decideStickySelection({
|
|
848
861
|
verdicts: poolVerdicts,
|
|
849
862
|
sticky: previousSticky,
|
|
850
863
|
nowMs: now,
|
|
851
864
|
minDwellMs,
|
|
865
|
+
strategy: pool.strategy,
|
|
852
866
|
});
|
|
853
867
|
const chosen = members.find((a) => a.id === decision.account) ?? members[0];
|
|
854
868
|
const previousAccount = previousSticky?.account ?? members[0].id;
|
|
@@ -960,7 +974,17 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
960
974
|
if (roster.length > 0)
|
|
961
975
|
env[RETRY_ROSTER_ENV] = JSON.stringify(roster);
|
|
962
976
|
}
|
|
963
|
-
env[SESSION_DIRS_ENV] = JSON.stringify([...new Set(
|
|
977
|
+
env[SESSION_DIRS_ENV] = JSON.stringify([...new Set(configuredMembers.map(accountConfigDir))]);
|
|
978
|
+
if (pool.sessionAffinity) {
|
|
979
|
+
env[SESSION_AFFINITY_ENV] = JSON.stringify({
|
|
980
|
+
file: join(homedir(), ".openclaw", "state", "multi-clawd", `pool-${poolId}.sessions.json`),
|
|
981
|
+
ttlMs: typeof pool.sessionAffinityTtlMs === "number" &&
|
|
982
|
+
Number.isFinite(pool.sessionAffinityTtlMs) && pool.sessionAffinityTtlMs > 0
|
|
983
|
+
? pool.sessionAffinityTtlMs : DEFAULT_SESSION_AFFINITY_TTL_MS,
|
|
984
|
+
verdicts: poolVerdicts.map((v) => ({ id: v.id, verdict: v.verdict })),
|
|
985
|
+
roster: buildRetryRoster(members, ""),
|
|
986
|
+
});
|
|
987
|
+
}
|
|
964
988
|
if (ladder.length > 0) {
|
|
965
989
|
const pinned = matchesPin(pins, {
|
|
966
990
|
agentDir: ctx.agentDir ?? "",
|
|
@@ -988,7 +1012,38 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
988
1012
|
logger.info(`[multi-clawd] pool ${poolId}: pinned lane keeps ${ctx.modelId} despite exhausted pool (will fail over via the chain)`);
|
|
989
1013
|
}
|
|
990
1014
|
}
|
|
991
|
-
|
|
1015
|
+
const affinityTransport = pool.sessionAffinity && transport && launch.kind === "verified"
|
|
1016
|
+
? async function* (execution) {
|
|
1017
|
+
const id = launchSessionId(execution.args);
|
|
1018
|
+
const pin = id ? readSessionPin(join(homedir(), ".openclaw", "state", "multi-clawd", `pool-${poolId}.sessions.json`), id, Date.now(), typeof pool.sessionAffinityTtlMs === "number" && pool.sessionAffinityTtlMs > 0
|
|
1019
|
+
? pool.sessionAffinityTtlMs : DEFAULT_SESSION_AFFINITY_TTL_MS) : undefined;
|
|
1020
|
+
const member = members.find((m) => m.id === pin?.account && m.oauthTokenRef);
|
|
1021
|
+
if (member && member.id !== execution.env.MULTI_CLAWD_ACCOUNT_ID &&
|
|
1022
|
+
poolVerdicts.some((v) => v.id === member.id &&
|
|
1023
|
+
(v.verdict === "ok" || v.verdict === "no_data"))) {
|
|
1024
|
+
let accountEnv;
|
|
1025
|
+
try {
|
|
1026
|
+
accountEnv = await buildAccountEnv(member, deps?.resolver);
|
|
1027
|
+
}
|
|
1028
|
+
catch {
|
|
1029
|
+
}
|
|
1030
|
+
if (accountEnv) {
|
|
1031
|
+
const selectedEnv = { ...execution.env };
|
|
1032
|
+
delete selectedEnv.CLAUDE_CODE_OAUTH_TOKEN;
|
|
1033
|
+
delete selectedEnv.CLAUDE_CONFIG_DIR;
|
|
1034
|
+
const pinnedRoster = buildRetryRoster(members, member.id);
|
|
1035
|
+
if (pinnedRoster.length > 0)
|
|
1036
|
+
selectedEnv[RETRY_ROSTER_ENV] = JSON.stringify(pinnedRoster);
|
|
1037
|
+
else
|
|
1038
|
+
delete selectedEnv[RETRY_ROSTER_ENV];
|
|
1039
|
+
yield* transport({ ...execution, env: { ...selectedEnv, ...accountEnv } });
|
|
1040
|
+
return;
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
yield* transport(execution);
|
|
1044
|
+
}
|
|
1045
|
+
: transport;
|
|
1046
|
+
return preparedForLaunch(launch, ctx, env, affinityTransport);
|
|
992
1047
|
};
|
|
993
1048
|
api.registerCliBackend(backend);
|
|
994
1049
|
api.registerProvider(buildCatalogProvider(poolAccount, members));
|
|
@@ -1027,6 +1082,11 @@ export function startDirectOrderSync(params) {
|
|
|
1027
1082
|
stopDirectOrderSync();
|
|
1028
1083
|
return { active: false, members: members.map((m) => m.accountId) };
|
|
1029
1084
|
}
|
|
1085
|
+
if (members.every((m) => m.disabled)) {
|
|
1086
|
+
stopDirectOrderSync();
|
|
1087
|
+
logger.warn("[multi-clawd] direct route: pool.accounts[].disabled: every direct member is disabled");
|
|
1088
|
+
return { active: false, members: [] };
|
|
1089
|
+
}
|
|
1030
1090
|
const requested = params.directRoute?.agents?.length ? params.directRoute.agents : ["main"];
|
|
1031
1091
|
const agents = requested.filter((a) => {
|
|
1032
1092
|
if (isValidAgentId(a))
|
|
@@ -1046,11 +1106,12 @@ export function startDirectOrderSync(params) {
|
|
|
1046
1106
|
rotateOnOverage: params.pool?.rotateOnOverage,
|
|
1047
1107
|
};
|
|
1048
1108
|
const signature = JSON.stringify({
|
|
1049
|
-
members: members.map((
|
|
1109
|
+
members: members.map(({ accountId, profileId, source, disabled }) => ({ accountId, profileId, source, disabled })),
|
|
1050
1110
|
agents,
|
|
1051
1111
|
intervalMs,
|
|
1052
1112
|
healthOptions,
|
|
1053
1113
|
minDwellMs: params.pool?.minDwellMs,
|
|
1114
|
+
strategy: params.pool?.strategy,
|
|
1054
1115
|
command: params.directRoute?.openclawCommand,
|
|
1055
1116
|
});
|
|
1056
1117
|
if (directController?.signature === signature && directTimer) {
|
|
@@ -1065,6 +1126,7 @@ export function startDirectOrderSync(params) {
|
|
|
1065
1126
|
readHealth: (id) => readHealthState(id),
|
|
1066
1127
|
healthOptions,
|
|
1067
1128
|
minDwellMs: params.pool?.minDwellMs,
|
|
1129
|
+
strategy: params.pool?.strategy,
|
|
1068
1130
|
configOrder: params.configOrder,
|
|
1069
1131
|
readSticky: () => readStickyEntry(stickyFile),
|
|
1070
1132
|
writeSticky: (entry) => writeStickyEntry(stickyFile, entry, logger),
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
2
|
+
import { closeSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import { resumeSessionId } from "./session-handover.js";
|
|
5
|
+
export const SESSION_AFFINITY_ENV = "MULTI_CLAWD_SESSION_AFFINITY";
|
|
6
|
+
export const DEFAULT_SESSION_AFFINITY_TTL_MS = 30 * 24 * 60 * 60 * 1000;
|
|
7
|
+
export const MAX_SESSION_PINS = 1000;
|
|
8
|
+
export function launchSessionId(argv) {
|
|
9
|
+
const resumed = resumeSessionId(argv);
|
|
10
|
+
if (resumed)
|
|
11
|
+
return resumed;
|
|
12
|
+
for (let i = 0; i < argv.length; i++) {
|
|
13
|
+
const arg = argv[i];
|
|
14
|
+
const value = arg === "--session-id" ? argv[i + 1]
|
|
15
|
+
: arg.startsWith("--session-id=") ? arg.slice("--session-id=".length) : undefined;
|
|
16
|
+
if (value !== undefined)
|
|
17
|
+
return /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(value) ? value : undefined;
|
|
18
|
+
}
|
|
19
|
+
return undefined;
|
|
20
|
+
}
|
|
21
|
+
export function eligibleSessionAccount(pin, currentAccount, verdicts, availableAccounts) {
|
|
22
|
+
if (!pin)
|
|
23
|
+
return undefined;
|
|
24
|
+
if (!verdicts.some((v) => v.id === pin.account &&
|
|
25
|
+
(v.verdict === "ok" || v.verdict === "no_data")))
|
|
26
|
+
return undefined;
|
|
27
|
+
return pin.account === currentAccount || availableAccounts.includes(pin.account)
|
|
28
|
+
? pin.account : undefined;
|
|
29
|
+
}
|
|
30
|
+
const WRITE_CLOCK_SLACK_MS = 60_000;
|
|
31
|
+
function readPins(file, now, ttlMs, futureSlackMs = 0) {
|
|
32
|
+
try {
|
|
33
|
+
const parsed = JSON.parse(readFileSync(file, "utf8"));
|
|
34
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
35
|
+
return {};
|
|
36
|
+
const pins = {};
|
|
37
|
+
for (const [id, value] of Object.entries(parsed)) {
|
|
38
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(id))
|
|
39
|
+
continue;
|
|
40
|
+
if (!value || typeof value !== "object")
|
|
41
|
+
continue;
|
|
42
|
+
const pin = value;
|
|
43
|
+
if (typeof pin.account !== "string" || !pin.account ||
|
|
44
|
+
typeof pin.servedAt !== "number" || !Number.isFinite(pin.servedAt) ||
|
|
45
|
+
pin.servedAt > now + futureSlackMs || now - pin.servedAt >= ttlMs)
|
|
46
|
+
continue;
|
|
47
|
+
pins[id] = { account: pin.account, servedAt: pin.servedAt };
|
|
48
|
+
}
|
|
49
|
+
return pins;
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
return {};
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
export function readSessionPin(file, sessionId, now, ttlMs) {
|
|
56
|
+
return readPins(file, now, ttlMs)[sessionId];
|
|
57
|
+
}
|
|
58
|
+
export function recordSessionPin(file, sessionId, account, now, ttlMs) {
|
|
59
|
+
const lock = `${file}.lock`;
|
|
60
|
+
let locked = false;
|
|
61
|
+
let tmp;
|
|
62
|
+
try {
|
|
63
|
+
mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
|
|
64
|
+
const deadline = Date.now() + 1000;
|
|
65
|
+
const wait = new Int32Array(new SharedArrayBuffer(4));
|
|
66
|
+
while (!locked) {
|
|
67
|
+
try {
|
|
68
|
+
closeSync(openSync(lock, "wx", 0o600));
|
|
69
|
+
locked = true;
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
try {
|
|
73
|
+
if (Date.now() - statSync(lock).mtimeMs > 5000)
|
|
74
|
+
rmSync(lock, { force: true });
|
|
75
|
+
}
|
|
76
|
+
catch { }
|
|
77
|
+
if (Date.now() >= deadline)
|
|
78
|
+
return;
|
|
79
|
+
Atomics.wait(wait, 0, 0, 10);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
const pins = readPins(file, now, ttlMs, WRITE_CLOCK_SLACK_MS);
|
|
83
|
+
pins[sessionId] = { account, servedAt: now };
|
|
84
|
+
const entries = Object.entries(pins);
|
|
85
|
+
const bounded = entries.length > MAX_SESSION_PINS
|
|
86
|
+
? entries.sort((a, b) => b[1].servedAt - a[1].servedAt).slice(0, MAX_SESSION_PINS)
|
|
87
|
+
: entries;
|
|
88
|
+
tmp = `${file}.tmp-${process.pid}-${randomBytes(6).toString("hex")}`;
|
|
89
|
+
writeFileSync(tmp, JSON.stringify(Object.fromEntries(bounded)), { mode: 0o600 });
|
|
90
|
+
renameSync(tmp, file);
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
}
|
|
94
|
+
finally {
|
|
95
|
+
if (tmp) {
|
|
96
|
+
try {
|
|
97
|
+
rmSync(tmp, { force: true });
|
|
98
|
+
}
|
|
99
|
+
catch { }
|
|
100
|
+
}
|
|
101
|
+
if (locked) {
|
|
102
|
+
try {
|
|
103
|
+
rmSync(lock, { force: true });
|
|
104
|
+
}
|
|
105
|
+
catch { }
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
package/dist/shim.js
CHANGED
|
@@ -6,6 +6,7 @@ import { rewriteModelArg } from "./degrade.js";
|
|
|
6
6
|
import { parseModelLimitError, recordModelLimit } from "./shim-core.js";
|
|
7
7
|
import { canonicalModelId } from "./models.js";
|
|
8
8
|
import { buildRetryEnv, chooseLaunchableRetryAccount, materializeRetryAccount, parseRetryRoster, retryArming, RETRY_ROSTER_ENV, } from "./retry-plan.js";
|
|
9
|
+
import { eligibleSessionAccount, launchSessionId, readSessionPin, recordSessionPin, SESSION_AFFINITY_ENV, } from "./session-affinity.js";
|
|
9
10
|
import { CLAUDE_ARGV0_ENV, CLAUDE_BIN_ENV, parseClaudeBinOverride, resolveExecutable, } from "./claude-command.js";
|
|
10
11
|
import { effectiveConfigDir, handoverForLaunch, handoverSnapshotTo, parseSessionDirs, SESSION_DIRS_ENV, snapshotResumeTranscript, } from "./session-handover.js";
|
|
11
12
|
function resolveClaudeCommand() {
|
|
@@ -104,15 +105,53 @@ if (modelOverride) {
|
|
|
104
105
|
process.stderr.write(`[multi-clawd shim] degrading model for this launch → ${modelOverride}\n`);
|
|
105
106
|
}
|
|
106
107
|
}
|
|
108
|
+
const sessionId = launchSessionId(childArgs);
|
|
109
|
+
let affinity;
|
|
110
|
+
let launchEnv = process.env;
|
|
111
|
+
try {
|
|
112
|
+
const raw = process.env[SESSION_AFFINITY_ENV];
|
|
113
|
+
if (raw && sessionId) {
|
|
114
|
+
const config = JSON.parse(raw);
|
|
115
|
+
if (typeof config.file === "string" && config.file && typeof config.ttlMs === "number" &&
|
|
116
|
+
Number.isFinite(config.ttlMs) && config.ttlMs > 0 && Array.isArray(config.verdicts)) {
|
|
117
|
+
affinity = {
|
|
118
|
+
file: config.file,
|
|
119
|
+
ttlMs: config.ttlMs,
|
|
120
|
+
verdicts: config.verdicts,
|
|
121
|
+
roster: parseRetryRoster(JSON.stringify(config.roster)),
|
|
122
|
+
};
|
|
123
|
+
const pin = readSessionPin(affinity.file, sessionId, Date.now(), affinity.ttlMs);
|
|
124
|
+
const eligible = eligibleSessionAccount(pin, accountId, affinity.verdicts, affinity.roster.map((r) => r.id));
|
|
125
|
+
if (eligible && eligible !== accountId) {
|
|
126
|
+
const member = affinity.roster.find((r) => r.id === eligible);
|
|
127
|
+
if (member) {
|
|
128
|
+
const ready = materializeRetryAccount(member, (path) => readFileSync(path, "utf8"));
|
|
129
|
+
if ("account" in ready) {
|
|
130
|
+
launchEnv = buildRetryEnv(process.env, ready.account);
|
|
131
|
+
accountId = ready.account.id;
|
|
132
|
+
stateFile = ready.account.stateFile;
|
|
133
|
+
state = { accountId, windows: {} };
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
affinity = undefined;
|
|
142
|
+
}
|
|
143
|
+
delete launchEnv[SESSION_AFFINITY_ENV];
|
|
107
144
|
{
|
|
108
|
-
const note = handoverForLaunch(childArgs,
|
|
145
|
+
const note = handoverForLaunch(childArgs, launchEnv);
|
|
109
146
|
if (note)
|
|
110
147
|
process.stderr.write(`[multi-clawd shim] ${note}\n`);
|
|
111
148
|
}
|
|
112
|
-
const retryRoster =
|
|
149
|
+
const retryRoster = affinity && launchEnv !== process.env
|
|
150
|
+
? affinity.roster.filter((r) => r.id !== accountId)
|
|
151
|
+
: parseRetryRoster(process.env[RETRY_ROSTER_ENV]);
|
|
113
152
|
delete process.env[RETRY_ROSTER_ENV];
|
|
114
|
-
const resumeSnapshot = retryRoster.length > 0 && parseSessionDirs(
|
|
115
|
-
? snapshotResumeTranscript(childArgs,
|
|
153
|
+
const resumeSnapshot = retryRoster.length > 0 && parseSessionDirs(launchEnv[SESSION_DIRS_ENV]).length > 0
|
|
154
|
+
? snapshotResumeTranscript(childArgs, launchEnv)
|
|
116
155
|
: undefined;
|
|
117
156
|
const arming = retryArming(childArgs, retryRoster, { resumeReady: resumeSnapshot !== undefined });
|
|
118
157
|
let retryArmed = arming.armed;
|
|
@@ -121,7 +160,7 @@ if (!retryArmed && retryRoster.length > 0 && arming.reason) {
|
|
|
121
160
|
}
|
|
122
161
|
let child = spawn(command, childArgs, {
|
|
123
162
|
stdio: ["pipe", "pipe", "inherit"],
|
|
124
|
-
env:
|
|
163
|
+
env: launchEnv,
|
|
125
164
|
...(claudeArgv0 ? { argv0: claudeArgv0 } : {}),
|
|
126
165
|
});
|
|
127
166
|
const STDIN_REPLAY_CAP_BYTES = 8 * 1024 * 1024;
|
|
@@ -301,7 +340,7 @@ function attemptRetry() {
|
|
|
301
340
|
sawSuccessResult = false;
|
|
302
341
|
child = spawn(command, childArgs, {
|
|
303
342
|
stdio: ["pipe", "pipe", "inherit"],
|
|
304
|
-
env: buildRetryEnv(
|
|
343
|
+
env: buildRetryEnv(launchEnv, target),
|
|
305
344
|
...(claudeArgv0 ? { argv0: claudeArgv0 } : {}),
|
|
306
345
|
});
|
|
307
346
|
attachChildHandlers();
|
|
@@ -419,6 +458,9 @@ function onChildClose(code, signal) {
|
|
|
419
458
|
if ((code ?? 0) === 0) {
|
|
420
459
|
clearRecordedAuthFailureOnSuccess();
|
|
421
460
|
clearRecordedModelRejectionOnSuccess();
|
|
461
|
+
if (affinity && sessionId && sawSuccessResult && !sawAuthFailure && !sawModelLimit) {
|
|
462
|
+
recordSessionPin(affinity.file, sessionId, accountId, Date.now(), affinity.ttlMs);
|
|
463
|
+
}
|
|
422
464
|
}
|
|
423
465
|
process.exit(code ?? 0);
|
|
424
466
|
}
|
package/dist/sticky.js
CHANGED
|
@@ -4,9 +4,19 @@ export function decideStickySelection(params) {
|
|
|
4
4
|
const { verdicts, sticky, nowMs } = params;
|
|
5
5
|
const minDwellMs = params.minDwellMs ?? DEFAULT_MIN_DWELL_MS;
|
|
6
6
|
const home = verdicts[0];
|
|
7
|
-
const healthChoice = choosePoolAccount(verdicts);
|
|
7
|
+
const healthChoice = choosePoolAccount(verdicts, params.strategy);
|
|
8
8
|
if (!healthChoice)
|
|
9
9
|
return { account: fallbackPoolAccount(verdicts, nowMs) };
|
|
10
|
+
if (params.strategy === "reset-aware") {
|
|
11
|
+
const current = sticky && verdicts.find((v) => v.id === sticky.account);
|
|
12
|
+
const stillUsable = current?.verdict === "ok" || current?.verdict === "no_data";
|
|
13
|
+
if (sticky && stillUsable && nowMs - sticky.since < minDwellMs) {
|
|
14
|
+
return { account: sticky.account, sticky };
|
|
15
|
+
}
|
|
16
|
+
if (sticky?.account === healthChoice)
|
|
17
|
+
return { account: healthChoice, sticky };
|
|
18
|
+
return { account: healthChoice, sticky: { account: healthChoice, since: nowMs } };
|
|
19
|
+
}
|
|
10
20
|
const stickyVerdict = sticky
|
|
11
21
|
? verdicts.find((v) => v.id === sticky.account)?.verdict
|
|
12
22
|
: undefined;
|
package/openclaw.plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "multi-clawd",
|
|
3
3
|
"name": "multi-clawd",
|
|
4
|
-
"version": "2.
|
|
4
|
+
"version": "2.4.0",
|
|
5
5
|
"description": "Register additional Claude Code logins (Max/Pro accounts) as first-class OpenClaw CLI backends for cross-account failover, keeping the full skills/MCP harness on every account.",
|
|
6
6
|
"cliBackends": [
|
|
7
7
|
"claw1",
|
|
@@ -41,7 +41,12 @@
|
|
|
41
41
|
"items": {
|
|
42
42
|
"type": "string"
|
|
43
43
|
},
|
|
44
|
-
"description": "Account ids (from accounts[]) in preference order; the first is the home account
|
|
44
|
+
"description": "Account ids (from accounts[]) in preference order; the first is the home account in the default ordered strategy."
|
|
45
|
+
},
|
|
46
|
+
"strategy": {
|
|
47
|
+
"type": "string",
|
|
48
|
+
"enum": ["ordered", "reset-aware"],
|
|
49
|
+
"description": "Healthy account selection: ordered (default) keeps pool order; reset-aware prefers the healthy account whose live seven-day quota window resets soonest."
|
|
45
50
|
},
|
|
46
51
|
"utilizationThreshold": {
|
|
47
52
|
"type": "number",
|
|
@@ -65,6 +70,15 @@
|
|
|
65
70
|
"type": "number",
|
|
66
71
|
"description": "After rotating away from home, stay on the rotated-to account at least this long before returning home (anti-flap hysteresis; health always overrides). Default 600000 (10min)."
|
|
67
72
|
},
|
|
73
|
+
"sessionAffinity": {
|
|
74
|
+
"type": "boolean",
|
|
75
|
+
"description": "Pin a Claude CLI session to the account that last served it, while that account remains healthy. CLI launches only. Default false."
|
|
76
|
+
},
|
|
77
|
+
"sessionAffinityTtlMs": {
|
|
78
|
+
"type": "number",
|
|
79
|
+
"exclusiveMinimum": 0,
|
|
80
|
+
"description": "Successful session pin lifetime in milliseconds. Default 2592000000 (30 days); only used when sessionAffinity is true."
|
|
81
|
+
},
|
|
68
82
|
"usagePoll": {
|
|
69
83
|
"type": "object",
|
|
70
84
|
"additionalProperties": false,
|
|
@@ -147,6 +161,11 @@
|
|
|
147
161
|
"type": "string",
|
|
148
162
|
"description": "Human-friendly label shown in status/catalog."
|
|
149
163
|
},
|
|
164
|
+
"disabled": {
|
|
165
|
+
"type": "boolean",
|
|
166
|
+
"default": false,
|
|
167
|
+
"description": "Exclude this account from pool selection, in-turn retries, session affinity and direct Anthropic order while keeping it configured and visible in status. Takes effect when the plugin next reads config."
|
|
168
|
+
},
|
|
150
169
|
"native": {
|
|
151
170
|
"type": "boolean",
|
|
152
171
|
"description": "Use the machine's native Claude login (default config dir; OS keychain on macOS). Sets neither CLAUDE_CONFIG_DIR nor a token. Use this to include the main account in the rotation pool."
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@drakon-systems/multi-clawd",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "Multi-account Claude Code failover for OpenClaw — register additional Claude (Max/Pro) logins as first-class CLI backends and keep the full skills/MCP harness across every account. Also imports those accounts' setup tokens into Hermes Agent's Anthropic credential pool.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { constants, copyFileSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { homedir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
|
|
5
|
+
/** Locate JSON object properties without reserializing the operator's config. */
|
|
6
|
+
export function jsonSpans(source, offset = 0) {
|
|
7
|
+
let at = offset;
|
|
8
|
+
const ws = () => { while (/\s/.test(source[at] ?? "")) at++; };
|
|
9
|
+
function string() {
|
|
10
|
+
const start = at++;
|
|
11
|
+
while (at < source.length) {
|
|
12
|
+
if (source[at] === "\\") { at += 2; continue; }
|
|
13
|
+
if (source[at++] === '"') return { value: JSON.parse(source.slice(start, at)), start, end: at };
|
|
14
|
+
}
|
|
15
|
+
throw new Error("unterminated JSON string");
|
|
16
|
+
}
|
|
17
|
+
function value() {
|
|
18
|
+
ws();
|
|
19
|
+
const start = at;
|
|
20
|
+
if (source[at] === "{") {
|
|
21
|
+
at++; ws(); const properties = [];
|
|
22
|
+
while (source[at] !== "}") {
|
|
23
|
+
const key = string(); ws();
|
|
24
|
+
if (source[at++] !== ":") throw new Error("invalid JSON object");
|
|
25
|
+
const child = value(); ws();
|
|
26
|
+
const end = at;
|
|
27
|
+
properties.push({ key: key.value, keyStart: key.start, start: child.start, end: child.end, propertyEnd: end });
|
|
28
|
+
if (source[at] !== ",") break;
|
|
29
|
+
at++; ws();
|
|
30
|
+
}
|
|
31
|
+
if (source[at++] !== "}") throw new Error("invalid JSON object");
|
|
32
|
+
return { start, end: at, properties };
|
|
33
|
+
}
|
|
34
|
+
if (source[at] === "[") {
|
|
35
|
+
at++; ws(); const items = [];
|
|
36
|
+
while (source[at] !== "]") {
|
|
37
|
+
items.push(value()); ws();
|
|
38
|
+
if (source[at] !== ",") break;
|
|
39
|
+
at++; ws();
|
|
40
|
+
}
|
|
41
|
+
if (source[at++] !== "]") throw new Error("invalid JSON array");
|
|
42
|
+
return { start, end: at, items };
|
|
43
|
+
}
|
|
44
|
+
if (source[at] === '"') return string();
|
|
45
|
+
while (at < source.length && !/[\s,}\]]/.test(source[at])) at++;
|
|
46
|
+
return { start, end: at };
|
|
47
|
+
}
|
|
48
|
+
const root = value(); ws();
|
|
49
|
+
if (offset === 0 && at !== source.length) throw new Error("trailing JSON content");
|
|
50
|
+
return root;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const property = (node, key) => {
|
|
54
|
+
if (!node?.properties) throw new Error("expected JSON object");
|
|
55
|
+
const matches = node.properties.filter((p) => p.key === key);
|
|
56
|
+
if (matches.length > 1) throw new Error(`duplicate ${key} property`);
|
|
57
|
+
return matches[0];
|
|
58
|
+
};
|
|
59
|
+
const child = (source, node, key) => {
|
|
60
|
+
const p = property(node, key);
|
|
61
|
+
return p && jsonSpans(source, p.start);
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
export function editAccountDisabled(source, id, disabled) {
|
|
65
|
+
const original = JSON.parse(source);
|
|
66
|
+
let node = jsonSpans(source);
|
|
67
|
+
for (const key of ["plugins", "entries", "multi-clawd", "config", "accounts"]) {
|
|
68
|
+
node = child(source, node, key);
|
|
69
|
+
if (!node) throw new Error(`missing plugins.entries["multi-clawd"].config.accounts`);
|
|
70
|
+
}
|
|
71
|
+
if (!node.items) throw new Error("accounts must be an array");
|
|
72
|
+
const accounts = original?.plugins?.entries?.["multi-clawd"]?.config?.accounts;
|
|
73
|
+
const matches = accounts?.map((a, i) => a?.id === id ? i : -1).filter((i) => i >= 0) ?? [];
|
|
74
|
+
if (matches.length === 0) throw new Error(`unknown account id "${id}"`);
|
|
75
|
+
if (matches.length !== 1) throw new Error(`duplicate account id "${id}"`);
|
|
76
|
+
const account = node.items[matches[0]];
|
|
77
|
+
const flag = property(account, "disabled");
|
|
78
|
+
if (flag && typeof accounts[matches[0]].disabled !== "boolean") throw new Error("disabled must be a boolean");
|
|
79
|
+
let edited;
|
|
80
|
+
if (flag) {
|
|
81
|
+
edited = source.slice(0, flag.start) + String(disabled) + source.slice(flag.end);
|
|
82
|
+
} else if (disabled) {
|
|
83
|
+
const indent = source.slice(source.lastIndexOf("\n", account.start - 1) + 1, account.start);
|
|
84
|
+
const space = source.slice(account.start + 1, account.end - 1);
|
|
85
|
+
const multiline = space.includes("\n");
|
|
86
|
+
const addition = account.properties.length
|
|
87
|
+
? `,${multiline ? `\n${indent} ` : " "}"disabled": true`
|
|
88
|
+
: `"disabled": true`;
|
|
89
|
+
edited = source.slice(0, account.end - 1) + addition + source.slice(account.end - 1);
|
|
90
|
+
} else {
|
|
91
|
+
return { edited: source, changed: false };
|
|
92
|
+
}
|
|
93
|
+
const expected = structuredClone(original);
|
|
94
|
+
expected.plugins.entries["multi-clawd"].config.accounts[matches[0]].disabled = disabled;
|
|
95
|
+
if (JSON.stringify(JSON.parse(edited)) !== JSON.stringify(expected)) throw new Error("targeted edit changed other config values");
|
|
96
|
+
return { edited, changed: edited !== source };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function changeAccount(id, disabled, dryRun = false, configFile = join(homedir(), ".openclaw", "openclaw.json")) {
|
|
100
|
+
const source = readFileSync(configFile, "utf8");
|
|
101
|
+
const { edited, changed } = editAccountDisabled(source, id, disabled);
|
|
102
|
+
if (dryRun || !changed) return { changed, dryRun };
|
|
103
|
+
if (readFileSync(configFile, "utf8") !== source) throw new Error("config changed during edit; retry");
|
|
104
|
+
const backup = `${configFile}.backup-${new Date().toISOString().replace(/[:.]/g, "-")}`;
|
|
105
|
+
copyFileSync(configFile, backup, constants.COPYFILE_EXCL);
|
|
106
|
+
const temporary = `${configFile}.tmp-${process.pid}-${Date.now()}`;
|
|
107
|
+
try {
|
|
108
|
+
writeFileSync(temporary, edited, { mode: statSync(configFile).mode & 0o777 });
|
|
109
|
+
if (readFileSync(configFile, "utf8") !== source) throw new Error("config changed during edit; retry");
|
|
110
|
+
renameSync(temporary, configFile);
|
|
111
|
+
} finally {
|
|
112
|
+
rmSync(temporary, { force: true });
|
|
113
|
+
}
|
|
114
|
+
return { changed, dryRun, backup };
|
|
115
|
+
}
|
package/scripts/cli.mjs
CHANGED
|
@@ -36,6 +36,7 @@ ${BOLD}🦞 multi-clawd${RESET} — multi-account Claude failover for OpenClaw
|
|
|
36
36
|
|
|
37
37
|
${BOLD}setup${RESET} guided setup wizard (accounts, pool, watchdog)
|
|
38
38
|
${BOLD}login${RESET} log a configured account in (or re-auth it) — right dir, right env
|
|
39
|
+
${BOLD}account${RESET} disable or enable a pool account (add --dry-run to preview)
|
|
39
40
|
${BOLD}explain${RESET} your setup in plain English — accounts, pool, fallback chain
|
|
40
41
|
${BOLD}chain${RESET} audit your model routing — what actually serves each turn
|
|
41
42
|
${BOLD}direct${RESET} the direct anthropic/* route — status, or \`direct sync\` to store profiles
|
|
@@ -215,6 +216,7 @@ async function loadDirect(config) {
|
|
|
215
216
|
configOrder,
|
|
216
217
|
sticky,
|
|
217
218
|
minDwellMs: pool?.minDwellMs,
|
|
219
|
+
strategy: pool?.strategy,
|
|
218
220
|
nowMs: Date.now(),
|
|
219
221
|
...opts,
|
|
220
222
|
});
|
|
@@ -692,9 +694,11 @@ async function explain() {
|
|
|
692
694
|
return { id: a.id, verdict: h.verdict, detail, usage };
|
|
693
695
|
});
|
|
694
696
|
let stickyAccount;
|
|
697
|
+
let stickyEntry;
|
|
695
698
|
if (pool) {
|
|
696
699
|
try {
|
|
697
700
|
const sticky = JSON.parse(rf(join(stateDir, `pool-${pool.id}.sticky.json`), "utf8"));
|
|
701
|
+
stickyEntry = sticky;
|
|
698
702
|
if (sticky?.account && sticky.account !== pool.accounts[0]) stickyAccount = sticky.account;
|
|
699
703
|
} catch {
|
|
700
704
|
/* no sticky state */
|
|
@@ -710,7 +714,7 @@ async function explain() {
|
|
|
710
714
|
}
|
|
711
715
|
console.log(`\n${BOLD}🦞 multi-clawd — your setup, in plain English${RESET}\n`);
|
|
712
716
|
console.log(
|
|
713
|
-
ec.renderExplanation({ accounts, pool, chain, health: healthRows, stickyAccount, nowMs: now, direct }),
|
|
717
|
+
ec.renderExplanation({ accounts, pool, chain, health: healthRows, stickyAccount, stickyEntry, nowMs: now, direct }),
|
|
714
718
|
);
|
|
715
719
|
console.log(`\n${DIM}(health checks: multi-clawd doctor · change things: multi-clawd setup)${RESET}`);
|
|
716
720
|
}
|
|
@@ -755,8 +759,9 @@ async function usageCommand(args) {
|
|
|
755
759
|
const stateDir = join(homedir(), ".openclaw", "state", "multi-clawd");
|
|
756
760
|
const now = Date.now();
|
|
757
761
|
const rows = [];
|
|
762
|
+
const resetById = new Map();
|
|
758
763
|
for (const a of accounts) {
|
|
759
|
-
const row = { id: a.id, label: a.label };
|
|
764
|
+
const row = { id: a.id, label: a.label, ...(a.disabled === true ? { disabled: true, status: "disabled by operator" } : {}) };
|
|
760
765
|
if (a.oauthTokenFile || a.oauthTokenRef) {
|
|
761
766
|
row.skipped = "setup-token login — not polled: the usage endpoint needs the user:profile scope, setup-tokens carry only user:inference (stream telemetry only)";
|
|
762
767
|
} else if (!a.native && !a.configDir) {
|
|
@@ -785,12 +790,21 @@ async function usageCommand(args) {
|
|
|
785
790
|
const h = health.classifyAccountHealth(state, healthOptions, now);
|
|
786
791
|
row.verdict = h.verdict;
|
|
787
792
|
row.reason = h.reason;
|
|
793
|
+
if (pool?.strategy === "reset-aware") resetById.set(a.id, health.sevenDayReset(state, healthOptions, now));
|
|
788
794
|
rows.push(row);
|
|
789
795
|
}
|
|
790
796
|
let next;
|
|
791
797
|
if (pool) {
|
|
792
|
-
const verdicts = pool.accounts.map((id) => ({ id, verdict: rows.find((r) => r.id === id)?.verdict ?? "no_data"
|
|
793
|
-
|
|
798
|
+
const verdicts = pool.accounts.filter((id) => !accounts.find((a) => a.id === id)?.disabled).map((id) => ({ id, verdict: rows.find((r) => r.id === id)?.verdict ?? "no_data",
|
|
799
|
+
sevenDayResetsAt: resetById.get(id) }));
|
|
800
|
+
if (pool.strategy === "reset-aware") {
|
|
801
|
+
const stickySelection = await import(resolve(__dirname, "..", "dist", "sticky.js"));
|
|
802
|
+
let sticky;
|
|
803
|
+
try { sticky = JSON.parse(rf(join(stateDir, `pool-${pool.id}.sticky.json`), "utf8")); } catch { /* none */ }
|
|
804
|
+
next = verdicts.length ? stickySelection.decideStickySelection({ verdicts, sticky, nowMs: now, minDwellMs: pool.minDwellMs, strategy: pool.strategy }).account : undefined;
|
|
805
|
+
} else {
|
|
806
|
+
next = verdicts.length ? health.choosePoolAccount(verdicts) : undefined;
|
|
807
|
+
}
|
|
794
808
|
}
|
|
795
809
|
if (args.includes("--json")) {
|
|
796
810
|
console.log(JSON.stringify({ at: new Date(now).toISOString(), rotateAt, warnThreshold, pool: pool?.id, nextLaunch: next, accounts: rows }, null, 2));
|
|
@@ -809,7 +823,7 @@ async function usageCommand(args) {
|
|
|
809
823
|
};
|
|
810
824
|
console.log(`\n${BOLD}🦞 multi-clawd — live usage${RESET} ${DIM}(rotate at ${pct(rotateAt)}${weeklyRotateAt !== undefined ? `, weekly ${pct(weeklyRotateAt)}` : ""}, warn at ${pct(warnThreshold)})${RESET}\n`);
|
|
811
825
|
for (const r of rows) {
|
|
812
|
-
console.log(` ${BOLD}${r.id}${RESET}${r.label ? ` ${DIM}— ${r.label}${RESET}` : ""} pool verdict: ${r.verdict}${r.reason ? ` ${DIM}(${r.reason})${RESET}` : ""}`);
|
|
826
|
+
console.log(` ${BOLD}${r.id}${RESET}${r.label ? ` ${DIM}— ${r.label}${RESET}` : ""} pool verdict: ${r.disabled ? "disabled by operator" : r.verdict}${r.reason ? ` ${DIM}(${r.reason})${RESET}` : ""}`);
|
|
813
827
|
if (r.snapshot) {
|
|
814
828
|
for (const [w, u] of Object.entries(r.snapshot.windows)) {
|
|
815
829
|
console.log(` ${mark(u.utilization, w)} ${up.usageWindowLabel(w).padEnd(12)} ${pct(u.utilization).padStart(4)}${until(u.resetsAt)}`);
|
|
@@ -825,7 +839,10 @@ async function usageCommand(args) {
|
|
|
825
839
|
console.log("");
|
|
826
840
|
}
|
|
827
841
|
if (pool) {
|
|
828
|
-
|
|
842
|
+
const allDisabled = pool.accounts.length > 0 && pool.accounts.every((id) => accounts.find((a) => a.id === id)?.disabled === true);
|
|
843
|
+
console.log(next ? ` next pooled launch (${pool.id}/…) runs on ${BOLD}${next}${RESET}` : allDisabled
|
|
844
|
+
? ` ${BOLD}pool.accounts[].disabled: every pool member is disabled${RESET}`
|
|
845
|
+
: ` ${BOLD}no selectable pool member${RESET}`);
|
|
829
846
|
}
|
|
830
847
|
console.log(`\n${DIM}(the gateway polls this every ${Math.round(up.effectiveUsagePollInterval(pool?.usagePoll) / 1000)}s and alerts via the heartbeat; see "Live usage" in the README)${RESET}`);
|
|
831
848
|
}
|
|
@@ -941,6 +958,25 @@ switch (cmd) {
|
|
|
941
958
|
case "login":
|
|
942
959
|
await login();
|
|
943
960
|
break;
|
|
961
|
+
case "account": {
|
|
962
|
+
const [action, id, ...flags] = rest;
|
|
963
|
+
if (!["enable", "disable"].includes(action) || !id || flags.some((f) => f !== "--dry-run")) {
|
|
964
|
+
console.error("usage: multi-clawd account <enable|disable> <id> [--dry-run]");
|
|
965
|
+
process.exitCode = 1;
|
|
966
|
+
break;
|
|
967
|
+
}
|
|
968
|
+
try {
|
|
969
|
+
const { changeAccount } = await import("./account-config.mjs");
|
|
970
|
+
const result = changeAccount(id, action === "disable", flags.includes("--dry-run"));
|
|
971
|
+
console.log(`${result.dryRun ? "dry run: " : ""}${id}: ${action === "disable" ? "disabled" : "enabled"}${result.changed ? "" : " (already set)"}`);
|
|
972
|
+
if (result.backup) console.log(`backup: ${result.backup}`);
|
|
973
|
+
if (!result.dryRun && result.changed) console.log("Takes effect when the plugin next reads config (for example after gateway reload); running turns continue.");
|
|
974
|
+
} catch (err) {
|
|
975
|
+
console.error(`account: ${err.message}`);
|
|
976
|
+
process.exitCode = 1;
|
|
977
|
+
}
|
|
978
|
+
break;
|
|
979
|
+
}
|
|
944
980
|
case "explain":
|
|
945
981
|
await explain();
|
|
946
982
|
break;
|
package/scripts/doctor.mjs
CHANGED
|
@@ -294,12 +294,13 @@ const { createSystemCredentialIo, findCredentialCopies, describeSplitStore } = a
|
|
|
294
294
|
["createSystemCredentialIo", "findCredentialCopies", "describeSplitStore"],
|
|
295
295
|
"cli",
|
|
296
296
|
);
|
|
297
|
-
const { summarizeWindowUsage, classifyAccountHealth, overdueProbeAccount, lastAuthSuccessAt } =
|
|
297
|
+
const { summarizeWindowUsage, classifyAccountHealth, overdueProbeAccount, lastAuthSuccessAt, sevenDayReset } =
|
|
298
298
|
await importDist("health.js", [
|
|
299
299
|
"summarizeWindowUsage",
|
|
300
300
|
"classifyAccountHealth",
|
|
301
301
|
"overdueProbeAccount",
|
|
302
302
|
"lastAuthSuccessAt",
|
|
303
|
+
"sevenDayReset",
|
|
303
304
|
]);
|
|
304
305
|
const { DEFAULT_UTILIZATION_THRESHOLD } = await import(join(REPO_DIR, "dist", "health.js"));
|
|
305
306
|
// "cli", like the chain audits: resolving WHICH login backs an account is
|
|
@@ -353,6 +354,7 @@ const accounts = pluginConfig.accounts ?? [];
|
|
|
353
354
|
const identities = [];
|
|
354
355
|
if (accounts.length === 0) warn("no accounts configured");
|
|
355
356
|
for (const account of accounts) {
|
|
357
|
+
if (account.disabled === true) note(`${account.id}: disabled by operator`);
|
|
356
358
|
// WHO this account is, before WHETHER its credential works. An id is a
|
|
357
359
|
// label the operator chose; the login is the thing quota is spent against,
|
|
358
360
|
// and the two drift apart silently (a config dir re-logged-in as the wrong
|
|
@@ -493,8 +495,16 @@ console.log("pool");
|
|
|
493
495
|
const pool = pluginConfig.pool;
|
|
494
496
|
if (!pool) warn("no pool configured — direct backends only, no proactive rotation");
|
|
495
497
|
else {
|
|
496
|
-
const
|
|
497
|
-
|
|
498
|
+
const configuredMembers = (pool.accounts ?? []).filter((id) => accounts.some((a) => a.id === id));
|
|
499
|
+
const members = configuredMembers.filter((id) => accounts.some((a) => a.id === id && a.disabled !== true));
|
|
500
|
+
const { isModernClaudeModelId } = await importDist("models.js", ["isModernClaudeModelId"], "cli");
|
|
501
|
+
const hasDegradeLadder = (pool.degrade?.ladder ?? []).some(isModernClaudeModelId);
|
|
502
|
+
if (members.length === 0 && configuredMembers.length > 0) bad("pool.accounts[].disabled: every pool member is disabled");
|
|
503
|
+
if (members.length === 1 && configuredMembers.length > 1) {
|
|
504
|
+
warn(`pool "${pool.id ?? "clawd"}" has 1 valid member(s); other configured members are disabled — pool remains available`);
|
|
505
|
+
} else if (members.length === 1 && hasDegradeLadder) {
|
|
506
|
+
note(`pool "${pool.id ?? "clawd"}" has 1 valid member(s) with a degrade ladder; pool remains available`);
|
|
507
|
+
} else if (members.length < 2) bad(`pool "${pool.id ?? "clawd"}" has ${members.length} valid member(s); needs ≥ 2`);
|
|
498
508
|
else ok(`pool "${pool.id ?? "clawd"}": ${members.join(" → ")}`);
|
|
499
509
|
const generalThreshold = pool.utilizationThreshold ?? DEFAULT_UTILIZATION_THRESHOLD;
|
|
500
510
|
const weeklyThreshold = pool.weeklyUtilizationThreshold ?? generalThreshold;
|
|
@@ -536,16 +546,17 @@ else {
|
|
|
536
546
|
weeklyUtilizationThreshold: pool.weeklyUtilizationThreshold,
|
|
537
547
|
rotateOnOverage: pool.rotateOnOverage,
|
|
538
548
|
};
|
|
539
|
-
const verdicts = members.map((id) =>
|
|
540
|
-
id
|
|
541
|
-
verdict: classifyAccountHealth(
|
|
542
|
-
.
|
|
543
|
-
})
|
|
549
|
+
const verdicts = members.map((id) => {
|
|
550
|
+
const state = readJson(join(STATE_DIR, `${id}.json`));
|
|
551
|
+
return { id, verdict: classifyAccountHealth(state, healthOptions, now).verdict,
|
|
552
|
+
sevenDayResetsAt: pool.strategy === "reset-aware" ? sevenDayReset(state, healthOptions, now) : undefined };
|
|
553
|
+
});
|
|
544
554
|
const decision = decideStickySelection({
|
|
545
555
|
verdicts,
|
|
546
556
|
sticky,
|
|
547
557
|
nowMs: now,
|
|
548
558
|
minDwellMs: pool.minDwellMs,
|
|
559
|
+
strategy: pool.strategy,
|
|
549
560
|
});
|
|
550
561
|
const identity = identities.find((i) => i.accountId === decision.account);
|
|
551
562
|
const who =
|
|
@@ -888,6 +899,7 @@ if (accounts.some((a) => a && a.direct !== undefined && a.direct !== false)) {
|
|
|
888
899
|
configOrder: Array.isArray(config?.auth?.order?.anthropic) ? config.auth.order.anthropic : undefined,
|
|
889
900
|
sticky: readJson(join(STATE_DIR, "direct-order.sticky.json")),
|
|
890
901
|
minDwellMs: pool?.minDwellMs,
|
|
902
|
+
strategy: pool?.strategy,
|
|
891
903
|
nowMs: Date.now(),
|
|
892
904
|
});
|
|
893
905
|
if (!status) continue;
|