@drakon-systems/multi-clawd 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -3
- package/dist/direct-report.js +7 -1
- package/dist/direct-route.js +7 -5
- package/dist/direct-sync.js +7 -4
- package/dist/explain-core.js +4 -4
- package/dist/index.js +61 -5
- package/dist/session-affinity.js +108 -0
- package/dist/shim.js +48 -6
- package/openclaw.plugin.json +15 -1
- package/package.json +1 -1
- package/scripts/account-config.mjs +115 -0
- package/scripts/cli.mjs +29 -6
- package/scripts/doctor.mjs +11 -2
package/README.md
CHANGED
|
@@ -30,6 +30,7 @@ multi-clawd update # install (or update) the OpenClaw plugin — right flags,
|
|
|
30
30
|
multi-clawd setup # guided wizard: accounts, isolated second login, pool, watchdog
|
|
31
31
|
multi-clawd login claw2 # launch the right Claude sign-in for an account (or re-auth it)
|
|
32
32
|
multi-clawd explain # your whole setup in plain English — accounts, chain, live health
|
|
33
|
+
multi-clawd account disable claw2 --dry-run # preview benching an account
|
|
33
34
|
multi-clawd chain # audit your model routing — what actually serves each turn
|
|
34
35
|
multi-clawd direct # the direct anthropic/* route: status, or `direct sync` (v1.9, optional)
|
|
35
36
|
multi-clawd doctor # health check (add --probe for a live end-to-end turn)
|
|
@@ -386,7 +387,7 @@ openclaw plugins install (Get-Location).Path
|
|
|
386
387
|
**Or let your agent install it.** Running an OpenClaw assistant or Claude
|
|
387
388
|
Code on the target machine already? Paste it this and go make coffee:
|
|
388
389
|
|
|
389
|
-
> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v2.
|
|
390
|
+
> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v2.4.0/SETUP-AGENT.md
|
|
390
391
|
> and follow it to set up multi-clawd on this machine. I own a second
|
|
391
392
|
> Claude account — ask me when you need me to log in.
|
|
392
393
|
|
|
@@ -546,6 +547,7 @@ error.
|
|
|
546
547
|
"accounts": [
|
|
547
548
|
{ "id": "claw1", "label": "Main Claude", "native": true },
|
|
548
549
|
{ "id": "claw2", "label": "Second Max", "configDir": "~/.claw2",
|
|
550
|
+
// "disabled": true, // optional; default false: bench without removing the pool member
|
|
549
551
|
// v0.3 preferred: resolve via your gateway's secret providers —
|
|
550
552
|
// no plaintext token file on disk
|
|
551
553
|
"oauthTokenRef": { "source": "exec", "provider": "onepassword",
|
|
@@ -558,6 +560,8 @@ error.
|
|
|
558
560
|
"utilizationThreshold": 0.85, // default hand-over; minute/hour and overage windows use this
|
|
559
561
|
"weeklyUtilizationThreshold": 0.97, // optional: day/week/month quota windows at 97%
|
|
560
562
|
"minDwellMs": 600000, // v0.3: anti-flap dwell before returning home
|
|
563
|
+
"sessionAffinity": true, // optional; default false: keep healthy CLI sessions on their serving account
|
|
564
|
+
"sessionAffinityTtlMs": 2592000000, // optional; default 30 days
|
|
561
565
|
"degrade": { // v0.3.5: last step before provider drop
|
|
562
566
|
"ladder": ["claude-opus-4-8"], // whole pool exhausted → same account, lower tier
|
|
563
567
|
"pins": [ // contractual lanes never degrade
|
|
@@ -578,6 +582,25 @@ error.
|
|
|
578
582
|
} }
|
|
579
583
|
```
|
|
580
584
|
|
|
585
|
+
Set `accounts[].disabled: true` to bench a member while keeping it in
|
|
586
|
+
`pool.accounts` and visible in `explain`, `doctor`, and `usage` as "disabled by
|
|
587
|
+
operator". It is excluded from pool selection, in-turn retries, session
|
|
588
|
+
affinity, and the managed direct Anthropic order. Transcript lookup still
|
|
589
|
+
searches its session directory. To edit `~/.openclaw/openclaw.json` with a
|
|
590
|
+
timestamped backup and without reformatting unrelated config, run
|
|
591
|
+
`multi-clawd account disable <id>` or `multi-clawd account enable <id>`.
|
|
592
|
+
Add `--dry-run` to preview without writing anything. If every pool member is
|
|
593
|
+
disabled, pooled launches fail closed with an error naming
|
|
594
|
+
`pool.accounts[].disabled`; they never fall back to a disabled member.
|
|
595
|
+
On the direct `anthropic/*` route, a disabled member is left out of the order
|
|
596
|
+
multi-clawd writes. If every direct member is disabled, multi-clawd stops
|
|
597
|
+
managing that order and logs a warning naming `pool.accounts[].disabled`, but
|
|
598
|
+
it cannot empty an order OpenClaw has already stored, so clear or edit that
|
|
599
|
+
stored order yourself if the direct route must not use those profiles.
|
|
600
|
+
Changes take effect when the plugin next reads its config, for example after
|
|
601
|
+
a gateway reload. A turn already admitted or running continues on its selected
|
|
602
|
+
account; the command does not interrupt it.
|
|
603
|
+
|
|
581
604
|
`pool.strategy` defaults to `"ordered"`, which retains the configured account
|
|
582
605
|
order. Set it to `"reset-aware"` to use healthy weekly quota before its
|
|
583
606
|
`seven_day` window resets: among `ok` accounts with live seven-day reset data,
|
|
@@ -587,6 +610,20 @@ times keep pool order. The existing `minDwellMs` delay still holds an account
|
|
|
587
610
|
that remains usable. Both CLI launches and the direct Anthropic route use the
|
|
588
611
|
same preference. Other values are rejected by the plugin config schema.
|
|
589
612
|
|
|
613
|
+
`pool.sessionAffinity: true` remembers the account that successfully served each
|
|
614
|
+
Claude CLI session (`--session-id` or `--resume`) and prefers it on later turns
|
|
615
|
+
while its health is `ok` or `no_data`. A near-limit or otherwise unhealthy
|
|
616
|
+
account loses its pin; the successful replacement becomes the new pin. New
|
|
617
|
+
sessions and launches without a session id keep normal pool selection. Pins
|
|
618
|
+
live in `~/.openclaw/state/multi-clawd/pool-<id>.sessions.json`, expire after
|
|
619
|
+
30 days by default (`sessionAffinityTtlMs`), and are capped at 1000 sessions.
|
|
620
|
+
This applies to shimmed CLI launches; direct Anthropic and side-question
|
|
621
|
+
launches keep their existing routing. The shim can switch to native,
|
|
622
|
+
config-dir, and token-file accounts. On verified OpenClaw launches, the
|
|
623
|
+
gateway also resolves a pinned secret-reference account at execution time.
|
|
624
|
+
Legacy shim launches fall back to normal selection when a secret-reference
|
|
625
|
+
pin needs a different account.
|
|
626
|
+
|
|
590
627
|
How it decides, per launch (all data from each account's live
|
|
591
628
|
`rate_limit_event` stream, captured by the shim):
|
|
592
629
|
|
|
@@ -878,7 +915,7 @@ callsite (the ones static scanners flag) and why each exists.
|
|
|
878
915
|
| Network | The gateway plugin polls `https://api.anthropic.com/api/oauth/usage` for eligible pooled accounts on an interval (default 120000 ms / 2 minutes); `pool.usagePoll.enabled: false` disables that poll. `multi-clawd usage` requests the same endpoint on demand. `doctor` runs `npm view` when its registry cache is stale; install and update commands contact the package registry. Claude Code subprocesses make their own provider requests. No plugin telemetry or analytics endpoint. |
|
|
879
916
|
| Credentials | Reads declared setup-token files or gateway-resolved `oauthTokenRef` values for Claude child launches; reads each eligible account's `.credentials.json` access token for the usage poll; reads Claude login credentials from files or the macOS Keychain via the `security` CLI for login checks and runtime auth; reads account identity for file-backed logins from `.claude.json` (`~/.claude.json` for a native login, or the account config dir). Setup tokens are passed to Claude through the child environment. |
|
|
880
917
|
| Files read | `~/.openclaw/openclaw.json`, declared account config dirs and token files, Claude `.credentials.json` and `.claude.json` files as described above, and local state files. |
|
|
881
|
-
| Files written | Under `~/.openclaw/state/multi-clawd/` by default: `<id>.json`, `<id>.usage.json`, `pool-<id>.sticky.json`, `direct-order.sticky.json`, `known-models.json`, `alerts-spool.jsonl`, `watchdog.json`, `registry-check.json`, `auth-profiles-<agent>.json`, `runtime/<hash>/` (copied runtime files), and `watchdog-launcher.mjs`. The watchdog state file can be redirected with `MULTI_CLAWD_WATCHDOG_STATE`. Setup writes `~/.openclaw/openclaw.json` and a dated backup. On session handover, transcript `.jsonl` files are copied between account config dirs. The optional watchdog installs `~/.config/systemd/user/multi-clawd-eviction-watchdog.service` and `.timer` on Linux, or `~/Library/LaunchAgents/com.multi-clawd.eviction-watchdog.plist` on macOS; the unit points at the state-dir launcher. |
|
|
918
|
+
| Files written | Under `~/.openclaw/state/multi-clawd/` by default: `<id>.json`, `<id>.usage.json`, `pool-<id>.sticky.json`, `pool-<id>.sessions.json` (when affinity is enabled), `direct-order.sticky.json`, `known-models.json`, `alerts-spool.jsonl`, `watchdog.json`, `registry-check.json`, `auth-profiles-<agent>.json`, `runtime/<hash>/` (copied runtime files), and `watchdog-launcher.mjs`. The watchdog state file can be redirected with `MULTI_CLAWD_WATCHDOG_STATE`. Setup writes `~/.openclaw/openclaw.json` and a dated backup. On session handover, transcript `.jsonl` files are copied between account config dirs. The optional watchdog installs `~/.config/systemd/user/multi-clawd-eviction-watchdog.service` and `.timer` on Linux, or `~/Library/LaunchAgents/com.multi-clawd.eviction-watchdog.plist` on macOS; the unit points at the state-dir launcher. |
|
|
882
919
|
| Processes | Launches Claude Code and the node + shim transport; CLI, setup and doctor run npm, OpenClaw, Claude and OS scheduler commands; the optional watchdog checks OpenClaw logs and may restart the gateway; Hermes integration runs its Python bridge. See [SECURITY.md](SECURITY.md) for the process-call table. |
|
|
883
920
|
| Consent | The wizard asks before every write, merges non-destructively, and never overwrites an existing account entry. `--dry-run` previews everything. |
|
|
884
921
|
|
|
@@ -977,8 +1014,10 @@ Early but real — built for and dogfooded in production.
|
|
|
977
1014
|
- **v1.9** — one pool, both transports: accounts can also serve OpenClaw's
|
|
978
1015
|
direct `anthropic/*` route; the gateway keeps the `anthropic` auth order in
|
|
979
1016
|
pool-health order; `multi-clawd direct` / `direct sync` ✅
|
|
1017
|
+
- **Unreleased** — optional per-session affinity for shimmed Claude CLI
|
|
1018
|
+
launches; health overrides the pin, with bounded expiring state ✅
|
|
980
1019
|
- **Next** — standalone localhost proxy (OpenAI-compatible) so Hermes and
|
|
981
|
-
custom runtimes can share the pool;
|
|
1020
|
+
custom runtimes can share the pool; local
|
|
982
1021
|
five-hour-window signal (turn counting); per-account lock for the shim
|
|
983
1022
|
persistence race
|
|
984
1023
|
|
package/dist/direct-report.js
CHANGED
|
@@ -2,6 +2,8 @@ import { classifyAccountHealth, sevenDayReset } from "./health.js";
|
|
|
2
2
|
import { DIRECT_PROVIDER, collectDirectMembers, } from "./direct-route.js";
|
|
3
3
|
import { parseUnusableProfiles, planAgentOrder, readDirectRouteSnapshot, safeCliError, } from "./direct-sync.js";
|
|
4
4
|
export function describeDirectSource(member) {
|
|
5
|
+
if (member.disabled)
|
|
6
|
+
return "disabled by operator";
|
|
5
7
|
const s = member.source;
|
|
6
8
|
switch (s.kind) {
|
|
7
9
|
case "ref":
|
|
@@ -24,6 +26,7 @@ export async function gatherDirectStatus(params) {
|
|
|
24
26
|
return {
|
|
25
27
|
accountId: m.accountId,
|
|
26
28
|
profileId: m.profileId,
|
|
29
|
+
disabled: m.disabled,
|
|
27
30
|
verdict: classifyAccountHealth(state, params.healthOptions, params.nowMs).verdict,
|
|
28
31
|
sevenDayResetsAt: params.strategy === "reset-aware"
|
|
29
32
|
? sevenDayReset(state, params.healthOptions, params.nowMs)
|
|
@@ -44,7 +47,7 @@ export async function gatherDirectStatus(params) {
|
|
|
44
47
|
errors.push(`models status ${safeCliError(status)}`);
|
|
45
48
|
}
|
|
46
49
|
let pendingOrder;
|
|
47
|
-
if (snapshot) {
|
|
50
|
+
if (snapshot && members.some((m) => !m.disabled)) {
|
|
48
51
|
pendingOrder = planAgentOrder({
|
|
49
52
|
members: verdicts,
|
|
50
53
|
snapshot,
|
|
@@ -55,6 +58,9 @@ export async function gatherDirectStatus(params) {
|
|
|
55
58
|
strategy: params.strategy,
|
|
56
59
|
}).order;
|
|
57
60
|
}
|
|
61
|
+
else if (members.length > 0 && members.every((m) => m.disabled)) {
|
|
62
|
+
errors.push("pool.accounts[].disabled: every direct member is disabled");
|
|
63
|
+
}
|
|
58
64
|
const byProfile = new Map(unusable.map((u) => [u.profileId, u]));
|
|
59
65
|
const order = snapshot?.storedOrder ?? (params.configOrder?.length ? [...params.configOrder] : snapshot ? [] : undefined);
|
|
60
66
|
const orderSource = snapshot?.storedOrder
|
package/dist/direct-route.js
CHANGED
|
@@ -115,7 +115,7 @@ export function collectDirectMembers(accounts, poolOrder = []) {
|
|
|
115
115
|
continue;
|
|
116
116
|
}
|
|
117
117
|
seenProfiles.add(profileId);
|
|
118
|
-
members.push({ accountId: account.id, profileId, source });
|
|
118
|
+
members.push({ accountId: account.id, profileId, source, disabled: account.disabled === true });
|
|
119
119
|
}
|
|
120
120
|
return { members, problems };
|
|
121
121
|
}
|
|
@@ -128,9 +128,11 @@ const VERDICT_RANK = {
|
|
|
128
128
|
credential_failed: 4,
|
|
129
129
|
};
|
|
130
130
|
export function planDirectOrder(params) {
|
|
131
|
-
const
|
|
132
|
-
if (members.length === 0)
|
|
131
|
+
const members = params.members.filter((m) => !m.disabled);
|
|
132
|
+
if (params.members.length === 0)
|
|
133
133
|
return undefined;
|
|
134
|
+
if (members.length === 0)
|
|
135
|
+
throw new Error("pool.accounts[].disabled: every direct member is disabled");
|
|
134
136
|
const decision = decideStickySelection({
|
|
135
137
|
verdicts: members.map((m) => ({ id: m.accountId, verdict: m.verdict, sevenDayResetsAt: m.sevenDayResetsAt })),
|
|
136
138
|
sticky: params.sticky,
|
|
@@ -147,7 +149,7 @@ export function planDirectOrder(params) {
|
|
|
147
149
|
? (a.m.sevenDayResetsAt ?? Infinity) - (b.m.sevenDayResetsAt ?? Infinity)
|
|
148
150
|
: 0) || a.index - b.index)
|
|
149
151
|
.map(({ m }) => m.profileId);
|
|
150
|
-
const managed = new Set(members.map((m) => m.profileId));
|
|
152
|
+
const managed = new Set(params.members.map((m) => m.profileId));
|
|
151
153
|
const unmanaged = (params.currentOrder ?? []).filter((id, i, all) => typeof id === "string" && !managed.has(id) && all.indexOf(id) === i);
|
|
152
154
|
const order = [first.profileId, ...rest, ...unmanaged];
|
|
153
155
|
const current = params.currentOrder ?? [];
|
|
@@ -157,5 +159,5 @@ export function planDirectOrder(params) {
|
|
|
157
159
|
export function directRoutePools(accounts, directRoute) {
|
|
158
160
|
if (directRoute?.manageOrder === false)
|
|
159
161
|
return false;
|
|
160
|
-
return collectDirectMembers(accounts).members.length >= 2;
|
|
162
|
+
return collectDirectMembers(accounts).members.filter((m) => !m.disabled).length >= 2;
|
|
161
163
|
}
|
package/dist/direct-sync.js
CHANGED
|
@@ -130,6 +130,8 @@ export async function syncDirectProfiles(params) {
|
|
|
130
130
|
const stored = new Set(listed.map((p) => p.id));
|
|
131
131
|
const results = [];
|
|
132
132
|
for (const member of params.members) {
|
|
133
|
+
if (member.disabled)
|
|
134
|
+
continue;
|
|
133
135
|
const base = { accountId: member.accountId, profileId: member.profileId };
|
|
134
136
|
const { source } = member;
|
|
135
137
|
if (source.kind === "existing") {
|
|
@@ -212,14 +214,14 @@ function sameOrder(a, b) {
|
|
|
212
214
|
}
|
|
213
215
|
export function planAgentOrder(params) {
|
|
214
216
|
const { snapshot } = params;
|
|
215
|
-
const present = params.members.filter((m) => snapshot.stored.has(m.profileId));
|
|
216
|
-
const missing = params.members.filter((m) => !snapshot.stored.has(m.profileId)).map((m) => m.profileId);
|
|
217
|
-
if (present.length === 0)
|
|
217
|
+
const present = params.members.filter((m) => !m.disabled && snapshot.stored.has(m.profileId));
|
|
218
|
+
const missing = params.members.filter((m) => !m.disabled && !snapshot.stored.has(m.profileId)).map((m) => m.profileId);
|
|
219
|
+
if (present.length === 0 && params.members.some((m) => !m.disabled))
|
|
218
220
|
return { missing };
|
|
219
221
|
const explicit = snapshot.storedOrder ?? (params.configOrder?.length ? [...params.configOrder] : null);
|
|
220
222
|
const base = (explicit ?? [...snapshot.stored]).filter((id) => snapshot.stored.has(id));
|
|
221
223
|
const plan = planDirectOrder({
|
|
222
|
-
members:
|
|
224
|
+
members: params.members.map((m) => ({ ...m, disabled: m.disabled || !snapshot.stored.has(m.profileId) })),
|
|
223
225
|
currentOrder: base,
|
|
224
226
|
sticky: params.sticky,
|
|
225
227
|
nowMs: params.nowMs,
|
|
@@ -250,6 +252,7 @@ export function createDirectOrderController(deps) {
|
|
|
250
252
|
return {
|
|
251
253
|
accountId: m.accountId,
|
|
252
254
|
profileId: m.profileId,
|
|
255
|
+
disabled: m.disabled,
|
|
253
256
|
verdict: classifyAccountHealth(state, deps.healthOptions, nowMs).verdict,
|
|
254
257
|
sevenDayResetsAt: deps.strategy === "reset-aware"
|
|
255
258
|
? sevenDayReset(state, deps.healthOptions, nowMs)
|
package/dist/explain-core.js
CHANGED
|
@@ -81,7 +81,7 @@ export function renderExplanation(model) {
|
|
|
81
81
|
const lines = [];
|
|
82
82
|
lines.push("ACCOUNTS");
|
|
83
83
|
for (const acc of model.accounts) {
|
|
84
|
-
lines.push(` ${acc.id}${acc.label ? ` "${acc.label}"` : ""}`);
|
|
84
|
+
lines.push(` ${acc.id}${acc.label ? ` "${acc.label}"` : ""}${acc.disabled ? " — disabled by operator" : ""}`);
|
|
85
85
|
lines.push(` → ${describeAccount(acc)}`);
|
|
86
86
|
}
|
|
87
87
|
lines.push("");
|
|
@@ -133,12 +133,12 @@ export function renderExplanation(model) {
|
|
|
133
133
|
lines.push("RIGHT NOW");
|
|
134
134
|
for (const h of model.health) {
|
|
135
135
|
const word = VERDICT_WORDS[h.verdict] ?? h.verdict;
|
|
136
|
-
lines.push(` ${h.id}: ${word}${h.detail ? ` — ${h.detail}` : ""}`);
|
|
136
|
+
lines.push(` ${h.id}: ${model.accounts.find((a) => a.id === h.id)?.disabled ? "disabled by operator" : word}${h.detail ? ` — ${h.detail}` : ""}`);
|
|
137
137
|
lines.push(` usage: ${h.usage?.length ? renderUsageLine(h.usage, nowMs) : "no live telemetry"}`);
|
|
138
138
|
}
|
|
139
139
|
if (model.pool) {
|
|
140
140
|
const strategy = poolStrategy(model.pool.strategy);
|
|
141
|
-
const verdicts = model.pool.accounts.map((id) => {
|
|
141
|
+
const verdicts = model.pool.accounts.filter((id) => !model.accounts.find((a) => a.id === id)?.disabled).map((id) => {
|
|
142
142
|
const h = model.health.find((row) => row.id === id);
|
|
143
143
|
return {
|
|
144
144
|
id,
|
|
@@ -167,7 +167,7 @@ export function renderExplanation(model) {
|
|
|
167
167
|
lines.push(` strategy: ${strategy} — picked ${decision.account}: ${reason}`);
|
|
168
168
|
}
|
|
169
169
|
else {
|
|
170
|
-
lines.push(` strategy: ${strategy}`);
|
|
170
|
+
lines.push(` strategy: ${strategy} — pool.accounts[].disabled: every pool member is disabled`);
|
|
171
171
|
}
|
|
172
172
|
lines.push(model.stickyAccount
|
|
173
173
|
? strategy === "reset-aware"
|
package/dist/index.js
CHANGED
|
@@ -9,6 +9,7 @@ import { resolveExecMode, permissionModeArgs } from "./exec-policy.js";
|
|
|
9
9
|
import { resolveBaseModelIds } from "./catalog-source.js";
|
|
10
10
|
import { allCredentialFailed, classifyAccountHealth, credentialFailureFor, lastAuthSuccessAt, overdueProbeAccount, sevenDayReset, } from "./health.js";
|
|
11
11
|
import { decideStickySelection } from "./sticky.js";
|
|
12
|
+
import { DEFAULT_SESSION_AFFINITY_TTL_MS, launchSessionId, readSessionPin, SESSION_AFFINITY_ENV, } from "./session-affinity.js";
|
|
12
13
|
import { mergeHealthStates, parseStoredState, recordCredentialFailure, } from "./shim-core.js";
|
|
13
14
|
import { createTokenRefResolver, isSecretRefShape, } from "./token-resolution.js";
|
|
14
15
|
import { resolveSecretRefValues } from "openclaw/plugin-sdk/secret-ref-runtime";
|
|
@@ -436,6 +437,7 @@ export function resolveBackendLaunch(params) {
|
|
|
436
437
|
const SHIM_ONLY_ENV = [
|
|
437
438
|
RETRY_ROSTER_ENV,
|
|
438
439
|
SESSION_DIRS_ENV,
|
|
440
|
+
SESSION_AFFINITY_ENV,
|
|
439
441
|
"MULTI_CLAWD_MODEL_OVERRIDE",
|
|
440
442
|
CLAUDE_BIN_ENV,
|
|
441
443
|
CLAUDE_ARGV0_ENV,
|
|
@@ -572,6 +574,8 @@ const launchTransports = new WeakMap();
|
|
|
572
574
|
export function buildRetryRoster(members, launchedId) {
|
|
573
575
|
const roster = [];
|
|
574
576
|
for (const member of members) {
|
|
577
|
+
if (member.disabled === true)
|
|
578
|
+
continue;
|
|
575
579
|
if (member.id === launchedId)
|
|
576
580
|
continue;
|
|
577
581
|
const tokenFile = !member.native && member.oauthTokenFile ? expandHome(member.oauthTokenFile) : undefined;
|
|
@@ -785,9 +789,10 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
785
789
|
return;
|
|
786
790
|
const poolId = pool.id?.trim() || "clawd";
|
|
787
791
|
const memberIds = (pool.accounts ?? []).filter((id) => registeredIds.has(id));
|
|
788
|
-
const
|
|
792
|
+
const configuredMembers = memberIds
|
|
789
793
|
.map((id) => accounts.find((a) => a.id.trim() === id))
|
|
790
794
|
.filter((a) => a !== undefined);
|
|
795
|
+
const members = configuredMembers.filter((a) => a.disabled !== true);
|
|
791
796
|
const ladder = (pool.degrade?.ladder ?? []).filter((m) => {
|
|
792
797
|
if (isModernClaudeModelId(m))
|
|
793
798
|
return true;
|
|
@@ -795,7 +800,10 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
795
800
|
return false;
|
|
796
801
|
});
|
|
797
802
|
const pins = pool.degrade?.pins ?? [];
|
|
798
|
-
if (members.length
|
|
803
|
+
if (members.length === 0 && memberIds.length > 0) {
|
|
804
|
+
logger.error(`[multi-clawd] pool "${poolId}": every member has pool.accounts[].disabled: true`);
|
|
805
|
+
}
|
|
806
|
+
if (members.length < 2 && !(members.length === 0 && memberIds.length > 0) && !(members.length === 1 && (ladder.length > 0 || memberIds.length > 1))) {
|
|
799
807
|
logger.warn(`[multi-clawd] pool "${poolId}" has ${members.length} registered account(s) — need at least 2 (or 1 with a degrade ladder); pool not registered`);
|
|
800
808
|
return;
|
|
801
809
|
}
|
|
@@ -820,6 +828,8 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
820
828
|
const backend = buildBackend(poolAccount, execMode, deps?.launch);
|
|
821
829
|
const { launch, transport } = launchTransports.get(backend);
|
|
822
830
|
backend.prepareExecution = async (ctx) => {
|
|
831
|
+
if (members.length === 0)
|
|
832
|
+
throw new Error(`[multi-clawd] pool "${poolId}": every member has pool.accounts[].disabled: true`);
|
|
823
833
|
const now = Date.now();
|
|
824
834
|
const requestedModel = canonicalModelId(ctx.modelId) ?? ctx.modelId;
|
|
825
835
|
const states = members.map((a) => ({ id: a.id, state: readHealthState(a.id) }));
|
|
@@ -964,7 +974,17 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
964
974
|
if (roster.length > 0)
|
|
965
975
|
env[RETRY_ROSTER_ENV] = JSON.stringify(roster);
|
|
966
976
|
}
|
|
967
|
-
env[SESSION_DIRS_ENV] = JSON.stringify([...new Set(
|
|
977
|
+
env[SESSION_DIRS_ENV] = JSON.stringify([...new Set(configuredMembers.map(accountConfigDir))]);
|
|
978
|
+
if (pool.sessionAffinity) {
|
|
979
|
+
env[SESSION_AFFINITY_ENV] = JSON.stringify({
|
|
980
|
+
file: join(homedir(), ".openclaw", "state", "multi-clawd", `pool-${poolId}.sessions.json`),
|
|
981
|
+
ttlMs: typeof pool.sessionAffinityTtlMs === "number" &&
|
|
982
|
+
Number.isFinite(pool.sessionAffinityTtlMs) && pool.sessionAffinityTtlMs > 0
|
|
983
|
+
? pool.sessionAffinityTtlMs : DEFAULT_SESSION_AFFINITY_TTL_MS,
|
|
984
|
+
verdicts: poolVerdicts.map((v) => ({ id: v.id, verdict: v.verdict })),
|
|
985
|
+
roster: buildRetryRoster(members, ""),
|
|
986
|
+
});
|
|
987
|
+
}
|
|
968
988
|
if (ladder.length > 0) {
|
|
969
989
|
const pinned = matchesPin(pins, {
|
|
970
990
|
agentDir: ctx.agentDir ?? "",
|
|
@@ -992,7 +1012,38 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
992
1012
|
logger.info(`[multi-clawd] pool ${poolId}: pinned lane keeps ${ctx.modelId} despite exhausted pool (will fail over via the chain)`);
|
|
993
1013
|
}
|
|
994
1014
|
}
|
|
995
|
-
|
|
1015
|
+
const affinityTransport = pool.sessionAffinity && transport && launch.kind === "verified"
|
|
1016
|
+
? async function* (execution) {
|
|
1017
|
+
const id = launchSessionId(execution.args);
|
|
1018
|
+
const pin = id ? readSessionPin(join(homedir(), ".openclaw", "state", "multi-clawd", `pool-${poolId}.sessions.json`), id, Date.now(), typeof pool.sessionAffinityTtlMs === "number" && pool.sessionAffinityTtlMs > 0
|
|
1019
|
+
? pool.sessionAffinityTtlMs : DEFAULT_SESSION_AFFINITY_TTL_MS) : undefined;
|
|
1020
|
+
const member = members.find((m) => m.id === pin?.account && m.oauthTokenRef);
|
|
1021
|
+
if (member && member.id !== execution.env.MULTI_CLAWD_ACCOUNT_ID &&
|
|
1022
|
+
poolVerdicts.some((v) => v.id === member.id &&
|
|
1023
|
+
(v.verdict === "ok" || v.verdict === "no_data"))) {
|
|
1024
|
+
let accountEnv;
|
|
1025
|
+
try {
|
|
1026
|
+
accountEnv = await buildAccountEnv(member, deps?.resolver);
|
|
1027
|
+
}
|
|
1028
|
+
catch {
|
|
1029
|
+
}
|
|
1030
|
+
if (accountEnv) {
|
|
1031
|
+
const selectedEnv = { ...execution.env };
|
|
1032
|
+
delete selectedEnv.CLAUDE_CODE_OAUTH_TOKEN;
|
|
1033
|
+
delete selectedEnv.CLAUDE_CONFIG_DIR;
|
|
1034
|
+
const pinnedRoster = buildRetryRoster(members, member.id);
|
|
1035
|
+
if (pinnedRoster.length > 0)
|
|
1036
|
+
selectedEnv[RETRY_ROSTER_ENV] = JSON.stringify(pinnedRoster);
|
|
1037
|
+
else
|
|
1038
|
+
delete selectedEnv[RETRY_ROSTER_ENV];
|
|
1039
|
+
yield* transport({ ...execution, env: { ...selectedEnv, ...accountEnv } });
|
|
1040
|
+
return;
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
yield* transport(execution);
|
|
1044
|
+
}
|
|
1045
|
+
: transport;
|
|
1046
|
+
return preparedForLaunch(launch, ctx, env, affinityTransport);
|
|
996
1047
|
};
|
|
997
1048
|
api.registerCliBackend(backend);
|
|
998
1049
|
api.registerProvider(buildCatalogProvider(poolAccount, members));
|
|
@@ -1031,6 +1082,11 @@ export function startDirectOrderSync(params) {
|
|
|
1031
1082
|
stopDirectOrderSync();
|
|
1032
1083
|
return { active: false, members: members.map((m) => m.accountId) };
|
|
1033
1084
|
}
|
|
1085
|
+
if (members.every((m) => m.disabled)) {
|
|
1086
|
+
stopDirectOrderSync();
|
|
1087
|
+
logger.warn("[multi-clawd] direct route: pool.accounts[].disabled: every direct member is disabled");
|
|
1088
|
+
return { active: false, members: [] };
|
|
1089
|
+
}
|
|
1034
1090
|
const requested = params.directRoute?.agents?.length ? params.directRoute.agents : ["main"];
|
|
1035
1091
|
const agents = requested.filter((a) => {
|
|
1036
1092
|
if (isValidAgentId(a))
|
|
@@ -1050,7 +1106,7 @@ export function startDirectOrderSync(params) {
|
|
|
1050
1106
|
rotateOnOverage: params.pool?.rotateOnOverage,
|
|
1051
1107
|
};
|
|
1052
1108
|
const signature = JSON.stringify({
|
|
1053
|
-
members: members.map((
|
|
1109
|
+
members: members.map(({ accountId, profileId, source, disabled }) => ({ accountId, profileId, source, disabled })),
|
|
1054
1110
|
agents,
|
|
1055
1111
|
intervalMs,
|
|
1056
1112
|
healthOptions,
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
2
|
+
import { closeSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import { resumeSessionId } from "./session-handover.js";
|
|
5
|
+
export const SESSION_AFFINITY_ENV = "MULTI_CLAWD_SESSION_AFFINITY";
|
|
6
|
+
export const DEFAULT_SESSION_AFFINITY_TTL_MS = 30 * 24 * 60 * 60 * 1000;
|
|
7
|
+
export const MAX_SESSION_PINS = 1000;
|
|
8
|
+
export function launchSessionId(argv) {
|
|
9
|
+
const resumed = resumeSessionId(argv);
|
|
10
|
+
if (resumed)
|
|
11
|
+
return resumed;
|
|
12
|
+
for (let i = 0; i < argv.length; i++) {
|
|
13
|
+
const arg = argv[i];
|
|
14
|
+
const value = arg === "--session-id" ? argv[i + 1]
|
|
15
|
+
: arg.startsWith("--session-id=") ? arg.slice("--session-id=".length) : undefined;
|
|
16
|
+
if (value !== undefined)
|
|
17
|
+
return /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(value) ? value : undefined;
|
|
18
|
+
}
|
|
19
|
+
return undefined;
|
|
20
|
+
}
|
|
21
|
+
export function eligibleSessionAccount(pin, currentAccount, verdicts, availableAccounts) {
|
|
22
|
+
if (!pin)
|
|
23
|
+
return undefined;
|
|
24
|
+
if (!verdicts.some((v) => v.id === pin.account &&
|
|
25
|
+
(v.verdict === "ok" || v.verdict === "no_data")))
|
|
26
|
+
return undefined;
|
|
27
|
+
return pin.account === currentAccount || availableAccounts.includes(pin.account)
|
|
28
|
+
? pin.account : undefined;
|
|
29
|
+
}
|
|
30
|
+
const WRITE_CLOCK_SLACK_MS = 60_000;
|
|
31
|
+
function readPins(file, now, ttlMs, futureSlackMs = 0) {
|
|
32
|
+
try {
|
|
33
|
+
const parsed = JSON.parse(readFileSync(file, "utf8"));
|
|
34
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
35
|
+
return {};
|
|
36
|
+
const pins = {};
|
|
37
|
+
for (const [id, value] of Object.entries(parsed)) {
|
|
38
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(id))
|
|
39
|
+
continue;
|
|
40
|
+
if (!value || typeof value !== "object")
|
|
41
|
+
continue;
|
|
42
|
+
const pin = value;
|
|
43
|
+
if (typeof pin.account !== "string" || !pin.account ||
|
|
44
|
+
typeof pin.servedAt !== "number" || !Number.isFinite(pin.servedAt) ||
|
|
45
|
+
pin.servedAt > now + futureSlackMs || now - pin.servedAt >= ttlMs)
|
|
46
|
+
continue;
|
|
47
|
+
pins[id] = { account: pin.account, servedAt: pin.servedAt };
|
|
48
|
+
}
|
|
49
|
+
return pins;
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
return {};
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
export function readSessionPin(file, sessionId, now, ttlMs) {
|
|
56
|
+
return readPins(file, now, ttlMs)[sessionId];
|
|
57
|
+
}
|
|
58
|
+
export function recordSessionPin(file, sessionId, account, now, ttlMs) {
|
|
59
|
+
const lock = `${file}.lock`;
|
|
60
|
+
let locked = false;
|
|
61
|
+
let tmp;
|
|
62
|
+
try {
|
|
63
|
+
mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
|
|
64
|
+
const deadline = Date.now() + 1000;
|
|
65
|
+
const wait = new Int32Array(new SharedArrayBuffer(4));
|
|
66
|
+
while (!locked) {
|
|
67
|
+
try {
|
|
68
|
+
closeSync(openSync(lock, "wx", 0o600));
|
|
69
|
+
locked = true;
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
try {
|
|
73
|
+
if (Date.now() - statSync(lock).mtimeMs > 5000)
|
|
74
|
+
rmSync(lock, { force: true });
|
|
75
|
+
}
|
|
76
|
+
catch { }
|
|
77
|
+
if (Date.now() >= deadline)
|
|
78
|
+
return;
|
|
79
|
+
Atomics.wait(wait, 0, 0, 10);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
const pins = readPins(file, now, ttlMs, WRITE_CLOCK_SLACK_MS);
|
|
83
|
+
pins[sessionId] = { account, servedAt: now };
|
|
84
|
+
const entries = Object.entries(pins);
|
|
85
|
+
const bounded = entries.length > MAX_SESSION_PINS
|
|
86
|
+
? entries.sort((a, b) => b[1].servedAt - a[1].servedAt).slice(0, MAX_SESSION_PINS)
|
|
87
|
+
: entries;
|
|
88
|
+
tmp = `${file}.tmp-${process.pid}-${randomBytes(6).toString("hex")}`;
|
|
89
|
+
writeFileSync(tmp, JSON.stringify(Object.fromEntries(bounded)), { mode: 0o600 });
|
|
90
|
+
renameSync(tmp, file);
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
}
|
|
94
|
+
finally {
|
|
95
|
+
if (tmp) {
|
|
96
|
+
try {
|
|
97
|
+
rmSync(tmp, { force: true });
|
|
98
|
+
}
|
|
99
|
+
catch { }
|
|
100
|
+
}
|
|
101
|
+
if (locked) {
|
|
102
|
+
try {
|
|
103
|
+
rmSync(lock, { force: true });
|
|
104
|
+
}
|
|
105
|
+
catch { }
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
package/dist/shim.js
CHANGED
|
@@ -6,6 +6,7 @@ import { rewriteModelArg } from "./degrade.js";
|
|
|
6
6
|
import { parseModelLimitError, recordModelLimit } from "./shim-core.js";
|
|
7
7
|
import { canonicalModelId } from "./models.js";
|
|
8
8
|
import { buildRetryEnv, chooseLaunchableRetryAccount, materializeRetryAccount, parseRetryRoster, retryArming, RETRY_ROSTER_ENV, } from "./retry-plan.js";
|
|
9
|
+
import { eligibleSessionAccount, launchSessionId, readSessionPin, recordSessionPin, SESSION_AFFINITY_ENV, } from "./session-affinity.js";
|
|
9
10
|
import { CLAUDE_ARGV0_ENV, CLAUDE_BIN_ENV, parseClaudeBinOverride, resolveExecutable, } from "./claude-command.js";
|
|
10
11
|
import { effectiveConfigDir, handoverForLaunch, handoverSnapshotTo, parseSessionDirs, SESSION_DIRS_ENV, snapshotResumeTranscript, } from "./session-handover.js";
|
|
11
12
|
function resolveClaudeCommand() {
|
|
@@ -104,15 +105,53 @@ if (modelOverride) {
|
|
|
104
105
|
process.stderr.write(`[multi-clawd shim] degrading model for this launch → ${modelOverride}\n`);
|
|
105
106
|
}
|
|
106
107
|
}
|
|
108
|
+
const sessionId = launchSessionId(childArgs);
|
|
109
|
+
let affinity;
|
|
110
|
+
let launchEnv = process.env;
|
|
111
|
+
try {
|
|
112
|
+
const raw = process.env[SESSION_AFFINITY_ENV];
|
|
113
|
+
if (raw && sessionId) {
|
|
114
|
+
const config = JSON.parse(raw);
|
|
115
|
+
if (typeof config.file === "string" && config.file && typeof config.ttlMs === "number" &&
|
|
116
|
+
Number.isFinite(config.ttlMs) && config.ttlMs > 0 && Array.isArray(config.verdicts)) {
|
|
117
|
+
affinity = {
|
|
118
|
+
file: config.file,
|
|
119
|
+
ttlMs: config.ttlMs,
|
|
120
|
+
verdicts: config.verdicts,
|
|
121
|
+
roster: parseRetryRoster(JSON.stringify(config.roster)),
|
|
122
|
+
};
|
|
123
|
+
const pin = readSessionPin(affinity.file, sessionId, Date.now(), affinity.ttlMs);
|
|
124
|
+
const eligible = eligibleSessionAccount(pin, accountId, affinity.verdicts, affinity.roster.map((r) => r.id));
|
|
125
|
+
if (eligible && eligible !== accountId) {
|
|
126
|
+
const member = affinity.roster.find((r) => r.id === eligible);
|
|
127
|
+
if (member) {
|
|
128
|
+
const ready = materializeRetryAccount(member, (path) => readFileSync(path, "utf8"));
|
|
129
|
+
if ("account" in ready) {
|
|
130
|
+
launchEnv = buildRetryEnv(process.env, ready.account);
|
|
131
|
+
accountId = ready.account.id;
|
|
132
|
+
stateFile = ready.account.stateFile;
|
|
133
|
+
state = { accountId, windows: {} };
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
affinity = undefined;
|
|
142
|
+
}
|
|
143
|
+
delete launchEnv[SESSION_AFFINITY_ENV];
|
|
107
144
|
{
|
|
108
|
-
const note = handoverForLaunch(childArgs,
|
|
145
|
+
const note = handoverForLaunch(childArgs, launchEnv);
|
|
109
146
|
if (note)
|
|
110
147
|
process.stderr.write(`[multi-clawd shim] ${note}\n`);
|
|
111
148
|
}
|
|
112
|
-
const retryRoster =
|
|
149
|
+
const retryRoster = affinity && launchEnv !== process.env
|
|
150
|
+
? affinity.roster.filter((r) => r.id !== accountId)
|
|
151
|
+
: parseRetryRoster(process.env[RETRY_ROSTER_ENV]);
|
|
113
152
|
delete process.env[RETRY_ROSTER_ENV];
|
|
114
|
-
const resumeSnapshot = retryRoster.length > 0 && parseSessionDirs(
|
|
115
|
-
? snapshotResumeTranscript(childArgs,
|
|
153
|
+
const resumeSnapshot = retryRoster.length > 0 && parseSessionDirs(launchEnv[SESSION_DIRS_ENV]).length > 0
|
|
154
|
+
? snapshotResumeTranscript(childArgs, launchEnv)
|
|
116
155
|
: undefined;
|
|
117
156
|
const arming = retryArming(childArgs, retryRoster, { resumeReady: resumeSnapshot !== undefined });
|
|
118
157
|
let retryArmed = arming.armed;
|
|
@@ -121,7 +160,7 @@ if (!retryArmed && retryRoster.length > 0 && arming.reason) {
|
|
|
121
160
|
}
|
|
122
161
|
let child = spawn(command, childArgs, {
|
|
123
162
|
stdio: ["pipe", "pipe", "inherit"],
|
|
124
|
-
env:
|
|
163
|
+
env: launchEnv,
|
|
125
164
|
...(claudeArgv0 ? { argv0: claudeArgv0 } : {}),
|
|
126
165
|
});
|
|
127
166
|
const STDIN_REPLAY_CAP_BYTES = 8 * 1024 * 1024;
|
|
@@ -301,7 +340,7 @@ function attemptRetry() {
|
|
|
301
340
|
sawSuccessResult = false;
|
|
302
341
|
child = spawn(command, childArgs, {
|
|
303
342
|
stdio: ["pipe", "pipe", "inherit"],
|
|
304
|
-
env: buildRetryEnv(
|
|
343
|
+
env: buildRetryEnv(launchEnv, target),
|
|
305
344
|
...(claudeArgv0 ? { argv0: claudeArgv0 } : {}),
|
|
306
345
|
});
|
|
307
346
|
attachChildHandlers();
|
|
@@ -419,6 +458,9 @@ function onChildClose(code, signal) {
|
|
|
419
458
|
if ((code ?? 0) === 0) {
|
|
420
459
|
clearRecordedAuthFailureOnSuccess();
|
|
421
460
|
clearRecordedModelRejectionOnSuccess();
|
|
461
|
+
if (affinity && sessionId && sawSuccessResult && !sawAuthFailure && !sawModelLimit) {
|
|
462
|
+
recordSessionPin(affinity.file, sessionId, accountId, Date.now(), affinity.ttlMs);
|
|
463
|
+
}
|
|
422
464
|
}
|
|
423
465
|
process.exit(code ?? 0);
|
|
424
466
|
}
|
package/openclaw.plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "multi-clawd",
|
|
3
3
|
"name": "multi-clawd",
|
|
4
|
-
"version": "2.
|
|
4
|
+
"version": "2.4.0",
|
|
5
5
|
"description": "Register additional Claude Code logins (Max/Pro accounts) as first-class OpenClaw CLI backends for cross-account failover, keeping the full skills/MCP harness on every account.",
|
|
6
6
|
"cliBackends": [
|
|
7
7
|
"claw1",
|
|
@@ -70,6 +70,15 @@
|
|
|
70
70
|
"type": "number",
|
|
71
71
|
"description": "After rotating away from home, stay on the rotated-to account at least this long before returning home (anti-flap hysteresis; health always overrides). Default 600000 (10min)."
|
|
72
72
|
},
|
|
73
|
+
"sessionAffinity": {
|
|
74
|
+
"type": "boolean",
|
|
75
|
+
"description": "Pin a Claude CLI session to the account that last served it, while that account remains healthy. CLI launches only. Default false."
|
|
76
|
+
},
|
|
77
|
+
"sessionAffinityTtlMs": {
|
|
78
|
+
"type": "number",
|
|
79
|
+
"exclusiveMinimum": 0,
|
|
80
|
+
"description": "Successful session pin lifetime in milliseconds. Default 2592000000 (30 days); only used when sessionAffinity is true."
|
|
81
|
+
},
|
|
73
82
|
"usagePoll": {
|
|
74
83
|
"type": "object",
|
|
75
84
|
"additionalProperties": false,
|
|
@@ -152,6 +161,11 @@
|
|
|
152
161
|
"type": "string",
|
|
153
162
|
"description": "Human-friendly label shown in status/catalog."
|
|
154
163
|
},
|
|
164
|
+
"disabled": {
|
|
165
|
+
"type": "boolean",
|
|
166
|
+
"default": false,
|
|
167
|
+
"description": "Exclude this account from pool selection, in-turn retries, session affinity and direct Anthropic order while keeping it configured and visible in status. Takes effect when the plugin next reads config."
|
|
168
|
+
},
|
|
155
169
|
"native": {
|
|
156
170
|
"type": "boolean",
|
|
157
171
|
"description": "Use the machine's native Claude login (default config dir; OS keychain on macOS). Sets neither CLAUDE_CONFIG_DIR nor a token. Use this to include the main account in the rotation pool."
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@drakon-systems/multi-clawd",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "Multi-account Claude Code failover for OpenClaw — register additional Claude (Max/Pro) logins as first-class CLI backends and keep the full skills/MCP harness across every account. Also imports those accounts' setup tokens into Hermes Agent's Anthropic credential pool.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { constants, copyFileSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { homedir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
|
|
5
|
+
/** Locate JSON object properties without reserializing the operator's config. */
|
|
6
|
+
export function jsonSpans(source, offset = 0) {
|
|
7
|
+
let at = offset;
|
|
8
|
+
const ws = () => { while (/\s/.test(source[at] ?? "")) at++; };
|
|
9
|
+
function string() {
|
|
10
|
+
const start = at++;
|
|
11
|
+
while (at < source.length) {
|
|
12
|
+
if (source[at] === "\\") { at += 2; continue; }
|
|
13
|
+
if (source[at++] === '"') return { value: JSON.parse(source.slice(start, at)), start, end: at };
|
|
14
|
+
}
|
|
15
|
+
throw new Error("unterminated JSON string");
|
|
16
|
+
}
|
|
17
|
+
function value() {
|
|
18
|
+
ws();
|
|
19
|
+
const start = at;
|
|
20
|
+
if (source[at] === "{") {
|
|
21
|
+
at++; ws(); const properties = [];
|
|
22
|
+
while (source[at] !== "}") {
|
|
23
|
+
const key = string(); ws();
|
|
24
|
+
if (source[at++] !== ":") throw new Error("invalid JSON object");
|
|
25
|
+
const child = value(); ws();
|
|
26
|
+
const end = at;
|
|
27
|
+
properties.push({ key: key.value, keyStart: key.start, start: child.start, end: child.end, propertyEnd: end });
|
|
28
|
+
if (source[at] !== ",") break;
|
|
29
|
+
at++; ws();
|
|
30
|
+
}
|
|
31
|
+
if (source[at++] !== "}") throw new Error("invalid JSON object");
|
|
32
|
+
return { start, end: at, properties };
|
|
33
|
+
}
|
|
34
|
+
if (source[at] === "[") {
|
|
35
|
+
at++; ws(); const items = [];
|
|
36
|
+
while (source[at] !== "]") {
|
|
37
|
+
items.push(value()); ws();
|
|
38
|
+
if (source[at] !== ",") break;
|
|
39
|
+
at++; ws();
|
|
40
|
+
}
|
|
41
|
+
if (source[at++] !== "]") throw new Error("invalid JSON array");
|
|
42
|
+
return { start, end: at, items };
|
|
43
|
+
}
|
|
44
|
+
if (source[at] === '"') return string();
|
|
45
|
+
while (at < source.length && !/[\s,}\]]/.test(source[at])) at++;
|
|
46
|
+
return { start, end: at };
|
|
47
|
+
}
|
|
48
|
+
const root = value(); ws();
|
|
49
|
+
if (offset === 0 && at !== source.length) throw new Error("trailing JSON content");
|
|
50
|
+
return root;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const property = (node, key) => {
|
|
54
|
+
if (!node?.properties) throw new Error("expected JSON object");
|
|
55
|
+
const matches = node.properties.filter((p) => p.key === key);
|
|
56
|
+
if (matches.length > 1) throw new Error(`duplicate ${key} property`);
|
|
57
|
+
return matches[0];
|
|
58
|
+
};
|
|
59
|
+
const child = (source, node, key) => {
|
|
60
|
+
const p = property(node, key);
|
|
61
|
+
return p && jsonSpans(source, p.start);
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
export function editAccountDisabled(source, id, disabled) {
|
|
65
|
+
const original = JSON.parse(source);
|
|
66
|
+
let node = jsonSpans(source);
|
|
67
|
+
for (const key of ["plugins", "entries", "multi-clawd", "config", "accounts"]) {
|
|
68
|
+
node = child(source, node, key);
|
|
69
|
+
if (!node) throw new Error(`missing plugins.entries["multi-clawd"].config.accounts`);
|
|
70
|
+
}
|
|
71
|
+
if (!node.items) throw new Error("accounts must be an array");
|
|
72
|
+
const accounts = original?.plugins?.entries?.["multi-clawd"]?.config?.accounts;
|
|
73
|
+
const matches = accounts?.map((a, i) => a?.id === id ? i : -1).filter((i) => i >= 0) ?? [];
|
|
74
|
+
if (matches.length === 0) throw new Error(`unknown account id "${id}"`);
|
|
75
|
+
if (matches.length !== 1) throw new Error(`duplicate account id "${id}"`);
|
|
76
|
+
const account = node.items[matches[0]];
|
|
77
|
+
const flag = property(account, "disabled");
|
|
78
|
+
if (flag && typeof accounts[matches[0]].disabled !== "boolean") throw new Error("disabled must be a boolean");
|
|
79
|
+
let edited;
|
|
80
|
+
if (flag) {
|
|
81
|
+
edited = source.slice(0, flag.start) + String(disabled) + source.slice(flag.end);
|
|
82
|
+
} else if (disabled) {
|
|
83
|
+
const indent = source.slice(source.lastIndexOf("\n", account.start - 1) + 1, account.start);
|
|
84
|
+
const space = source.slice(account.start + 1, account.end - 1);
|
|
85
|
+
const multiline = space.includes("\n");
|
|
86
|
+
const addition = account.properties.length
|
|
87
|
+
? `,${multiline ? `\n${indent} ` : " "}"disabled": true`
|
|
88
|
+
: `"disabled": true`;
|
|
89
|
+
edited = source.slice(0, account.end - 1) + addition + source.slice(account.end - 1);
|
|
90
|
+
} else {
|
|
91
|
+
return { edited: source, changed: false };
|
|
92
|
+
}
|
|
93
|
+
const expected = structuredClone(original);
|
|
94
|
+
expected.plugins.entries["multi-clawd"].config.accounts[matches[0]].disabled = disabled;
|
|
95
|
+
if (JSON.stringify(JSON.parse(edited)) !== JSON.stringify(expected)) throw new Error("targeted edit changed other config values");
|
|
96
|
+
return { edited, changed: edited !== source };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function changeAccount(id, disabled, dryRun = false, configFile = join(homedir(), ".openclaw", "openclaw.json")) {
|
|
100
|
+
const source = readFileSync(configFile, "utf8");
|
|
101
|
+
const { edited, changed } = editAccountDisabled(source, id, disabled);
|
|
102
|
+
if (dryRun || !changed) return { changed, dryRun };
|
|
103
|
+
if (readFileSync(configFile, "utf8") !== source) throw new Error("config changed during edit; retry");
|
|
104
|
+
const backup = `${configFile}.backup-${new Date().toISOString().replace(/[:.]/g, "-")}`;
|
|
105
|
+
copyFileSync(configFile, backup, constants.COPYFILE_EXCL);
|
|
106
|
+
const temporary = `${configFile}.tmp-${process.pid}-${Date.now()}`;
|
|
107
|
+
try {
|
|
108
|
+
writeFileSync(temporary, edited, { mode: statSync(configFile).mode & 0o777 });
|
|
109
|
+
if (readFileSync(configFile, "utf8") !== source) throw new Error("config changed during edit; retry");
|
|
110
|
+
renameSync(temporary, configFile);
|
|
111
|
+
} finally {
|
|
112
|
+
rmSync(temporary, { force: true });
|
|
113
|
+
}
|
|
114
|
+
return { changed, dryRun, backup };
|
|
115
|
+
}
|
package/scripts/cli.mjs
CHANGED
|
@@ -36,6 +36,7 @@ ${BOLD}🦞 multi-clawd${RESET} — multi-account Claude failover for OpenClaw
|
|
|
36
36
|
|
|
37
37
|
${BOLD}setup${RESET} guided setup wizard (accounts, pool, watchdog)
|
|
38
38
|
${BOLD}login${RESET} log a configured account in (or re-auth it) — right dir, right env
|
|
39
|
+
${BOLD}account${RESET} disable or enable a pool account (add --dry-run to preview)
|
|
39
40
|
${BOLD}explain${RESET} your setup in plain English — accounts, pool, fallback chain
|
|
40
41
|
${BOLD}chain${RESET} audit your model routing — what actually serves each turn
|
|
41
42
|
${BOLD}direct${RESET} the direct anthropic/* route — status, or \`direct sync\` to store profiles
|
|
@@ -760,7 +761,7 @@ async function usageCommand(args) {
|
|
|
760
761
|
const rows = [];
|
|
761
762
|
const resetById = new Map();
|
|
762
763
|
for (const a of accounts) {
|
|
763
|
-
const row = { id: a.id, label: a.label };
|
|
764
|
+
const row = { id: a.id, label: a.label, ...(a.disabled === true ? { disabled: true, status: "disabled by operator" } : {}) };
|
|
764
765
|
if (a.oauthTokenFile || a.oauthTokenRef) {
|
|
765
766
|
row.skipped = "setup-token login — not polled: the usage endpoint needs the user:profile scope, setup-tokens carry only user:inference (stream telemetry only)";
|
|
766
767
|
} else if (!a.native && !a.configDir) {
|
|
@@ -794,15 +795,15 @@ async function usageCommand(args) {
|
|
|
794
795
|
}
|
|
795
796
|
let next;
|
|
796
797
|
if (pool) {
|
|
797
|
-
const verdicts = pool.accounts.map((id) => ({ id, verdict: rows.find((r) => r.id === id)?.verdict ?? "no_data",
|
|
798
|
+
const verdicts = pool.accounts.filter((id) => !accounts.find((a) => a.id === id)?.disabled).map((id) => ({ id, verdict: rows.find((r) => r.id === id)?.verdict ?? "no_data",
|
|
798
799
|
sevenDayResetsAt: resetById.get(id) }));
|
|
799
800
|
if (pool.strategy === "reset-aware") {
|
|
800
801
|
const stickySelection = await import(resolve(__dirname, "..", "dist", "sticky.js"));
|
|
801
802
|
let sticky;
|
|
802
803
|
try { sticky = JSON.parse(rf(join(stateDir, `pool-${pool.id}.sticky.json`), "utf8")); } catch { /* none */ }
|
|
803
|
-
next = stickySelection.decideStickySelection({ verdicts, sticky, nowMs: now, minDwellMs: pool.minDwellMs, strategy: pool.strategy }).account;
|
|
804
|
+
next = verdicts.length ? stickySelection.decideStickySelection({ verdicts, sticky, nowMs: now, minDwellMs: pool.minDwellMs, strategy: pool.strategy }).account : undefined;
|
|
804
805
|
} else {
|
|
805
|
-
next = health.choosePoolAccount(verdicts);
|
|
806
|
+
next = verdicts.length ? health.choosePoolAccount(verdicts) : undefined;
|
|
806
807
|
}
|
|
807
808
|
}
|
|
808
809
|
if (args.includes("--json")) {
|
|
@@ -822,7 +823,7 @@ async function usageCommand(args) {
|
|
|
822
823
|
};
|
|
823
824
|
console.log(`\n${BOLD}🦞 multi-clawd — live usage${RESET} ${DIM}(rotate at ${pct(rotateAt)}${weeklyRotateAt !== undefined ? `, weekly ${pct(weeklyRotateAt)}` : ""}, warn at ${pct(warnThreshold)})${RESET}\n`);
|
|
824
825
|
for (const r of rows) {
|
|
825
|
-
console.log(` ${BOLD}${r.id}${RESET}${r.label ? ` ${DIM}— ${r.label}${RESET}` : ""} pool verdict: ${r.verdict}${r.reason ? ` ${DIM}(${r.reason})${RESET}` : ""}`);
|
|
826
|
+
console.log(` ${BOLD}${r.id}${RESET}${r.label ? ` ${DIM}— ${r.label}${RESET}` : ""} pool verdict: ${r.disabled ? "disabled by operator" : r.verdict}${r.reason ? ` ${DIM}(${r.reason})${RESET}` : ""}`);
|
|
826
827
|
if (r.snapshot) {
|
|
827
828
|
for (const [w, u] of Object.entries(r.snapshot.windows)) {
|
|
828
829
|
console.log(` ${mark(u.utilization, w)} ${up.usageWindowLabel(w).padEnd(12)} ${pct(u.utilization).padStart(4)}${until(u.resetsAt)}`);
|
|
@@ -838,7 +839,10 @@ async function usageCommand(args) {
|
|
|
838
839
|
console.log("");
|
|
839
840
|
}
|
|
840
841
|
if (pool) {
|
|
841
|
-
|
|
842
|
+
const allDisabled = pool.accounts.length > 0 && pool.accounts.every((id) => accounts.find((a) => a.id === id)?.disabled === true);
|
|
843
|
+
console.log(next ? ` next pooled launch (${pool.id}/…) runs on ${BOLD}${next}${RESET}` : allDisabled
|
|
844
|
+
? ` ${BOLD}pool.accounts[].disabled: every pool member is disabled${RESET}`
|
|
845
|
+
: ` ${BOLD}no selectable pool member${RESET}`);
|
|
842
846
|
}
|
|
843
847
|
console.log(`\n${DIM}(the gateway polls this every ${Math.round(up.effectiveUsagePollInterval(pool?.usagePoll) / 1000)}s and alerts via the heartbeat; see "Live usage" in the README)${RESET}`);
|
|
844
848
|
}
|
|
@@ -954,6 +958,25 @@ switch (cmd) {
|
|
|
954
958
|
case "login":
|
|
955
959
|
await login();
|
|
956
960
|
break;
|
|
961
|
+
case "account": {
|
|
962
|
+
const [action, id, ...flags] = rest;
|
|
963
|
+
if (!["enable", "disable"].includes(action) || !id || flags.some((f) => f !== "--dry-run")) {
|
|
964
|
+
console.error("usage: multi-clawd account <enable|disable> <id> [--dry-run]");
|
|
965
|
+
process.exitCode = 1;
|
|
966
|
+
break;
|
|
967
|
+
}
|
|
968
|
+
try {
|
|
969
|
+
const { changeAccount } = await import("./account-config.mjs");
|
|
970
|
+
const result = changeAccount(id, action === "disable", flags.includes("--dry-run"));
|
|
971
|
+
console.log(`${result.dryRun ? "dry run: " : ""}${id}: ${action === "disable" ? "disabled" : "enabled"}${result.changed ? "" : " (already set)"}`);
|
|
972
|
+
if (result.backup) console.log(`backup: ${result.backup}`);
|
|
973
|
+
if (!result.dryRun && result.changed) console.log("Takes effect when the plugin next reads config (for example after gateway reload); running turns continue.");
|
|
974
|
+
} catch (err) {
|
|
975
|
+
console.error(`account: ${err.message}`);
|
|
976
|
+
process.exitCode = 1;
|
|
977
|
+
}
|
|
978
|
+
break;
|
|
979
|
+
}
|
|
957
980
|
case "explain":
|
|
958
981
|
await explain();
|
|
959
982
|
break;
|
package/scripts/doctor.mjs
CHANGED
|
@@ -354,6 +354,7 @@ const accounts = pluginConfig.accounts ?? [];
|
|
|
354
354
|
const identities = [];
|
|
355
355
|
if (accounts.length === 0) warn("no accounts configured");
|
|
356
356
|
for (const account of accounts) {
|
|
357
|
+
if (account.disabled === true) note(`${account.id}: disabled by operator`);
|
|
357
358
|
// WHO this account is, before WHETHER its credential works. An id is a
|
|
358
359
|
// label the operator chose; the login is the thing quota is spent against,
|
|
359
360
|
// and the two drift apart silently (a config dir re-logged-in as the wrong
|
|
@@ -494,8 +495,16 @@ console.log("pool");
|
|
|
494
495
|
const pool = pluginConfig.pool;
|
|
495
496
|
if (!pool) warn("no pool configured — direct backends only, no proactive rotation");
|
|
496
497
|
else {
|
|
497
|
-
const
|
|
498
|
-
|
|
498
|
+
const configuredMembers = (pool.accounts ?? []).filter((id) => accounts.some((a) => a.id === id));
|
|
499
|
+
const members = configuredMembers.filter((id) => accounts.some((a) => a.id === id && a.disabled !== true));
|
|
500
|
+
const { isModernClaudeModelId } = await importDist("models.js", ["isModernClaudeModelId"], "cli");
|
|
501
|
+
const hasDegradeLadder = (pool.degrade?.ladder ?? []).some(isModernClaudeModelId);
|
|
502
|
+
if (members.length === 0 && configuredMembers.length > 0) bad("pool.accounts[].disabled: every pool member is disabled");
|
|
503
|
+
if (members.length === 1 && configuredMembers.length > 1) {
|
|
504
|
+
warn(`pool "${pool.id ?? "clawd"}" has 1 valid member(s); other configured members are disabled — pool remains available`);
|
|
505
|
+
} else if (members.length === 1 && hasDegradeLadder) {
|
|
506
|
+
note(`pool "${pool.id ?? "clawd"}" has 1 valid member(s) with a degrade ladder; pool remains available`);
|
|
507
|
+
} else if (members.length < 2) bad(`pool "${pool.id ?? "clawd"}" has ${members.length} valid member(s); needs ≥ 2`);
|
|
499
508
|
else ok(`pool "${pool.id ?? "clawd"}": ${members.join(" → ")}`);
|
|
500
509
|
const generalThreshold = pool.utilizationThreshold ?? DEFAULT_UTILIZATION_THRESHOLD;
|
|
501
510
|
const weeklyThreshold = pool.weeklyUtilizationThreshold ?? generalThreshold;
|