@drakon-systems/multi-clawd 2.3.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +121 -31
- package/SECURITY.md +3 -0
- package/dist/credential-state.js +8 -24
- package/dist/direct-report.js +12 -1
- package/dist/direct-route.js +12 -6
- package/dist/direct-sync.js +12 -4
- package/dist/explain-core.js +33 -10
- package/dist/health-journal.js +123 -0
- package/dist/health.js +13 -1
- package/dist/history.js +103 -0
- package/dist/index.js +89 -38
- package/dist/retry-plan.js +2 -0
- package/dist/session-affinity.js +108 -0
- package/dist/shim-core.js +197 -34
- package/dist/shim-lock.js +106 -0
- package/dist/shim.js +170 -79
- package/dist/spread.js +54 -0
- package/dist/sticky.js +2 -0
- package/dist/update-core.js +75 -1
- package/openclaw.plugin.json +26 -3
- package/package.json +1 -1
- package/scripts/account-config.mjs +115 -0
- package/scripts/cli.mjs +131 -25
- package/scripts/doctor.mjs +113 -6
package/README.md
CHANGED
|
@@ -30,6 +30,8 @@ multi-clawd update # install (or update) the OpenClaw plugin — right flags,
|
|
|
30
30
|
multi-clawd setup # guided wizard: accounts, isolated second login, pool, watchdog
|
|
31
31
|
multi-clawd login claw2 # launch the right Claude sign-in for an account (or re-auth it)
|
|
32
32
|
multi-clawd explain # your whole setup in plain English — accounts, chain, live health
|
|
33
|
+
multi-clawd explain --history --since 7d # per-account launches, retries, rotations
|
|
34
|
+
multi-clawd account disable claw2 --dry-run # preview benching an account
|
|
33
35
|
multi-clawd chain # audit your model routing — what actually serves each turn
|
|
34
36
|
multi-clawd direct # the direct anthropic/* route: status, or `direct sync` (v1.9, optional)
|
|
35
37
|
multi-clawd doctor # health check (add --probe for a live end-to-end turn)
|
|
@@ -55,6 +57,12 @@ install anything? Every command also runs as
|
|
|
55
57
|
|
|
56
58
|
Then `multi-clawd explain` shows you exactly what you built.
|
|
57
59
|
|
|
60
|
+
`multi-clawd explain --history` adds per-account served launches, in-turn retry
|
|
61
|
+
hops, rotations, and the last served time for the past seven days. Use
|
|
62
|
+
`--since 24h` or another positive `Nd`/`Nh` duration to change the window.
|
|
63
|
+
The local history is size capped; it records account and model IDs and short
|
|
64
|
+
reason codes, never prompts, credentials, transcripts, tokens, or costs.
|
|
65
|
+
|
|
58
66
|
## Hermes Agent integration
|
|
59
67
|
|
|
60
68
|
multi-clawd can also import the configured Claude subscriptions into Hermes
|
|
@@ -255,6 +263,13 @@ claude-cli/claude-fable-5 # main login
|
|
|
255
263
|
session is handed to the sibling as it stood before the refused attempt.
|
|
256
264
|
One retry. Works for config-dir, native, and setup-token-file accounts; the
|
|
257
265
|
shim is handed paths only and reads a token file at the moment it retries.
|
|
266
|
+
A **transient** provider error (529 overload, 5xx, or a throttle 429 that is
|
|
267
|
+
not a spent window) gets the same single hop to an idle sibling — but
|
|
268
|
+
records nothing against the account that met it, because it is not spent,
|
|
269
|
+
and is passed through unchanged when no idle sibling exists (never onto an
|
|
270
|
+
account the pool believes is spent). The Claude CLI retries these itself
|
|
271
|
+
first; the shim keeps the hop armed through those announced retry sleeps,
|
|
272
|
+
bounded (`MULTI_CLAWD_HOLD_RETRY_MAX_MS`, default 120s, hard cap 150s).
|
|
258
273
|
- 📟 **Operator alerts (v0.3)** — dead logins (probed every 15 min without
|
|
259
274
|
spending quota), pool rotations, whole-pool exhaustion, and watchdog
|
|
260
275
|
restarts surface through your agent's next heartbeat (e.g. straight into
|
|
@@ -302,7 +317,7 @@ argument arrays, never through a shell string. Launch mode varies by platform.
|
|
|
302
317
|
npm i -g @drakon-systems/multi-clawd && multi-clawd update
|
|
303
318
|
```
|
|
304
319
|
|
|
305
|
-
`update`
|
|
320
|
+
`update` installs from ClawHub first, offers the gateway
|
|
306
321
|
restart, and finishes with a doctor health check.
|
|
307
322
|
|
|
308
323
|
> ### ⚠️ Keeping up to date: there are two halves
|
|
@@ -333,39 +348,53 @@ restart, and finishes with a doctor health check.
|
|
|
333
348
|
> version` shows both halves, and since v1.6.0 says plainly whether the pair
|
|
334
349
|
> is a problem.
|
|
335
350
|
>
|
|
336
|
-
> ### ⚠️
|
|
351
|
+
> ### ⚠️ Pinned installs need `multi-clawd update`
|
|
337
352
|
>
|
|
338
|
-
>
|
|
339
|
-
>
|
|
340
|
-
> install specs, so pinning is the right side of that trade), and OpenClaw
|
|
353
|
+
> For a versioned ClawHub install (or npm `--pin` fallback), `openclaw plugins
|
|
354
|
+
> update --all` can tell you the plugin is up to date when it is not. OpenClaw
|
|
341
355
|
> resolves registry metadata *for the pinned spec* — comparing 1.6.0 against
|
|
342
356
|
> 1.6.0 and returning "up to date" while a newer version sits on npm. The
|
|
343
357
|
> honest "pinned to X; registry default resolves to Y" message exists in
|
|
344
358
|
> OpenClaw but is built inside its `--dry-run` branch, so a real update run
|
|
345
359
|
> never prints it.
|
|
346
360
|
>
|
|
347
|
-
> **Use `multi-clawd update
|
|
361
|
+
> **Use `multi-clawd update` for versioned installs.** Since v1.7.2 `doctor` also checks the registry
|
|
348
362
|
> itself (cached, and silent when offline), so a lagging plugin is reported
|
|
349
363
|
> rather than left to a command that reassures you wrongly.
|
|
350
364
|
|
|
351
365
|
Prefer the raw form?
|
|
352
366
|
|
|
353
367
|
```bash
|
|
354
|
-
openclaw plugins install
|
|
355
|
-
|
|
368
|
+
openclaw plugins install clawhub:@drakon-systems/multi-clawd
|
|
369
|
+
# Only after a successful install, and with operator approval:
|
|
370
|
+
# openclaw gateway restart
|
|
356
371
|
```
|
|
357
372
|
|
|
358
373
|
Either way the gateway pulls the prebuilt package — no clone, no build step,
|
|
359
|
-
nothing to keep in sync.
|
|
360
|
-
|
|
374
|
+
nothing to keep in sync. ClawHub is the source OpenClaw trusts; npm installs show a source caution.
|
|
375
|
+
For existing installs, prefer the guarded `multi-clawd update` command rather than a raw source retry.
|
|
376
|
+
|
|
377
|
+
**Raw-install fallback boundary:** use npm only after positively identifying a
|
|
378
|
+
ClawHub availability failure (for example, package/version not found or registry
|
|
379
|
+
unreachable), or an older OpenClaw CLI explicitly rejecting the `clawhub:` spec
|
|
380
|
+
as unsupported. An old CLI version alone is not evidence of that incompatibility.
|
|
381
|
+
**Stop; do not retry through npm** for moderation, security, integrity, trust,
|
|
382
|
+
signature or policy failures, unknown errors, cancellation or interruption.
|
|
383
|
+
These stops take precedence even if availability diagnostics appeared first.
|
|
384
|
+
If the cause is unclear, stop and investigate; do not switch sources or restart
|
|
385
|
+
the gateway after a failed or interrupted install.
|
|
386
|
+
|
|
387
|
+
Only after the availability/unsupported-spec check above, an unversioned fresh
|
|
388
|
+
install may use `openclaw plugins install @drakon-systems/multi-clawd --pin`.
|
|
389
|
+
For a versioned install, retain the same selected exact target: if ClawHub was
|
|
390
|
+
`clawhub:@drakon-systems/multi-clawd@X.Y.Z`, the permitted npm retry is
|
|
391
|
+
`openclaw plugins install @drakon-systems/multi-clawd@X.Y.Z --pin`.
|
|
392
|
+
Replace `X.Y.Z` with that same exact version; never drop it, re-resolve latest,
|
|
393
|
+
or substitute a different version during fallback.
|
|
394
|
+
An unversioned ClawHub spec follows newer releases; add `@X.Y.Z` to keep a specific release.
|
|
395
|
+
`openclaw` itself is a *peer*
|
|
361
396
|
dependency (the host provides it), so the install stays lean.
|
|
362
397
|
|
|
363
|
-
**From ClawHub (alternative registry):**
|
|
364
|
-
|
|
365
|
-
```bash
|
|
366
|
-
openclaw plugins install clawhub:@drakon-systems/multi-clawd
|
|
367
|
-
```
|
|
368
|
-
|
|
369
398
|
**From source (contributors, or ahead of a release):**
|
|
370
399
|
|
|
371
400
|
```bash
|
|
@@ -386,7 +415,7 @@ openclaw plugins install (Get-Location).Path
|
|
|
386
415
|
**Or let your agent install it.** Running an OpenClaw assistant or Claude
|
|
387
416
|
Code on the target machine already? Paste it this and go make coffee:
|
|
388
417
|
|
|
389
|
-
> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v2.
|
|
418
|
+
> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v2.5.0/SETUP-AGENT.md
|
|
390
419
|
> and follow it to set up multi-clawd on this machine. I own a second
|
|
391
420
|
> Claude account — ask me when you need me to log in.
|
|
392
421
|
|
|
@@ -411,8 +440,9 @@ host-provided peer.
|
|
|
411
440
|
npx @drakon-systems/multi-clawd update
|
|
412
441
|
```
|
|
413
442
|
|
|
414
|
-
One command: checks the registry, installs the
|
|
415
|
-
|
|
443
|
+
One command: checks the registry, installs the chosen version from ClawHub (with npm
|
|
444
|
+
fallback only when ClawHub is unavailable), offers the gateway restart, and finishes with a doctor health check.
|
|
445
|
+
Security, integrity, and policy failures stop the update without trying npm.
|
|
416
446
|
(`npm i -g @drakon-systems/multi-clawd` once, and it's just `multi-clawd
|
|
417
447
|
update` — with `multi-clawd setup`, `multi-clawd explain` (your setup in
|
|
418
448
|
plain English), and `multi-clawd doctor` alongside.)
|
|
@@ -546,6 +576,7 @@ error.
|
|
|
546
576
|
"accounts": [
|
|
547
577
|
{ "id": "claw1", "label": "Main Claude", "native": true },
|
|
548
578
|
{ "id": "claw2", "label": "Second Max", "configDir": "~/.claw2",
|
|
579
|
+
// "disabled": true, // optional; default false: bench without removing the pool member
|
|
549
580
|
// v0.3 preferred: resolve via your gateway's secret providers —
|
|
550
581
|
// no plaintext token file on disk
|
|
551
582
|
"oauthTokenRef": { "source": "exec", "provider": "onepassword",
|
|
@@ -554,10 +585,14 @@ error.
|
|
|
554
585
|
"pool": {
|
|
555
586
|
"id": "clawd",
|
|
556
587
|
"accounts": ["claw1", "claw2"], // preference order; first = home for ordered strategy
|
|
557
|
-
"strategy": "ordered", // optional: "reset-aware" drains healthy weekly quota that resets soonest
|
|
588
|
+
"strategy": "ordered", // optional: "reset-aware" drains healthy weekly quota that resets soonest;
|
|
589
|
+
// "spread" starts each new session on the least-served healthy account
|
|
590
|
+
// "spreadWindowMs": 3600000, // optional, spread only: how far back served launches count (default 1h)
|
|
558
591
|
"utilizationThreshold": 0.85, // default hand-over; minute/hour and overage windows use this
|
|
559
592
|
"weeklyUtilizationThreshold": 0.97, // optional: day/week/month quota windows at 97%
|
|
560
593
|
"minDwellMs": 600000, // v0.3: anti-flap dwell before returning home
|
|
594
|
+
"sessionAffinity": true, // optional; default false: keep healthy CLI sessions on their serving account
|
|
595
|
+
"sessionAffinityTtlMs": 2592000000, // optional; default 30 days
|
|
561
596
|
"degrade": { // v0.3.5: last step before provider drop
|
|
562
597
|
"ladder": ["claude-opus-4-8"], // whole pool exhausted → same account, lower tier
|
|
563
598
|
"pins": [ // contractual lanes never degrade
|
|
@@ -578,6 +613,25 @@ error.
|
|
|
578
613
|
} }
|
|
579
614
|
```
|
|
580
615
|
|
|
616
|
+
Set `accounts[].disabled: true` to bench a member while keeping it in
|
|
617
|
+
`pool.accounts` and visible in `explain`, `doctor`, and `usage` as "disabled by
|
|
618
|
+
operator". It is excluded from pool selection, in-turn retries, session
|
|
619
|
+
affinity, and the managed direct Anthropic order. Transcript lookup still
|
|
620
|
+
searches its session directory. To edit `~/.openclaw/openclaw.json` with a
|
|
621
|
+
timestamped backup and without reformatting unrelated config, run
|
|
622
|
+
`multi-clawd account disable <id>` or `multi-clawd account enable <id>`.
|
|
623
|
+
Add `--dry-run` to preview without writing anything. If every pool member is
|
|
624
|
+
disabled, pooled launches fail closed with an error naming
|
|
625
|
+
`pool.accounts[].disabled`; they never fall back to a disabled member.
|
|
626
|
+
On the direct `anthropic/*` route, a disabled member is left out of the order
|
|
627
|
+
multi-clawd writes. If every direct member is disabled, multi-clawd stops
|
|
628
|
+
managing that order and logs a warning naming `pool.accounts[].disabled`, but
|
|
629
|
+
it cannot empty an order OpenClaw has already stored, so clear or edit that
|
|
630
|
+
stored order yourself if the direct route must not use those profiles.
|
|
631
|
+
Changes take effect when the plugin next reads its config, for example after
|
|
632
|
+
a gateway reload. A turn already admitted or running continues on its selected
|
|
633
|
+
account; the command does not interrupt it.
|
|
634
|
+
|
|
581
635
|
`pool.strategy` defaults to `"ordered"`, which retains the configured account
|
|
582
636
|
order. Set it to `"reset-aware"` to use healthy weekly quota before its
|
|
583
637
|
`seven_day` window resets: among `ok` accounts with live seven-day reset data,
|
|
@@ -587,6 +641,39 @@ times keep pool order. The existing `minDwellMs` delay still holds an account
|
|
|
587
641
|
that remains usable. Both CLI launches and the direct Anthropic route use the
|
|
588
642
|
same preference. Other values are rejected by the plugin config schema.
|
|
589
643
|
|
|
644
|
+
Set `pool.strategy` to `"spread"` when several agent lanes start sessions at
|
|
645
|
+
once and you want them on different accounts instead of queueing behind one
|
|
646
|
+
account's concurrency ceiling. Each **new** session starts on the healthy
|
|
647
|
+
(`ok` or `no_data`) account that served the fewest launches in the last
|
|
648
|
+
`spreadWindowMs` (default one hour; clamped between one minute and seven
|
|
649
|
+
days), counted from the served-launch journal `explain --history` reads. On a
|
|
650
|
+
tie the account served least recently wins, then pool order, so equally idle
|
|
651
|
+
accounts are used in turn. Near-limit, exhausted and credential-failed
|
|
652
|
+
accounts are never chosen; when no healthy account is left the pool answers
|
|
653
|
+
exactly as `ordered` does. Spread keeps every session on the account that
|
|
654
|
+
first served it: it switches `sessionAffinity` on for the pool (an explicit
|
|
655
|
+
`sessionAffinity: false` is overridden with a warning, because without it a
|
|
656
|
+
conversation would move on every turn). A pinned session moves only when its
|
|
657
|
+
account stops being healthy, exactly as under `sessionAffinity`. Launches
|
|
658
|
+
without a session id take the spread pick. There is no home account, no
|
|
659
|
+
dwell and no rotation alert under spread; `explain`, `usage` and `doctor`
|
|
660
|
+
show the strategy, the next placement and the served counts behind it. The
|
|
661
|
+
direct Anthropic route orders its profiles by the same counts.
|
|
662
|
+
|
|
663
|
+
`pool.sessionAffinity: true` remembers the account that successfully served each
|
|
664
|
+
Claude CLI session (`--session-id` or `--resume`) and prefers it on later turns
|
|
665
|
+
while its health is `ok` or `no_data`. A near-limit or otherwise unhealthy
|
|
666
|
+
account loses its pin; the successful replacement becomes the new pin. New
|
|
667
|
+
sessions and launches without a session id keep normal pool selection. Pins
|
|
668
|
+
live in `~/.openclaw/state/multi-clawd/pool-<id>.sessions.json`, expire after
|
|
669
|
+
30 days by default (`sessionAffinityTtlMs`), and are capped at 1000 sessions.
|
|
670
|
+
This applies to shimmed CLI launches; direct Anthropic and side-question
|
|
671
|
+
launches keep their existing routing. The shim can switch to native,
|
|
672
|
+
config-dir, and token-file accounts. On verified OpenClaw launches, the
|
|
673
|
+
gateway also resolves a pinned secret-reference account at execution time.
|
|
674
|
+
Legacy shim launches fall back to normal selection when a secret-reference
|
|
675
|
+
pin needs a different account.
|
|
676
|
+
|
|
590
677
|
How it decides, per launch (all data from each account's live
|
|
591
678
|
`rate_limit_event` stream, captured by the shim):
|
|
592
679
|
|
|
@@ -855,15 +942,16 @@ event, and no plugin API can force a rebuild. See `DESIGN.md`.
|
|
|
855
942
|
after the assessment or 24 hours after that turn, whichever comes first. A
|
|
856
943
|
CLI login failure recorded on a real turn removes it at once.
|
|
857
944
|
|
|
858
|
-
- **Shim
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
945
|
+
- **Shim state and retry coordination is best effort under lock contention.**
|
|
946
|
+
Concurrent shims for one account use a short advisory lock around each
|
|
947
|
+
state merge and rename. A second short lock covers recording a refusal and
|
|
948
|
+
reading sibling health for the in-turn retry. Later decisions can therefore
|
|
949
|
+
see rejections already recorded by another shim. If a lock stays busy beyond
|
|
950
|
+
150 ms or filesystem locking fails, the shim proceeds without it so the
|
|
951
|
+
turn still runs; concurrent writes in that fallback can lose an observation.
|
|
952
|
+
A replacement between the lock's final ownership check and removal can
|
|
953
|
+
still remove another live holder's lock because those steps are not atomic.
|
|
954
|
+
The guard does not pace requests sent to a newly chosen sibling.
|
|
867
955
|
|
|
868
956
|
## Security
|
|
869
957
|
|
|
@@ -878,7 +966,7 @@ callsite (the ones static scanners flag) and why each exists.
|
|
|
878
966
|
| Network | The gateway plugin polls `https://api.anthropic.com/api/oauth/usage` for eligible pooled accounts on an interval (default 120000 ms / 2 minutes); `pool.usagePoll.enabled: false` disables that poll. `multi-clawd usage` requests the same endpoint on demand. `doctor` runs `npm view` when its registry cache is stale; install and update commands contact the package registry. Claude Code subprocesses make their own provider requests. No plugin telemetry or analytics endpoint. |
|
|
879
967
|
| Credentials | Reads declared setup-token files or gateway-resolved `oauthTokenRef` values for Claude child launches; reads each eligible account's `.credentials.json` access token for the usage poll; reads Claude login credentials from files or the macOS Keychain via the `security` CLI for login checks and runtime auth; reads account identity for file-backed logins from `.claude.json` (`~/.claude.json` for a native login, or the account config dir). Setup tokens are passed to Claude through the child environment. |
|
|
880
968
|
| Files read | `~/.openclaw/openclaw.json`, declared account config dirs and token files, Claude `.credentials.json` and `.claude.json` files as described above, and local state files. |
|
|
881
|
-
| Files written | Under `~/.openclaw/state/multi-clawd/` by default: `<id>.json`, `<id>.usage.json`, `pool-<id>.sticky.json`, `direct-order.sticky.json`, `known-models.json`, `alerts-spool.jsonl`, `watchdog.json`, `registry-check.json`, `auth-profiles-<agent>.json`, `runtime/<hash>/` (copied runtime files), and `watchdog-launcher.mjs`. The watchdog state file can be redirected with `MULTI_CLAWD_WATCHDOG_STATE`. Setup writes `~/.openclaw/openclaw.json` and a dated backup. On session handover, transcript `.jsonl` files are copied between account config dirs. The optional watchdog installs `~/.config/systemd/user/multi-clawd-eviction-watchdog.service` and `.timer` on Linux, or `~/Library/LaunchAgents/com.multi-clawd.eviction-watchdog.plist` on macOS; the unit points at the state-dir launcher. |
|
|
969
|
+
| Files written | Under `~/.openclaw/state/multi-clawd/` by default: `<id>.json`, `<id>.usage.json`, `history.jsonl` (bounded account activity), `pool-<id>.sticky.json`, `pool-<id>.sessions.json` (when affinity is enabled), `direct-order.sticky.json`, `known-models.json`, `alerts-spool.jsonl`, `watchdog.json`, `registry-check.json`, `auth-profiles-<agent>.json`, `runtime/<hash>/` (copied runtime files), and `watchdog-launcher.mjs`. The watchdog state file can be redirected with `MULTI_CLAWD_WATCHDOG_STATE`. Setup writes `~/.openclaw/openclaw.json` and a dated backup. On session handover, transcript `.jsonl` files are copied between account config dirs. The optional watchdog installs `~/.config/systemd/user/multi-clawd-eviction-watchdog.service` and `.timer` on Linux, or `~/Library/LaunchAgents/com.multi-clawd.eviction-watchdog.plist` on macOS; the unit points at the state-dir launcher. |
|
|
882
970
|
| Processes | Launches Claude Code and the node + shim transport; CLI, setup and doctor run npm, OpenClaw, Claude and OS scheduler commands; the optional watchdog checks OpenClaw logs and may restart the gateway; Hermes integration runs its Python bridge. See [SECURITY.md](SECURITY.md) for the process-call table. |
|
|
883
971
|
| Consent | The wizard asks before every write, merges non-destructively, and never overwrites an existing account entry. `--dry-run` previews everything. |
|
|
884
972
|
|
|
@@ -977,8 +1065,10 @@ Early but real — built for and dogfooded in production.
|
|
|
977
1065
|
- **v1.9** — one pool, both transports: accounts can also serve OpenClaw's
|
|
978
1066
|
direct `anthropic/*` route; the gateway keeps the `anthropic` auth order in
|
|
979
1067
|
pool-health order; `multi-clawd direct` / `direct sync` ✅
|
|
1068
|
+
- **Unreleased** — optional per-session affinity for shimmed Claude CLI
|
|
1069
|
+
launches; health overrides the pin, with bounded expiring state ✅
|
|
980
1070
|
- **Next** — standalone localhost proxy (OpenAI-compatible) so Hermes and
|
|
981
|
-
custom runtimes can share the pool;
|
|
1071
|
+
custom runtimes can share the pool; local
|
|
982
1072
|
five-hour-window signal (turn counting); per-account lock for the shim
|
|
983
1073
|
persistence race
|
|
984
1074
|
|
package/SECURITY.md
CHANGED
|
@@ -37,6 +37,9 @@ credit you (unless you'd rather we didn't), and note it in the CHANGELOG.
|
|
|
37
37
|
- **Per-account health and usage files are written `0600`,** using temporary
|
|
38
38
|
files and rename, and contain rate-limit data rather than credentials. Other
|
|
39
39
|
state files include model, alert, registry and watchdog data.
|
|
40
|
+
- **`history.jsonl` is a bounded `0600` account activity file** in the same
|
|
41
|
+
state directory. It contains timestamps, account/model IDs and short reason
|
|
42
|
+
codes, with no prompt, transcript, token, cost, credential, or environment values.
|
|
40
43
|
|
|
41
44
|
## Permissions
|
|
42
45
|
|
package/dist/credential-state.js
CHANGED
|
@@ -1,33 +1,17 @@
|
|
|
1
|
-
import { readFileSync, writeFileSync, mkdirSync, renameSync } from "node:fs";
|
|
2
1
|
import { homedir } from "node:os";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { persistHealthState, readHealthStateFile } from "./health-journal.js";
|
|
4
|
+
import { clearCredentialFailure, } from "./shim-core.js";
|
|
5
5
|
export function healthStateFile(accountId) {
|
|
6
6
|
return join(homedir(), ".openclaw", "state", "multi-clawd", `${accountId}.json`);
|
|
7
7
|
}
|
|
8
8
|
export function usageStateFile(accountId) {
|
|
9
9
|
return join(homedir(), ".openclaw", "state", "multi-clawd", `${accountId}.usage.json`);
|
|
10
10
|
}
|
|
11
|
-
export function clearAccountCredentialFailure(accountId) {
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
try {
|
|
15
|
-
state = parseStoredState(readFileSync(file, "utf8")) ?? { accountId, windows: {} };
|
|
16
|
-
}
|
|
17
|
-
catch {
|
|
11
|
+
export function clearAccountCredentialFailure(accountId, file = healthStateFile(accountId)) {
|
|
12
|
+
const state = readHealthStateFile(file);
|
|
13
|
+
if (state?.credential?.status !== "failed")
|
|
18
14
|
return false;
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
return false;
|
|
22
|
-
const cleared = mergeHealthStates(state, clearCredentialFailure(state, Date.now()), Date.now());
|
|
23
|
-
try {
|
|
24
|
-
mkdirSync(dirname(file), { recursive: true });
|
|
25
|
-
const tmp = `${file}.tmp-${process.pid}`;
|
|
26
|
-
writeFileSync(tmp, JSON.stringify(cleared, null, 2), { mode: 0o600 });
|
|
27
|
-
renameSync(tmp, file);
|
|
28
|
-
return true;
|
|
29
|
-
}
|
|
30
|
-
catch {
|
|
31
|
-
return false;
|
|
32
|
-
}
|
|
15
|
+
persistHealthState(file, clearCredentialFailure(state, Date.now()));
|
|
16
|
+
return readHealthStateFile(file)?.credential?.status === "ok";
|
|
33
17
|
}
|
package/dist/direct-report.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { classifyAccountHealth, sevenDayReset } from "./health.js";
|
|
2
|
+
import { spreadStatsFor, spreadStatsFromHistory, spreadWindowMs } from "./spread.js";
|
|
2
3
|
import { DIRECT_PROVIDER, collectDirectMembers, } from "./direct-route.js";
|
|
3
4
|
import { parseUnusableProfiles, planAgentOrder, readDirectRouteSnapshot, safeCliError, } from "./direct-sync.js";
|
|
4
5
|
export function describeDirectSource(member) {
|
|
6
|
+
if (member.disabled)
|
|
7
|
+
return "disabled by operator";
|
|
5
8
|
const s = member.source;
|
|
6
9
|
switch (s.kind) {
|
|
7
10
|
case "ref":
|
|
@@ -19,15 +22,20 @@ export async function gatherDirectStatus(params) {
|
|
|
19
22
|
if (members.length === 0 && problems.length === 0)
|
|
20
23
|
return undefined;
|
|
21
24
|
const errors = [];
|
|
25
|
+
const spread = params.strategy === "spread"
|
|
26
|
+
? spreadStatsFromHistory(params.readHistory?.() ?? [], params.nowMs, spreadWindowMs(params.spreadWindowMs))
|
|
27
|
+
: undefined;
|
|
22
28
|
const verdicts = members.map((m) => {
|
|
23
29
|
const state = params.readHealth(m.accountId);
|
|
24
30
|
return {
|
|
25
31
|
accountId: m.accountId,
|
|
26
32
|
profileId: m.profileId,
|
|
33
|
+
disabled: m.disabled,
|
|
27
34
|
verdict: classifyAccountHealth(state, params.healthOptions, params.nowMs).verdict,
|
|
28
35
|
sevenDayResetsAt: params.strategy === "reset-aware"
|
|
29
36
|
? sevenDayReset(state, params.healthOptions, params.nowMs)
|
|
30
37
|
: undefined,
|
|
38
|
+
...(spread ? spreadStatsFor(spread, m.accountId) : {}),
|
|
31
39
|
};
|
|
32
40
|
});
|
|
33
41
|
const read = members.length > 0 ? await readDirectRouteSnapshot(params.runner, params.agentId) : {};
|
|
@@ -44,7 +52,7 @@ export async function gatherDirectStatus(params) {
|
|
|
44
52
|
errors.push(`models status ${safeCliError(status)}`);
|
|
45
53
|
}
|
|
46
54
|
let pendingOrder;
|
|
47
|
-
if (snapshot) {
|
|
55
|
+
if (snapshot && members.some((m) => !m.disabled)) {
|
|
48
56
|
pendingOrder = planAgentOrder({
|
|
49
57
|
members: verdicts,
|
|
50
58
|
snapshot,
|
|
@@ -55,6 +63,9 @@ export async function gatherDirectStatus(params) {
|
|
|
55
63
|
strategy: params.strategy,
|
|
56
64
|
}).order;
|
|
57
65
|
}
|
|
66
|
+
else if (members.length > 0 && members.every((m) => m.disabled)) {
|
|
67
|
+
errors.push("pool.accounts[].disabled: every direct member is disabled");
|
|
68
|
+
}
|
|
58
69
|
const byProfile = new Map(unusable.map((u) => [u.profileId, u]));
|
|
59
70
|
const order = snapshot?.storedOrder ?? (params.configOrder?.length ? [...params.configOrder] : snapshot ? [] : undefined);
|
|
60
71
|
const orderSource = snapshot?.storedOrder
|
package/dist/direct-route.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isSecretRefShape } from "./token-resolution.js";
|
|
2
2
|
import { decideStickySelection } from "./sticky.js";
|
|
3
|
+
import { spreadBefore } from "./spread.js";
|
|
3
4
|
export const DIRECT_PROVIDER = "anthropic";
|
|
4
5
|
export const DIRECT_PROFILE_PREFIX = `${DIRECT_PROVIDER}:`;
|
|
5
6
|
export const DIRECT_SETUP_TOKEN_GUIDANCE = "a native or config-dir Claude login is a rotating single-use OAuth grant — copying it into " +
|
|
@@ -115,7 +116,7 @@ export function collectDirectMembers(accounts, poolOrder = []) {
|
|
|
115
116
|
continue;
|
|
116
117
|
}
|
|
117
118
|
seenProfiles.add(profileId);
|
|
118
|
-
members.push({ accountId: account.id, profileId, source });
|
|
119
|
+
members.push({ accountId: account.id, profileId, source, disabled: account.disabled === true });
|
|
119
120
|
}
|
|
120
121
|
return { members, problems };
|
|
121
122
|
}
|
|
@@ -128,11 +129,13 @@ const VERDICT_RANK = {
|
|
|
128
129
|
credential_failed: 4,
|
|
129
130
|
};
|
|
130
131
|
export function planDirectOrder(params) {
|
|
131
|
-
const
|
|
132
|
-
if (members.length === 0)
|
|
132
|
+
const members = params.members.filter((m) => !m.disabled);
|
|
133
|
+
if (params.members.length === 0)
|
|
133
134
|
return undefined;
|
|
135
|
+
if (members.length === 0)
|
|
136
|
+
throw new Error("pool.accounts[].disabled: every direct member is disabled");
|
|
134
137
|
const decision = decideStickySelection({
|
|
135
|
-
verdicts: members.map((m) => ({ id: m.accountId, verdict: m.verdict, sevenDayResetsAt: m.sevenDayResetsAt })),
|
|
138
|
+
verdicts: members.map((m) => ({ id: m.accountId, verdict: m.verdict, sevenDayResetsAt: m.sevenDayResetsAt, servedInWindow: m.servedInWindow, lastServedAt: m.lastServedAt })),
|
|
136
139
|
sticky: params.sticky,
|
|
137
140
|
nowMs: params.nowMs,
|
|
138
141
|
minDwellMs: params.minDwellMs,
|
|
@@ -145,9 +148,12 @@ export function planDirectOrder(params) {
|
|
|
145
148
|
.sort((a, b) => VERDICT_RANK[a.m.verdict] - VERDICT_RANK[b.m.verdict] ||
|
|
146
149
|
(params.strategy === "reset-aware" && a.m.verdict === "ok"
|
|
147
150
|
? (a.m.sevenDayResetsAt ?? Infinity) - (b.m.sevenDayResetsAt ?? Infinity)
|
|
151
|
+
: 0) ||
|
|
152
|
+
(params.strategy === "spread" && VERDICT_RANK[a.m.verdict] === 0
|
|
153
|
+
? (spreadBefore(a.m, b.m) ? -1 : spreadBefore(b.m, a.m) ? 1 : 0)
|
|
148
154
|
: 0) || a.index - b.index)
|
|
149
155
|
.map(({ m }) => m.profileId);
|
|
150
|
-
const managed = new Set(members.map((m) => m.profileId));
|
|
156
|
+
const managed = new Set(params.members.map((m) => m.profileId));
|
|
151
157
|
const unmanaged = (params.currentOrder ?? []).filter((id, i, all) => typeof id === "string" && !managed.has(id) && all.indexOf(id) === i);
|
|
152
158
|
const order = [first.profileId, ...rest, ...unmanaged];
|
|
153
159
|
const current = params.currentOrder ?? [];
|
|
@@ -157,5 +163,5 @@ export function planDirectOrder(params) {
|
|
|
157
163
|
export function directRoutePools(accounts, directRoute) {
|
|
158
164
|
if (directRoute?.manageOrder === false)
|
|
159
165
|
return false;
|
|
160
|
-
return collectDirectMembers(accounts).members.length >= 2;
|
|
166
|
+
return collectDirectMembers(accounts).members.filter((m) => !m.disabled).length >= 2;
|
|
161
167
|
}
|
package/dist/direct-sync.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { classifyAccountHealth, sevenDayReset } from "./health.js";
|
|
2
|
+
import { spreadStatsFor, spreadStatsFromHistory, spreadWindowMs } from "./spread.js";
|
|
2
3
|
import { DIRECT_PROVIDER, planDirectOrder } from "./direct-route.js";
|
|
3
4
|
import { parseClaudeSetupToken } from "./hermes-core.js";
|
|
4
5
|
export function safeCliError(result) {
|
|
@@ -130,6 +131,8 @@ export async function syncDirectProfiles(params) {
|
|
|
130
131
|
const stored = new Set(listed.map((p) => p.id));
|
|
131
132
|
const results = [];
|
|
132
133
|
for (const member of params.members) {
|
|
134
|
+
if (member.disabled)
|
|
135
|
+
continue;
|
|
133
136
|
const base = { accountId: member.accountId, profileId: member.profileId };
|
|
134
137
|
const { source } = member;
|
|
135
138
|
if (source.kind === "existing") {
|
|
@@ -212,14 +215,14 @@ function sameOrder(a, b) {
|
|
|
212
215
|
}
|
|
213
216
|
export function planAgentOrder(params) {
|
|
214
217
|
const { snapshot } = params;
|
|
215
|
-
const present = params.members.filter((m) => snapshot.stored.has(m.profileId));
|
|
216
|
-
const missing = params.members.filter((m) => !snapshot.stored.has(m.profileId)).map((m) => m.profileId);
|
|
217
|
-
if (present.length === 0)
|
|
218
|
+
const present = params.members.filter((m) => !m.disabled && snapshot.stored.has(m.profileId));
|
|
219
|
+
const missing = params.members.filter((m) => !m.disabled && !snapshot.stored.has(m.profileId)).map((m) => m.profileId);
|
|
220
|
+
if (present.length === 0 && params.members.some((m) => !m.disabled))
|
|
218
221
|
return { missing };
|
|
219
222
|
const explicit = snapshot.storedOrder ?? (params.configOrder?.length ? [...params.configOrder] : null);
|
|
220
223
|
const base = (explicit ?? [...snapshot.stored]).filter((id) => snapshot.stored.has(id));
|
|
221
224
|
const plan = planDirectOrder({
|
|
222
|
-
members:
|
|
225
|
+
members: params.members.map((m) => ({ ...m, disabled: m.disabled || !snapshot.stored.has(m.profileId) })),
|
|
223
226
|
currentOrder: base,
|
|
224
227
|
sticky: params.sticky,
|
|
225
228
|
nowMs: params.nowMs,
|
|
@@ -245,15 +248,20 @@ export function createDirectOrderController(deps) {
|
|
|
245
248
|
return { managedOrder: [], agents: [] };
|
|
246
249
|
running = true;
|
|
247
250
|
try {
|
|
251
|
+
const spread = deps.strategy === "spread"
|
|
252
|
+
? spreadStatsFromHistory(deps.readHistory?.() ?? [], nowMs, spreadWindowMs(deps.spreadWindowMs))
|
|
253
|
+
: undefined;
|
|
248
254
|
const verdicts = deps.members.map((m) => {
|
|
249
255
|
const state = deps.readHealth(m.accountId);
|
|
250
256
|
return {
|
|
251
257
|
accountId: m.accountId,
|
|
252
258
|
profileId: m.profileId,
|
|
259
|
+
disabled: m.disabled,
|
|
253
260
|
verdict: classifyAccountHealth(state, deps.healthOptions, nowMs).verdict,
|
|
254
261
|
sevenDayResetsAt: deps.strategy === "reset-aware"
|
|
255
262
|
? sevenDayReset(state, deps.healthOptions, nowMs)
|
|
256
263
|
: undefined,
|
|
264
|
+
...(spread ? spreadStatsFor(spread, m.accountId) : {}),
|
|
257
265
|
};
|
|
258
266
|
});
|
|
259
267
|
const sticky = deps.readSticky();
|
package/dist/explain-core.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { DEFAULT_UTILIZATION_THRESHOLD, choosePoolAccount, poolStrategy } from "./health.js";
|
|
2
2
|
import { decideStickySelection } from "./sticky.js";
|
|
3
|
+
import { describeSpreadPick, describeWindow, spreadStatsFromHistory, spreadWindowMs, withSpreadStats } from "./spread.js";
|
|
3
4
|
const WINDOW_LABELS = {
|
|
4
5
|
five_hour: "5-hour",
|
|
5
6
|
seven_day: "weekly",
|
|
@@ -16,6 +17,9 @@ export function relativeUntil(ms, nowMs) {
|
|
|
16
17
|
return `~${hours}h`;
|
|
17
18
|
return `~${Math.round(hours / 24)}d`;
|
|
18
19
|
}
|
|
20
|
+
export function relativeSince(ms, nowMs) {
|
|
21
|
+
return relativeUntil(nowMs + Math.max(0, nowMs - ms), nowMs);
|
|
22
|
+
}
|
|
19
23
|
export function renderUsageLine(usage, nowMs) {
|
|
20
24
|
return usage
|
|
21
25
|
.map((u) => {
|
|
@@ -81,7 +85,7 @@ export function renderExplanation(model) {
|
|
|
81
85
|
const lines = [];
|
|
82
86
|
lines.push("ACCOUNTS");
|
|
83
87
|
for (const acc of model.accounts) {
|
|
84
|
-
lines.push(` ${acc.id}${acc.label ? ` "${acc.label}"` : ""}`);
|
|
88
|
+
lines.push(` ${acc.id}${acc.label ? ` "${acc.label}"` : ""}${acc.disabled ? " — disabled by operator" : ""}`);
|
|
85
89
|
lines.push(` → ${describeAccount(acc)}`);
|
|
86
90
|
}
|
|
87
91
|
lines.push("");
|
|
@@ -93,6 +97,12 @@ export function renderExplanation(model) {
|
|
|
93
97
|
if (poolStrategy(model.pool.strategy) === "reset-aware") {
|
|
94
98
|
lines.push(" Healthy accounts with a live seven-day reset are preferred by soonest reset.");
|
|
95
99
|
}
|
|
100
|
+
else if (poolStrategy(model.pool.strategy) === "spread") {
|
|
101
|
+
lines.push(` Each new session starts on the healthy account with the fewest served launches in the last ${describeWindow(spreadWindowMs(model.pool.spreadWindowMs))};`);
|
|
102
|
+
lines.push(" a session then stays on its account while that account is healthy (session affinity is on).");
|
|
103
|
+
if (model.pool.sessionAffinity === false)
|
|
104
|
+
lines.push(" sessionAffinity: false is ignored under spread — without it a conversation would move every turn.");
|
|
105
|
+
}
|
|
96
106
|
else {
|
|
97
107
|
lines.push(" Every Claude launch runs on the first account below its threshold;");
|
|
98
108
|
lines.push(" the home account reclaims work");
|
|
@@ -133,12 +143,14 @@ export function renderExplanation(model) {
|
|
|
133
143
|
lines.push("RIGHT NOW");
|
|
134
144
|
for (const h of model.health) {
|
|
135
145
|
const word = VERDICT_WORDS[h.verdict] ?? h.verdict;
|
|
136
|
-
lines.push(` ${h.id}: ${word}${h.detail ? ` — ${h.detail}` : ""}`);
|
|
146
|
+
lines.push(` ${h.id}: ${model.accounts.find((a) => a.id === h.id)?.disabled ? "disabled by operator" : word}${h.detail ? ` — ${h.detail}` : ""}`);
|
|
137
147
|
lines.push(` usage: ${h.usage?.length ? renderUsageLine(h.usage, nowMs) : "no live telemetry"}`);
|
|
138
148
|
}
|
|
139
149
|
if (model.pool) {
|
|
140
150
|
const strategy = poolStrategy(model.pool.strategy);
|
|
141
|
-
const
|
|
151
|
+
const windowMs = spreadWindowMs(model.pool.spreadWindowMs);
|
|
152
|
+
const spread = strategy === "spread" ? spreadStatsFromHistory(model.history ?? [], nowMs, windowMs) : undefined;
|
|
153
|
+
const rawVerdicts = model.pool.accounts.filter((id) => !model.accounts.find((a) => a.id === id)?.disabled).map((id) => {
|
|
142
154
|
const h = model.health.find((row) => row.id === id);
|
|
143
155
|
return {
|
|
144
156
|
id,
|
|
@@ -147,6 +159,7 @@ export function renderExplanation(model) {
|
|
|
147
159
|
?? h?.usage?.find((u) => u.window === "seven_day" && u.resetsAt !== undefined && u.resetsAt > nowMs))?.resetsAt,
|
|
148
160
|
};
|
|
149
161
|
});
|
|
162
|
+
const verdicts = spread ? withSpreadStats(rawVerdicts, spread) : rawVerdicts;
|
|
150
163
|
if (verdicts.length) {
|
|
151
164
|
const preferred = choosePoolAccount(verdicts, strategy);
|
|
152
165
|
const decision = decideStickySelection({ verdicts, sticky: model.stickyEntry, nowMs, minDwellMs: model.pool.minDwellMs, strategy });
|
|
@@ -156,6 +169,8 @@ export function renderExplanation(model) {
|
|
|
156
169
|
reason = "sticky dwell keeps this usable account";
|
|
157
170
|
else if (strategy === "ordered")
|
|
158
171
|
reason = picked?.verdict === "near_limit" ? "first near-limit member in pool order" : "first healthy member in pool order";
|
|
172
|
+
else if (strategy === "spread")
|
|
173
|
+
reason = picked && (picked.verdict === "ok" || picked.verdict === "no_data") ? describeSpreadPick(picked, windowMs) : "no healthy member; using the existing fallback";
|
|
159
174
|
else if (picked?.verdict === "ok" && picked.sevenDayResetsAt !== undefined)
|
|
160
175
|
reason = `seven-day window resets soonest (in ${relativeUntil(picked.sevenDayResetsAt, nowMs)})`;
|
|
161
176
|
else if (picked?.verdict === "ok")
|
|
@@ -164,16 +179,24 @@ export function renderExplanation(model) {
|
|
|
164
179
|
reason = "first member without live health data";
|
|
165
180
|
else
|
|
166
181
|
reason = "no healthy member; using the existing fallback";
|
|
167
|
-
lines.push(` strategy: ${strategy} — picked ${decision.account}: ${reason}`);
|
|
182
|
+
lines.push(` strategy: ${strategy} — ${strategy === "spread" ? "next new session on" : "picked"} ${decision.account}: ${reason}`);
|
|
183
|
+
if (spread) {
|
|
184
|
+
for (const v of verdicts) {
|
|
185
|
+
lines.push(` ${v.id}: ${v.servedInWindow ?? 0} served in the last ${describeWindow(windowMs)}${v.lastServedAt !== undefined ? `, last ${relativeSince(v.lastServedAt, nowMs)} ago` : ""}`);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
168
188
|
}
|
|
169
189
|
else {
|
|
170
|
-
lines.push(` strategy: ${strategy}`);
|
|
190
|
+
lines.push(` strategy: ${strategy} — pool.accounts[].disabled: every pool member is disabled`);
|
|
171
191
|
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
192
|
+
if (strategy === "spread")
|
|
193
|
+
lines.push(" pool has no home account under spread; existing sessions keep the account that served them");
|
|
194
|
+
else
|
|
195
|
+
lines.push(model.stickyAccount
|
|
196
|
+
? strategy === "reset-aware"
|
|
197
|
+
? ` pool is on ${model.stickyAccount} (dwell applies while it remains usable)`
|
|
198
|
+
: ` pool is rotated onto ${model.stickyAccount} (returns home when the home window resets)`
|
|
199
|
+
: ` pool is on its home account (no rotation active)`);
|
|
177
200
|
}
|
|
178
201
|
}
|
|
179
202
|
return lines.join("\n");
|