@drakon-systems/multi-clawd 2.4.0 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +90 -31
- package/SECURITY.md +3 -0
- package/dist/credential-state.js +8 -24
- package/dist/direct-report.js +8 -1
- package/dist/direct-route.js +9 -1
- package/dist/direct-sync.js +8 -1
- package/dist/explain-core.js +49 -8
- package/dist/health-journal.js +123 -0
- package/dist/health.js +49 -2
- package/dist/history.js +103 -0
- package/dist/index.js +31 -35
- package/dist/retry-plan.js +2 -0
- package/dist/shim-core.js +197 -34
- package/dist/shim-lock.js +106 -0
- package/dist/shim.js +122 -73
- package/dist/spread.js +54 -0
- package/dist/sticky.js +2 -0
- package/dist/update-core.js +75 -1
- package/openclaw.plugin.json +12 -3
- package/package.json +1 -1
- package/scripts/cli.mjs +107 -22
- package/scripts/doctor.mjs +104 -5
package/README.md
CHANGED
|
@@ -30,6 +30,7 @@ multi-clawd update # install (or update) the OpenClaw plugin — right flags,
|
|
|
30
30
|
multi-clawd setup # guided wizard: accounts, isolated second login, pool, watchdog
|
|
31
31
|
multi-clawd login claw2 # launch the right Claude sign-in for an account (or re-auth it)
|
|
32
32
|
multi-clawd explain # your whole setup in plain English — accounts, chain, live health
|
|
33
|
+
multi-clawd explain --history --since 7d # per-account launches, retries, rotations
|
|
33
34
|
multi-clawd account disable claw2 --dry-run # preview benching an account
|
|
34
35
|
multi-clawd chain # audit your model routing — what actually serves each turn
|
|
35
36
|
multi-clawd direct # the direct anthropic/* route: status, or `direct sync` (v1.9, optional)
|
|
@@ -56,6 +57,12 @@ install anything? Every command also runs as
|
|
|
56
57
|
|
|
57
58
|
Then `multi-clawd explain` shows you exactly what you built.
|
|
58
59
|
|
|
60
|
+
`multi-clawd explain --history` adds per-account served launches, in-turn retry
|
|
61
|
+
hops, rotations, and the last served time for the past seven days. Use
|
|
62
|
+
`--since 24h` or another positive `Nd`/`Nh` duration to change the window.
|
|
63
|
+
The local history is size capped; it records account and model IDs and short
|
|
64
|
+
reason codes, never prompts, credentials, transcripts, tokens, or costs.
|
|
65
|
+
|
|
59
66
|
## Hermes Agent integration
|
|
60
67
|
|
|
61
68
|
multi-clawd can also import the configured Claude subscriptions into Hermes
|
|
@@ -256,6 +263,13 @@ claude-cli/claude-fable-5 # main login
|
|
|
256
263
|
session is handed to the sibling as it stood before the refused attempt.
|
|
257
264
|
One retry. Works for config-dir, native, and setup-token-file accounts; the
|
|
258
265
|
shim is handed paths only and reads a token file at the moment it retries.
|
|
266
|
+
A **transient** provider error (529 overload, 5xx, or a throttle 429 that is
|
|
267
|
+
not a spent window) gets the same single hop to an idle sibling — but
|
|
268
|
+
records nothing against the account that met it, because it is not spent,
|
|
269
|
+
and is passed through unchanged when no idle sibling exists (never onto an
|
|
270
|
+
account the pool believes is spent). The Claude CLI retries these itself
|
|
271
|
+
first; the shim keeps the hop armed through those announced retry sleeps,
|
|
272
|
+
bounded (`MULTI_CLAWD_HOLD_RETRY_MAX_MS`, default 120s, hard cap 150s).
|
|
259
273
|
- 📟 **Operator alerts (v0.3)** — dead logins (probed every 15 min without
|
|
260
274
|
spending quota), pool rotations, whole-pool exhaustion, and watchdog
|
|
261
275
|
restarts surface through your agent's next heartbeat (e.g. straight into
|
|
@@ -303,7 +317,7 @@ argument arrays, never through a shell string. Launch mode varies by platform.
|
|
|
303
317
|
npm i -g @drakon-systems/multi-clawd && multi-clawd update
|
|
304
318
|
```
|
|
305
319
|
|
|
306
|
-
`update`
|
|
320
|
+
`update` installs from ClawHub first, offers the gateway
|
|
307
321
|
restart, and finishes with a doctor health check.
|
|
308
322
|
|
|
309
323
|
> ### ⚠️ Keeping up to date: there are two halves
|
|
@@ -334,39 +348,53 @@ restart, and finishes with a doctor health check.
|
|
|
334
348
|
> version` shows both halves, and since v1.6.0 says plainly whether the pair
|
|
335
349
|
> is a problem.
|
|
336
350
|
>
|
|
337
|
-
> ### ⚠️
|
|
351
|
+
> ### ⚠️ Pinned installs need `multi-clawd update`
|
|
338
352
|
>
|
|
339
|
-
>
|
|
340
|
-
>
|
|
341
|
-
> install specs, so pinning is the right side of that trade), and OpenClaw
|
|
353
|
+
> For a versioned ClawHub install (or npm `--pin` fallback), `openclaw plugins
|
|
354
|
+
> update --all` can tell you the plugin is up to date when it is not. OpenClaw
|
|
342
355
|
> resolves registry metadata *for the pinned spec* — comparing 1.6.0 against
|
|
343
356
|
> 1.6.0 and returning "up to date" while a newer version sits on npm. The
|
|
344
357
|
> honest "pinned to X; registry default resolves to Y" message exists in
|
|
345
358
|
> OpenClaw but is built inside its `--dry-run` branch, so a real update run
|
|
346
359
|
> never prints it.
|
|
347
360
|
>
|
|
348
|
-
> **Use `multi-clawd update
|
|
361
|
+
> **Use `multi-clawd update` for versioned installs.** Since v1.7.2 `doctor` also checks the registry
|
|
349
362
|
> itself (cached, and silent when offline), so a lagging plugin is reported
|
|
350
363
|
> rather than left to a command that reassures you wrongly.
|
|
351
364
|
|
|
352
365
|
Prefer the raw form?
|
|
353
366
|
|
|
354
367
|
```bash
|
|
355
|
-
openclaw plugins install
|
|
356
|
-
|
|
368
|
+
openclaw plugins install clawhub:@drakon-systems/multi-clawd
|
|
369
|
+
# Only after a successful install, and with operator approval:
|
|
370
|
+
# openclaw gateway restart
|
|
357
371
|
```
|
|
358
372
|
|
|
359
373
|
Either way the gateway pulls the prebuilt package — no clone, no build step,
|
|
360
|
-
nothing to keep in sync.
|
|
361
|
-
|
|
374
|
+
nothing to keep in sync. ClawHub is the source OpenClaw trusts; npm installs show a source caution.
|
|
375
|
+
For existing installs, prefer the guarded `multi-clawd update` command rather than a raw source retry.
|
|
376
|
+
|
|
377
|
+
**Raw-install fallback boundary:** use npm only after positively identifying a
|
|
378
|
+
ClawHub availability failure (for example, package/version not found or registry
|
|
379
|
+
unreachable), or an older OpenClaw CLI explicitly rejecting the `clawhub:` spec
|
|
380
|
+
as unsupported. An old CLI version alone is not evidence of that incompatibility.
|
|
381
|
+
**Stop; do not retry through npm** for moderation, security, integrity, trust,
|
|
382
|
+
signature or policy failures, unknown errors, cancellation or interruption.
|
|
383
|
+
These stops take precedence even if availability diagnostics appeared first.
|
|
384
|
+
If the cause is unclear, stop and investigate; do not switch sources or restart
|
|
385
|
+
the gateway after a failed or interrupted install.
|
|
386
|
+
|
|
387
|
+
Only after the availability/unsupported-spec check above, an unversioned fresh
|
|
388
|
+
install may use `openclaw plugins install @drakon-systems/multi-clawd --pin`.
|
|
389
|
+
For a versioned install, retain the same selected exact target: if ClawHub was
|
|
390
|
+
`clawhub:@drakon-systems/multi-clawd@X.Y.Z`, the permitted npm retry is
|
|
391
|
+
`openclaw plugins install @drakon-systems/multi-clawd@X.Y.Z --pin`.
|
|
392
|
+
Replace `X.Y.Z` with that same exact version; never drop it, re-resolve latest,
|
|
393
|
+
or substitute a different version during fallback.
|
|
394
|
+
An unversioned ClawHub spec follows newer releases; add `@X.Y.Z` to keep a specific release.
|
|
395
|
+
`openclaw` itself is a *peer*
|
|
362
396
|
dependency (the host provides it), so the install stays lean.
|
|
363
397
|
|
|
364
|
-
**From ClawHub (alternative registry):**
|
|
365
|
-
|
|
366
|
-
```bash
|
|
367
|
-
openclaw plugins install clawhub:@drakon-systems/multi-clawd
|
|
368
|
-
```
|
|
369
|
-
|
|
370
398
|
**From source (contributors, or ahead of a release):**
|
|
371
399
|
|
|
372
400
|
```bash
|
|
@@ -387,7 +415,7 @@ openclaw plugins install (Get-Location).Path
|
|
|
387
415
|
**Or let your agent install it.** Running an OpenClaw assistant or Claude
|
|
388
416
|
Code on the target machine already? Paste it this and go make coffee:
|
|
389
417
|
|
|
390
|
-
> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v2.
|
|
418
|
+
> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v2.5.1/SETUP-AGENT.md
|
|
391
419
|
> and follow it to set up multi-clawd on this machine. I own a second
|
|
392
420
|
> Claude account — ask me when you need me to log in.
|
|
393
421
|
|
|
@@ -412,8 +440,9 @@ host-provided peer.
|
|
|
412
440
|
npx @drakon-systems/multi-clawd update
|
|
413
441
|
```
|
|
414
442
|
|
|
415
|
-
One command: checks the registry, installs the
|
|
416
|
-
|
|
443
|
+
One command: checks the registry, installs the chosen version from ClawHub (with npm
|
|
444
|
+
fallback only when ClawHub is unavailable), offers the gateway restart, and finishes with a doctor health check.
|
|
445
|
+
Security, integrity, and policy failures stop the update without trying npm.
|
|
417
446
|
(`npm i -g @drakon-systems/multi-clawd` once, and it's just `multi-clawd
|
|
418
447
|
update` — with `multi-clawd setup`, `multi-clawd explain` (your setup in
|
|
419
448
|
plain English), and `multi-clawd doctor` alongside.)
|
|
@@ -556,7 +585,9 @@ error.
|
|
|
556
585
|
"pool": {
|
|
557
586
|
"id": "clawd",
|
|
558
587
|
"accounts": ["claw1", "claw2"], // preference order; first = home for ordered strategy
|
|
559
|
-
"strategy": "ordered", // optional: "reset-aware" drains healthy weekly quota that resets soonest
|
|
588
|
+
"strategy": "ordered", // optional: "reset-aware" drains healthy weekly quota that resets soonest;
|
|
589
|
+
// "spread" starts each new session on the least-served healthy account
|
|
590
|
+
// "spreadWindowMs": 3600000, // optional, spread only: how far back served launches count (default 1h)
|
|
560
591
|
"utilizationThreshold": 0.85, // default hand-over; minute/hour and overage windows use this
|
|
561
592
|
"weeklyUtilizationThreshold": 0.97, // optional: day/week/month quota windows at 97%
|
|
562
593
|
"minDwellMs": 600000, // v0.3: anti-flap dwell before returning home
|
|
@@ -602,7 +633,14 @@ a gateway reload. A turn already admitted or running continues on its selected
|
|
|
602
633
|
account; the command does not interrupt it.
|
|
603
634
|
|
|
604
635
|
`pool.strategy` defaults to `"ordered"`, which retains the configured account
|
|
605
|
-
order.
|
|
636
|
+
order. In every strategy, when no `ok` or `no_data` account is left but some
|
|
637
|
+
are near their limit, the launch goes to the near-limit account with the most
|
|
638
|
+
headroom: the lowest fresh utilization, taking the newer of the stream and
|
|
639
|
+
usage-poll records for each window. Equal figures keep pool order, an account
|
|
640
|
+
with a figure outranks one without, and two without keep pool order.
|
|
641
|
+
Exhausted and credential-failed accounts are never chosen. `explain` says
|
|
642
|
+
"all near limit, picked X at N%" and the direct Anthropic route orders its
|
|
643
|
+
near-limit profiles the same way. Set it to `"reset-aware"` to use healthy weekly quota before its
|
|
606
644
|
`seven_day` window resets: among `ok` accounts with live seven-day reset data,
|
|
607
645
|
the soonest reset wins. Accounts without that data follow in pool order;
|
|
608
646
|
near-limit or exhausted accounts never outrank a healthy one. Equal reset
|
|
@@ -610,6 +648,25 @@ times keep pool order. The existing `minDwellMs` delay still holds an account
|
|
|
610
648
|
that remains usable. Both CLI launches and the direct Anthropic route use the
|
|
611
649
|
same preference. Other values are rejected by the plugin config schema.
|
|
612
650
|
|
|
651
|
+
Set `pool.strategy` to `"spread"` when several agent lanes start sessions at
|
|
652
|
+
once and you want them on different accounts instead of queueing behind one
|
|
653
|
+
account's concurrency ceiling. Each **new** session starts on the healthy
|
|
654
|
+
(`ok` or `no_data`) account that served the fewest launches in the last
|
|
655
|
+
`spreadWindowMs` (default one hour; clamped between one minute and seven
|
|
656
|
+
days), counted from the served-launch journal `explain --history` reads. On a
|
|
657
|
+
tie the account served least recently wins, then pool order, so equally idle
|
|
658
|
+
accounts are used in turn. Near-limit, exhausted and credential-failed
|
|
659
|
+
accounts are never chosen; when no healthy account is left the pool answers
|
|
660
|
+
exactly as `ordered` does. Spread keeps every session on the account that
|
|
661
|
+
first served it: it switches `sessionAffinity` on for the pool (an explicit
|
|
662
|
+
`sessionAffinity: false` is overridden with a warning, because without it a
|
|
663
|
+
conversation would move on every turn). A pinned session moves only when its
|
|
664
|
+
account stops being healthy, exactly as under `sessionAffinity`. Launches
|
|
665
|
+
without a session id take the spread pick. There is no home account, no
|
|
666
|
+
dwell and no rotation alert under spread; `explain`, `usage` and `doctor`
|
|
667
|
+
show the strategy, the next placement and the served counts behind it. The
|
|
668
|
+
direct Anthropic route orders its profiles by the same counts.
|
|
669
|
+
|
|
613
670
|
`pool.sessionAffinity: true` remembers the account that successfully served each
|
|
614
671
|
Claude CLI session (`--session-id` or `--resume`) and prefers it on later turns
|
|
615
672
|
while its health is `ok` or `no_data`. A near-limit or otherwise unhealthy
|
|
@@ -637,6 +694,7 @@ How it decides, per launch (all data from each account's live
|
|
|
637
694
|
| `allowed_warning` and **no** number, on a short window | skipped (v1.7.2) — see below |
|
|
638
695
|
| `allowed_warning` and no number, on the weekly window | still used — weekly warns from ~30% and would flap |
|
|
639
696
|
| no data / stale data | used — never rotate on missing evidence |
|
|
697
|
+
| every usable account near its limit | the near-limit account with the most headroom (lowest fresh utilization) serves; equal figures and accounts without a number keep pool order |
|
|
640
698
|
| whole pool exhausted | home account anyway → real limit error → your chain drops provider |
|
|
641
699
|
|
|
642
700
|
**Why the short-window rule exists (v1.7.2).** (v1.10's live usage poll now
|
|
@@ -892,15 +950,16 @@ event, and no plugin API can force a rebuild. See `DESIGN.md`.
|
|
|
892
950
|
after the assessment or 24 hours after that turn, whichever comes first. A
|
|
893
951
|
CLI login failure recorded on a real turn removes it at once.
|
|
894
952
|
|
|
895
|
-
- **Shim
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
953
|
+
- **Shim state and retry coordination is best effort under lock contention.**
|
|
954
|
+
Concurrent shims for one account use a short advisory lock around each
|
|
955
|
+
state merge and rename. A second short lock covers recording a refusal and
|
|
956
|
+
reading sibling health for the in-turn retry. Later decisions can therefore
|
|
957
|
+
see rejections already recorded by another shim. If a lock stays busy beyond
|
|
958
|
+
150 ms or filesystem locking fails, the shim proceeds without it so the
|
|
959
|
+
turn still runs; concurrent writes in that fallback can lose an observation.
|
|
960
|
+
A replacement between the lock's final ownership check and removal can
|
|
961
|
+
still remove another live holder's lock because those steps are not atomic.
|
|
962
|
+
The guard does not pace requests sent to a newly chosen sibling.
|
|
904
963
|
|
|
905
964
|
## Security
|
|
906
965
|
|
|
@@ -915,7 +974,7 @@ callsite (the ones static scanners flag) and why each exists.
|
|
|
915
974
|
| Network | The gateway plugin polls `https://api.anthropic.com/api/oauth/usage` for eligible pooled accounts on an interval (default 120000 ms / 2 minutes); `pool.usagePoll.enabled: false` disables that poll. `multi-clawd usage` requests the same endpoint on demand. `doctor` runs `npm view` when its registry cache is stale; install and update commands contact the package registry. Claude Code subprocesses make their own provider requests. No plugin telemetry or analytics endpoint. |
|
|
916
975
|
| Credentials | Reads declared setup-token files or gateway-resolved `oauthTokenRef` values for Claude child launches; reads each eligible account's `.credentials.json` access token for the usage poll; reads Claude login credentials from files or the macOS Keychain via the `security` CLI for login checks and runtime auth; reads account identity for file-backed logins from `.claude.json` (`~/.claude.json` for a native login, or the account config dir). Setup tokens are passed to Claude through the child environment. |
|
|
917
976
|
| Files read | `~/.openclaw/openclaw.json`, declared account config dirs and token files, Claude `.credentials.json` and `.claude.json` files as described above, and local state files. |
|
|
918
|
-
| Files written | Under `~/.openclaw/state/multi-clawd/` by default: `<id>.json`, `<id>.usage.json`, `pool-<id>.sticky.json`, `pool-<id>.sessions.json` (when affinity is enabled), `direct-order.sticky.json`, `known-models.json`, `alerts-spool.jsonl`, `watchdog.json`, `registry-check.json`, `auth-profiles-<agent>.json`, `runtime/<hash>/` (copied runtime files), and `watchdog-launcher.mjs`. The watchdog state file can be redirected with `MULTI_CLAWD_WATCHDOG_STATE`. Setup writes `~/.openclaw/openclaw.json` and a dated backup. On session handover, transcript `.jsonl` files are copied between account config dirs. The optional watchdog installs `~/.config/systemd/user/multi-clawd-eviction-watchdog.service` and `.timer` on Linux, or `~/Library/LaunchAgents/com.multi-clawd.eviction-watchdog.plist` on macOS; the unit points at the state-dir launcher. |
|
|
977
|
+
| Files written | Under `~/.openclaw/state/multi-clawd/` by default: `<id>.json`, `<id>.usage.json`, `history.jsonl` (bounded account activity), `pool-<id>.sticky.json`, `pool-<id>.sessions.json` (when affinity is enabled), `direct-order.sticky.json`, `known-models.json`, `alerts-spool.jsonl`, `watchdog.json`, `registry-check.json`, `auth-profiles-<agent>.json`, `runtime/<hash>/` (copied runtime files), and `watchdog-launcher.mjs`. The watchdog state file can be redirected with `MULTI_CLAWD_WATCHDOG_STATE`. Setup writes `~/.openclaw/openclaw.json` and a dated backup. On session handover, transcript `.jsonl` files are copied between account config dirs. The optional watchdog installs `~/.config/systemd/user/multi-clawd-eviction-watchdog.service` and `.timer` on Linux, or `~/Library/LaunchAgents/com.multi-clawd.eviction-watchdog.plist` on macOS; the unit points at the state-dir launcher. |
|
|
919
978
|
| Processes | Launches Claude Code and the node + shim transport; CLI, setup and doctor run npm, OpenClaw, Claude and OS scheduler commands; the optional watchdog checks OpenClaw logs and may restart the gateway; Hermes integration runs its Python bridge. See [SECURITY.md](SECURITY.md) for the process-call table. |
|
|
920
979
|
| Consent | The wizard asks before every write, merges non-destructively, and never overwrites an existing account entry. `--dry-run` previews everything. |
|
|
921
980
|
|
package/SECURITY.md
CHANGED
|
@@ -37,6 +37,9 @@ credit you (unless you'd rather we didn't), and note it in the CHANGELOG.
|
|
|
37
37
|
- **Per-account health and usage files are written `0600`,** using temporary
|
|
38
38
|
files and rename, and contain rate-limit data rather than credentials. Other
|
|
39
39
|
state files include model, alert, registry and watchdog data.
|
|
40
|
+
- **`history.jsonl` is a bounded `0600` account activity file** in the same
|
|
41
|
+
state directory. It contains timestamps, account/model IDs and short reason
|
|
42
|
+
codes, with no prompt, transcript, token, cost, credential, or environment values.
|
|
40
43
|
|
|
41
44
|
## Permissions
|
|
42
45
|
|
package/dist/credential-state.js
CHANGED
|
@@ -1,33 +1,17 @@
|
|
|
1
|
-
import { readFileSync, writeFileSync, mkdirSync, renameSync } from "node:fs";
|
|
2
1
|
import { homedir } from "node:os";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { persistHealthState, readHealthStateFile } from "./health-journal.js";
|
|
4
|
+
import { clearCredentialFailure, } from "./shim-core.js";
|
|
5
5
|
export function healthStateFile(accountId) {
|
|
6
6
|
return join(homedir(), ".openclaw", "state", "multi-clawd", `${accountId}.json`);
|
|
7
7
|
}
|
|
8
8
|
export function usageStateFile(accountId) {
|
|
9
9
|
return join(homedir(), ".openclaw", "state", "multi-clawd", `${accountId}.usage.json`);
|
|
10
10
|
}
|
|
11
|
-
export function clearAccountCredentialFailure(accountId) {
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
try {
|
|
15
|
-
state = parseStoredState(readFileSync(file, "utf8")) ?? { accountId, windows: {} };
|
|
16
|
-
}
|
|
17
|
-
catch {
|
|
11
|
+
export function clearAccountCredentialFailure(accountId, file = healthStateFile(accountId)) {
|
|
12
|
+
const state = readHealthStateFile(file);
|
|
13
|
+
if (state?.credential?.status !== "failed")
|
|
18
14
|
return false;
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
return false;
|
|
22
|
-
const cleared = mergeHealthStates(state, clearCredentialFailure(state, Date.now()), Date.now());
|
|
23
|
-
try {
|
|
24
|
-
mkdirSync(dirname(file), { recursive: true });
|
|
25
|
-
const tmp = `${file}.tmp-${process.pid}`;
|
|
26
|
-
writeFileSync(tmp, JSON.stringify(cleared, null, 2), { mode: 0o600 });
|
|
27
|
-
renameSync(tmp, file);
|
|
28
|
-
return true;
|
|
29
|
-
}
|
|
30
|
-
catch {
|
|
31
|
-
return false;
|
|
32
|
-
}
|
|
15
|
+
persistHealthState(file, clearCredentialFailure(state, Date.now()));
|
|
16
|
+
return readHealthStateFile(file)?.credential?.status === "ok";
|
|
33
17
|
}
|
package/dist/direct-report.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { classifyAccountHealth, sevenDayReset } from "./health.js";
|
|
2
|
+
import { spreadStatsFor, spreadStatsFromHistory, spreadWindowMs } from "./spread.js";
|
|
2
3
|
import { DIRECT_PROVIDER, collectDirectMembers, } from "./direct-route.js";
|
|
3
4
|
import { parseUnusableProfiles, planAgentOrder, readDirectRouteSnapshot, safeCliError, } from "./direct-sync.js";
|
|
4
5
|
export function describeDirectSource(member) {
|
|
@@ -21,16 +22,22 @@ export async function gatherDirectStatus(params) {
|
|
|
21
22
|
if (members.length === 0 && problems.length === 0)
|
|
22
23
|
return undefined;
|
|
23
24
|
const errors = [];
|
|
25
|
+
const spread = params.strategy === "spread"
|
|
26
|
+
? spreadStatsFromHistory(params.readHistory?.() ?? [], params.nowMs, spreadWindowMs(params.spreadWindowMs))
|
|
27
|
+
: undefined;
|
|
24
28
|
const verdicts = members.map((m) => {
|
|
25
29
|
const state = params.readHealth(m.accountId);
|
|
30
|
+
const health = classifyAccountHealth(state, params.healthOptions, params.nowMs);
|
|
26
31
|
return {
|
|
27
32
|
accountId: m.accountId,
|
|
28
33
|
profileId: m.profileId,
|
|
29
34
|
disabled: m.disabled,
|
|
30
|
-
verdict:
|
|
35
|
+
verdict: health.verdict,
|
|
36
|
+
utilization: health.utilization,
|
|
31
37
|
sevenDayResetsAt: params.strategy === "reset-aware"
|
|
32
38
|
? sevenDayReset(state, params.healthOptions, params.nowMs)
|
|
33
39
|
: undefined,
|
|
40
|
+
...(spread ? spreadStatsFor(spread, m.accountId) : {}),
|
|
34
41
|
};
|
|
35
42
|
});
|
|
36
43
|
const read = members.length > 0 ? await readDirectRouteSnapshot(params.runner, params.agentId) : {};
|
package/dist/direct-route.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { isSecretRefShape } from "./token-resolution.js";
|
|
2
|
+
import { headroomBefore } from "./health.js";
|
|
2
3
|
import { decideStickySelection } from "./sticky.js";
|
|
4
|
+
import { spreadBefore } from "./spread.js";
|
|
3
5
|
export const DIRECT_PROVIDER = "anthropic";
|
|
4
6
|
export const DIRECT_PROFILE_PREFIX = `${DIRECT_PROVIDER}:`;
|
|
5
7
|
export const DIRECT_SETUP_TOKEN_GUIDANCE = "a native or config-dir Claude login is a rotating single-use OAuth grant — copying it into " +
|
|
@@ -134,7 +136,7 @@ export function planDirectOrder(params) {
|
|
|
134
136
|
if (members.length === 0)
|
|
135
137
|
throw new Error("pool.accounts[].disabled: every direct member is disabled");
|
|
136
138
|
const decision = decideStickySelection({
|
|
137
|
-
verdicts: members.map((m) => ({ id: m.accountId, verdict: m.verdict, sevenDayResetsAt: m.sevenDayResetsAt })),
|
|
139
|
+
verdicts: members.map((m) => ({ id: m.accountId, verdict: m.verdict, sevenDayResetsAt: m.sevenDayResetsAt, utilization: m.utilization, servedInWindow: m.servedInWindow, lastServedAt: m.lastServedAt })),
|
|
138
140
|
sticky: params.sticky,
|
|
139
141
|
nowMs: params.nowMs,
|
|
140
142
|
minDwellMs: params.minDwellMs,
|
|
@@ -147,6 +149,12 @@ export function planDirectOrder(params) {
|
|
|
147
149
|
.sort((a, b) => VERDICT_RANK[a.m.verdict] - VERDICT_RANK[b.m.verdict] ||
|
|
148
150
|
(params.strategy === "reset-aware" && a.m.verdict === "ok"
|
|
149
151
|
? (a.m.sevenDayResetsAt ?? Infinity) - (b.m.sevenDayResetsAt ?? Infinity)
|
|
152
|
+
: 0) ||
|
|
153
|
+
(params.strategy === "spread" && VERDICT_RANK[a.m.verdict] === 0
|
|
154
|
+
? (spreadBefore(a.m, b.m) ? -1 : spreadBefore(b.m, a.m) ? 1 : 0)
|
|
155
|
+
: 0) ||
|
|
156
|
+
(a.m.verdict === "near_limit" && b.m.verdict === "near_limit"
|
|
157
|
+
? (headroomBefore(a.m, b.m) ? -1 : headroomBefore(b.m, a.m) ? 1 : 0)
|
|
150
158
|
: 0) || a.index - b.index)
|
|
151
159
|
.map(({ m }) => m.profileId);
|
|
152
160
|
const managed = new Set(params.members.map((m) => m.profileId));
|
package/dist/direct-sync.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { classifyAccountHealth, sevenDayReset } from "./health.js";
|
|
2
|
+
import { spreadStatsFor, spreadStatsFromHistory, spreadWindowMs } from "./spread.js";
|
|
2
3
|
import { DIRECT_PROVIDER, planDirectOrder } from "./direct-route.js";
|
|
3
4
|
import { parseClaudeSetupToken } from "./hermes-core.js";
|
|
4
5
|
export function safeCliError(result) {
|
|
@@ -247,16 +248,22 @@ export function createDirectOrderController(deps) {
|
|
|
247
248
|
return { managedOrder: [], agents: [] };
|
|
248
249
|
running = true;
|
|
249
250
|
try {
|
|
251
|
+
const spread = deps.strategy === "spread"
|
|
252
|
+
? spreadStatsFromHistory(deps.readHistory?.() ?? [], nowMs, spreadWindowMs(deps.spreadWindowMs))
|
|
253
|
+
: undefined;
|
|
250
254
|
const verdicts = deps.members.map((m) => {
|
|
251
255
|
const state = deps.readHealth(m.accountId);
|
|
256
|
+
const health = classifyAccountHealth(state, deps.healthOptions, nowMs);
|
|
252
257
|
return {
|
|
253
258
|
accountId: m.accountId,
|
|
254
259
|
profileId: m.profileId,
|
|
255
260
|
disabled: m.disabled,
|
|
256
|
-
verdict:
|
|
261
|
+
verdict: health.verdict,
|
|
262
|
+
utilization: health.utilization,
|
|
257
263
|
sevenDayResetsAt: deps.strategy === "reset-aware"
|
|
258
264
|
? sevenDayReset(state, deps.healthOptions, nowMs)
|
|
259
265
|
: undefined,
|
|
266
|
+
...(spread ? spreadStatsFor(spread, m.accountId) : {}),
|
|
260
267
|
};
|
|
261
268
|
});
|
|
262
269
|
const sticky = deps.readSticky();
|
package/dist/explain-core.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { DEFAULT_UTILIZATION_THRESHOLD, choosePoolAccount, poolStrategy } from "./health.js";
|
|
2
2
|
import { decideStickySelection } from "./sticky.js";
|
|
3
|
+
import { describeSpreadPick, describeWindow, spreadStatsFromHistory, spreadWindowMs, withSpreadStats } from "./spread.js";
|
|
3
4
|
const WINDOW_LABELS = {
|
|
4
5
|
five_hour: "5-hour",
|
|
5
6
|
seven_day: "weekly",
|
|
@@ -16,6 +17,9 @@ export function relativeUntil(ms, nowMs) {
|
|
|
16
17
|
return `~${hours}h`;
|
|
17
18
|
return `~${Math.round(hours / 24)}d`;
|
|
18
19
|
}
|
|
20
|
+
export function relativeSince(ms, nowMs) {
|
|
21
|
+
return relativeUntil(nowMs + Math.max(0, nowMs - ms), nowMs);
|
|
22
|
+
}
|
|
19
23
|
export function renderUsageLine(usage, nowMs) {
|
|
20
24
|
return usage
|
|
21
25
|
.map((u) => {
|
|
@@ -93,6 +97,12 @@ export function renderExplanation(model) {
|
|
|
93
97
|
if (poolStrategy(model.pool.strategy) === "reset-aware") {
|
|
94
98
|
lines.push(" Healthy accounts with a live seven-day reset are preferred by soonest reset.");
|
|
95
99
|
}
|
|
100
|
+
else if (poolStrategy(model.pool.strategy) === "spread") {
|
|
101
|
+
lines.push(` Each new session starts on the healthy account with the fewest served launches in the last ${describeWindow(spreadWindowMs(model.pool.spreadWindowMs))};`);
|
|
102
|
+
lines.push(" a session then stays on its account while that account is healthy (session affinity is on).");
|
|
103
|
+
if (model.pool.sessionAffinity === false)
|
|
104
|
+
lines.push(" sessionAffinity: false is ignored under spread — without it a conversation would move every turn.");
|
|
105
|
+
}
|
|
96
106
|
else {
|
|
97
107
|
lines.push(" Every Claude launch runs on the first account below its threshold;");
|
|
98
108
|
lines.push(" the home account reclaims work");
|
|
@@ -138,15 +148,19 @@ export function renderExplanation(model) {
|
|
|
138
148
|
}
|
|
139
149
|
if (model.pool) {
|
|
140
150
|
const strategy = poolStrategy(model.pool.strategy);
|
|
141
|
-
const
|
|
151
|
+
const windowMs = spreadWindowMs(model.pool.spreadWindowMs);
|
|
152
|
+
const spread = strategy === "spread" ? spreadStatsFromHistory(model.history ?? [], nowMs, windowMs) : undefined;
|
|
153
|
+
const rawVerdicts = model.pool.accounts.filter((id) => !model.accounts.find((a) => a.id === id)?.disabled).map((id) => {
|
|
142
154
|
const h = model.health.find((row) => row.id === id);
|
|
143
155
|
return {
|
|
144
156
|
id,
|
|
145
157
|
verdict: h?.verdict ?? "no_data",
|
|
158
|
+
utilization: h?.utilization,
|
|
146
159
|
sevenDayResetsAt: (h?.usage?.find((u) => u.window === "usage:seven_day" && u.resetsAt !== undefined && u.resetsAt > nowMs)
|
|
147
160
|
?? h?.usage?.find((u) => u.window === "seven_day" && u.resetsAt !== undefined && u.resetsAt > nowMs))?.resetsAt,
|
|
148
161
|
};
|
|
149
162
|
});
|
|
163
|
+
const verdicts = spread ? withSpreadStats(rawVerdicts, spread) : rawVerdicts;
|
|
150
164
|
if (verdicts.length) {
|
|
151
165
|
const preferred = choosePoolAccount(verdicts, strategy);
|
|
152
166
|
const decision = decideStickySelection({ verdicts, sticky: model.stickyEntry, nowMs, minDwellMs: model.pool.minDwellMs, strategy });
|
|
@@ -154,8 +168,12 @@ export function renderExplanation(model) {
|
|
|
154
168
|
let reason;
|
|
155
169
|
if (preferred && decision.account !== preferred)
|
|
156
170
|
reason = "sticky dwell keeps this usable account";
|
|
171
|
+
else if (picked?.verdict === "near_limit")
|
|
172
|
+
reason = describeNearLimitPick(verdicts, picked);
|
|
157
173
|
else if (strategy === "ordered")
|
|
158
|
-
reason =
|
|
174
|
+
reason = "first healthy member in pool order";
|
|
175
|
+
else if (strategy === "spread")
|
|
176
|
+
reason = picked && (picked.verdict === "ok" || picked.verdict === "no_data") ? describeSpreadPick(picked, windowMs) : "no healthy member; using the existing fallback";
|
|
159
177
|
else if (picked?.verdict === "ok" && picked.sevenDayResetsAt !== undefined)
|
|
160
178
|
reason = `seven-day window resets soonest (in ${relativeUntil(picked.sevenDayResetsAt, nowMs)})`;
|
|
161
179
|
else if (picked?.verdict === "ok")
|
|
@@ -164,20 +182,43 @@ export function renderExplanation(model) {
|
|
|
164
182
|
reason = "first member without live health data";
|
|
165
183
|
else
|
|
166
184
|
reason = "no healthy member; using the existing fallback";
|
|
167
|
-
lines.push(` strategy: ${strategy} — picked ${decision.account}: ${reason}`);
|
|
185
|
+
lines.push(` strategy: ${strategy} — ${strategy === "spread" ? "next new session on" : "picked"} ${decision.account}: ${reason}`);
|
|
186
|
+
if (spread) {
|
|
187
|
+
for (const v of verdicts) {
|
|
188
|
+
lines.push(` ${v.id}: ${v.servedInWindow ?? 0} served in the last ${describeWindow(windowMs)}${v.lastServedAt !== undefined ? `, last ${relativeSince(v.lastServedAt, nowMs)} ago` : ""}`);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
168
191
|
}
|
|
169
192
|
else {
|
|
170
193
|
lines.push(` strategy: ${strategy} — pool.accounts[].disabled: every pool member is disabled`);
|
|
171
194
|
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
195
|
+
if (strategy === "spread")
|
|
196
|
+
lines.push(" pool has no home account under spread; existing sessions keep the account that served them");
|
|
197
|
+
else
|
|
198
|
+
lines.push(model.stickyAccount
|
|
199
|
+
? strategy === "reset-aware"
|
|
200
|
+
? ` pool is on ${model.stickyAccount} (dwell applies while it remains usable)`
|
|
201
|
+
: ` pool is rotated onto ${model.stickyAccount} (returns home when the home window resets)`
|
|
202
|
+
: ` pool is on its home account (no rotation active)`);
|
|
177
203
|
}
|
|
178
204
|
}
|
|
179
205
|
return lines.join("\n");
|
|
180
206
|
}
|
|
207
|
+
export function describeNearLimitPick(verdicts, picked) {
|
|
208
|
+
const nearLimit = verdicts.filter((v) => v.verdict === "near_limit");
|
|
209
|
+
const scope = nearLimit.length === verdicts.length ? "all near limit" : "no healthy member, the rest near limit";
|
|
210
|
+
const others = nearLimit.filter((v) => v.id !== picked.id);
|
|
211
|
+
if (picked.utilization === undefined) {
|
|
212
|
+
return `${scope}, picked ${picked.id} (no utilization number; pool order)`;
|
|
213
|
+
}
|
|
214
|
+
const pct = `${Math.round(picked.utilization * 100)}%`;
|
|
215
|
+
if (others.length === 0)
|
|
216
|
+
return `${scope}, picked ${picked.id} at ${pct}`;
|
|
217
|
+
const shown = others
|
|
218
|
+
.map((v) => `${v.id} ${v.utilization === undefined ? "no number" : `${Math.round(v.utilization * 100)}%`}`)
|
|
219
|
+
.join(", ");
|
|
220
|
+
return `${scope}, picked ${picked.id} at ${pct} — most headroom (${shown})`;
|
|
221
|
+
}
|
|
181
222
|
export function renderDirectSection(direct, nowMs) {
|
|
182
223
|
const lines = [];
|
|
183
224
|
lines.push("DIRECT ROUTE anthropic/* (gateway → Anthropic API with each account's setup-token)");
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { mkdirSync, readFileSync, readdirSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { basename, dirname, join } from "node:path";
|
|
4
|
+
import { acquireShimLock } from "./shim-lock.js";
|
|
5
|
+
import { mergeHealthStates, parseStoredState } from "./shim-core.js";
|
|
6
|
+
function pendingFiles(file) {
|
|
7
|
+
const prefix = `${basename(file)}.pending-`;
|
|
8
|
+
try {
|
|
9
|
+
return readdirSync(dirname(file)).filter((name) => name.startsWith(prefix) && /^\d+-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name.slice(prefix.length))).map((name) => join(dirname(file), name));
|
|
10
|
+
}
|
|
11
|
+
catch {
|
|
12
|
+
return [];
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
function snapshot(file, onError, strict = false) {
|
|
16
|
+
let raw;
|
|
17
|
+
try {
|
|
18
|
+
raw = readFileSync(file, "utf8");
|
|
19
|
+
}
|
|
20
|
+
catch (err) {
|
|
21
|
+
if (err.code !== "ENOENT") {
|
|
22
|
+
onError?.(undefined);
|
|
23
|
+
if (strict)
|
|
24
|
+
throw err;
|
|
25
|
+
}
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
const state = parseStoredState(raw);
|
|
29
|
+
if (!state) {
|
|
30
|
+
onError?.(raw);
|
|
31
|
+
if (strict)
|
|
32
|
+
throw new Error("not valid health-state JSON");
|
|
33
|
+
}
|
|
34
|
+
return state;
|
|
35
|
+
}
|
|
36
|
+
export function readHealthStateFile(file, opts = {}) {
|
|
37
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
38
|
+
let state = snapshot(file, opts.onSnapshotError, opts.strictSnapshot);
|
|
39
|
+
const files = pendingFiles(file);
|
|
40
|
+
opts.afterList?.(files);
|
|
41
|
+
let missing = false;
|
|
42
|
+
for (const pending of files) {
|
|
43
|
+
let raw;
|
|
44
|
+
try {
|
|
45
|
+
raw = readFileSync(pending, "utf8");
|
|
46
|
+
}
|
|
47
|
+
catch (err) {
|
|
48
|
+
if (err.code === "ENOENT")
|
|
49
|
+
missing = true;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
const delta = parseStoredState(raw);
|
|
53
|
+
if (delta)
|
|
54
|
+
state = state ? mergeHealthStates(state, delta) : delta;
|
|
55
|
+
}
|
|
56
|
+
if (!missing || attempt === 2)
|
|
57
|
+
return state;
|
|
58
|
+
}
|
|
59
|
+
return undefined;
|
|
60
|
+
}
|
|
61
|
+
function publish(file, state) {
|
|
62
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
63
|
+
const tmp = `${file}.tmp-${process.pid}-${randomUUID()}`;
|
|
64
|
+
try {
|
|
65
|
+
writeFileSync(tmp, JSON.stringify(state, null, 2), { mode: 0o600 });
|
|
66
|
+
renameSync(tmp, file);
|
|
67
|
+
}
|
|
68
|
+
catch (err) {
|
|
69
|
+
try {
|
|
70
|
+
unlinkSync(tmp);
|
|
71
|
+
}
|
|
72
|
+
catch { }
|
|
73
|
+
throw err;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
export function persistHealthState(file, state, opts = {}) {
|
|
77
|
+
const release = acquireShimLock(file);
|
|
78
|
+
try {
|
|
79
|
+
if (!release) {
|
|
80
|
+
publish(`${file}.pending-${process.pid}-${randomUUID()}`, state);
|
|
81
|
+
return state;
|
|
82
|
+
}
|
|
83
|
+
let merged = snapshot(file, opts.onSnapshotError);
|
|
84
|
+
const files = pendingFiles(file);
|
|
85
|
+
opts.afterList?.(files);
|
|
86
|
+
const folded = [];
|
|
87
|
+
for (const pending of files) {
|
|
88
|
+
let raw;
|
|
89
|
+
try {
|
|
90
|
+
raw = readFileSync(pending, "utf8");
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
const delta = parseStoredState(raw);
|
|
96
|
+
if (!delta) {
|
|
97
|
+
try {
|
|
98
|
+
renameSync(pending, `${pending}.corrupt-${Date.now()}`);
|
|
99
|
+
}
|
|
100
|
+
catch { }
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
merged = merged ? mergeHealthStates(merged, delta) : delta;
|
|
104
|
+
folded.push(pending);
|
|
105
|
+
}
|
|
106
|
+
merged = merged ? mergeHealthStates(merged, state, Date.now()) : mergeHealthStates(state, state, Date.now());
|
|
107
|
+
publish(file, merged);
|
|
108
|
+
for (const pending of folded) {
|
|
109
|
+
try {
|
|
110
|
+
unlinkSync(pending);
|
|
111
|
+
}
|
|
112
|
+
catch { }
|
|
113
|
+
}
|
|
114
|
+
return merged;
|
|
115
|
+
}
|
|
116
|
+
catch (err) {
|
|
117
|
+
process.stderr.write(`[multi-clawd shim] state write failed: ${String(err)}\n`);
|
|
118
|
+
return state;
|
|
119
|
+
}
|
|
120
|
+
finally {
|
|
121
|
+
release?.();
|
|
122
|
+
}
|
|
123
|
+
}
|