@drakon-systems/multi-clawd 2.4.0 → 2.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -30,6 +30,7 @@ multi-clawd update # install (or update) the OpenClaw plugin — right flags,
30
30
  multi-clawd setup # guided wizard: accounts, isolated second login, pool, watchdog
31
31
  multi-clawd login claw2 # launch the right Claude sign-in for an account (or re-auth it)
32
32
  multi-clawd explain # your whole setup in plain English — accounts, chain, live health
33
+ multi-clawd explain --history --since 7d # per-account launches, retries, rotations
33
34
  multi-clawd account disable claw2 --dry-run # preview benching an account
34
35
  multi-clawd chain # audit your model routing — what actually serves each turn
35
36
  multi-clawd direct # the direct anthropic/* route: status, or `direct sync` (v1.9, optional)
@@ -56,6 +57,12 @@ install anything? Every command also runs as
56
57
 
57
58
  Then `multi-clawd explain` shows you exactly what you built.
58
59
 
60
+ `multi-clawd explain --history` adds per-account served launches, in-turn retry
61
+ hops, rotations, and the last served time for the past seven days. Use
62
+ `--since 24h` or another positive `Nd`/`Nh` duration to change the window.
63
+ The local history is size capped; it records account and model IDs and short
64
+ reason codes, never prompts, credentials, transcripts, tokens, or costs.
65
+
59
66
  ## Hermes Agent integration
60
67
 
61
68
  multi-clawd can also import the configured Claude subscriptions into Hermes
@@ -256,6 +263,13 @@ claude-cli/claude-fable-5 # main login
256
263
  session is handed to the sibling as it stood before the refused attempt.
257
264
  One retry. Works for config-dir, native, and setup-token-file accounts; the
258
265
  shim is handed paths only and reads a token file at the moment it retries.
266
+ A **transient** provider error (529 overload, 5xx, or a throttle 429 that is
267
+ not a spent window) gets the same single hop to an idle sibling — but
268
+ records nothing against the account that met it, because it is not spent,
269
+ and is passed through unchanged when no idle sibling exists (never onto an
270
+ account the pool believes is spent). The Claude CLI retries these itself
271
+ first; the shim keeps the hop armed through those announced retry sleeps,
272
+ bounded (`MULTI_CLAWD_HOLD_RETRY_MAX_MS`, default 120s, hard cap 150s).
259
273
  - 📟 **Operator alerts (v0.3)** — dead logins (probed every 15 min without
260
274
  spending quota), pool rotations, whole-pool exhaustion, and watchdog
261
275
  restarts surface through your agent's next heartbeat (e.g. straight into
@@ -303,7 +317,7 @@ argument arrays, never through a shell string. Launch mode varies by platform.
303
317
  npm i -g @drakon-systems/multi-clawd && multi-clawd update
304
318
  ```
305
319
 
306
- `update` runs the registry install with the right flags, offers the gateway
320
+ `update` installs from ClawHub first, offers the gateway
307
321
  restart, and finishes with a doctor health check.
308
322
 
309
323
  > ### ⚠️ Keeping up to date: there are two halves
@@ -334,39 +348,53 @@ restart, and finishes with a doctor health check.
334
348
  > version` shows both halves, and since v1.6.0 says plainly whether the pair
335
349
  > is a problem.
336
350
  >
337
- > ### ⚠️ `openclaw plugins update --all` cannot move this plugin
351
+ > ### ⚠️ Pinned installs need `multi-clawd update`
338
352
  >
339
- > It will tell you the plugin is up to date when it is not. We install with
340
- > `--pin` (OpenClaw's own security audit raises a HIGH finding for unpinned
341
- > install specs, so pinning is the right side of that trade), and OpenClaw
353
+ > For a versioned ClawHub install (or npm `--pin` fallback), `openclaw plugins
354
+ > update --all` can tell you the plugin is up to date when it is not. OpenClaw
342
355
  > resolves registry metadata *for the pinned spec* — comparing 1.6.0 against
343
356
  > 1.6.0 and returning "up to date" while a newer version sits on npm. The
344
357
  > honest "pinned to X; registry default resolves to Y" message exists in
345
358
  > OpenClaw but is built inside its `--dry-run` branch, so a real update run
346
359
  > never prints it.
347
360
  >
348
- > **Use `multi-clawd update`.** Since v1.7.2 `doctor` also checks the registry
361
+ > **Use `multi-clawd update` for versioned installs.** Since v1.7.2 `doctor` also checks the registry
349
362
  > itself (cached, and silent when offline), so a lagging plugin is reported
350
363
  > rather than left to a command that reassures you wrongly.
351
364
 
352
365
  Prefer the raw form?
353
366
 
354
367
  ```bash
355
- openclaw plugins install @drakon-systems/multi-clawd --pin
356
- openclaw gateway restart
368
+ openclaw plugins install clawhub:@drakon-systems/multi-clawd
369
+ # Only after a successful install, and with operator approval:
370
+ # openclaw gateway restart
357
371
  ```
358
372
 
359
373
  Either way the gateway pulls the prebuilt package — no clone, no build step,
360
- nothing to keep in sync. `--pin` records the exact resolved version, so an
361
- upgrade is a deliberate act, never a surprise. `openclaw` itself is a *peer*
374
+ nothing to keep in sync. ClawHub is the source OpenClaw trusts; npm installs show a source caution.
375
+ For existing installs, prefer the guarded `multi-clawd update` command rather than a raw source retry.
376
+
377
+ **Raw-install fallback boundary:** use npm only after positively identifying a
378
+ ClawHub availability failure (for example, package/version not found or registry
379
+ unreachable), or an older OpenClaw CLI explicitly rejecting the `clawhub:` spec
380
+ as unsupported. An old CLI version alone is not evidence of that incompatibility.
381
+ **Stop; do not retry through npm** for moderation, security, integrity, trust,
382
+ signature or policy failures, unknown errors, cancellation or interruption.
383
+ These stops take precedence even if availability diagnostics appeared first.
384
+ If the cause is unclear, stop and investigate; do not switch sources or restart
385
+ the gateway after a failed or interrupted install.
386
+
387
+ Only after the availability/unsupported-spec check above, an unversioned fresh
388
+ install may use `openclaw plugins install @drakon-systems/multi-clawd --pin`.
389
+ For a versioned install, retain the same selected exact target: if ClawHub was
390
+ `clawhub:@drakon-systems/multi-clawd@X.Y.Z`, the permitted npm retry is
391
+ `openclaw plugins install @drakon-systems/multi-clawd@X.Y.Z --pin`.
392
+ Replace `X.Y.Z` with that same exact version; never drop it, re-resolve latest,
393
+ or substitute a different version during fallback.
394
+ An unversioned ClawHub spec follows newer releases; add `@X.Y.Z` to keep a specific release.
395
+ `openclaw` itself is a *peer*
362
396
  dependency (the host provides it), so the install stays lean.
363
397
 
364
- **From ClawHub (alternative registry):**
365
-
366
- ```bash
367
- openclaw plugins install clawhub:@drakon-systems/multi-clawd
368
- ```
369
-
370
398
  **From source (contributors, or ahead of a release):**
371
399
 
372
400
  ```bash
@@ -387,7 +415,7 @@ openclaw plugins install (Get-Location).Path
387
415
  **Or let your agent install it.** Running an OpenClaw assistant or Claude
388
416
  Code on the target machine already? Paste it this and go make coffee:
389
417
 
390
- > Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v2.4.0/SETUP-AGENT.md
418
+ > Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v2.5.1/SETUP-AGENT.md
391
419
  > and follow it to set up multi-clawd on this machine. I own a second
392
420
  > Claude account — ask me when you need me to log in.
393
421
 
@@ -412,8 +440,9 @@ host-provided peer.
412
440
  npx @drakon-systems/multi-clawd update
413
441
  ```
414
442
 
415
- One command: checks the registry, installs the new version with the right
416
- flags, offers the gateway restart, and finishes with a doctor health check.
443
+ One command: checks the registry, installs the chosen version from ClawHub (with npm
444
+ fallback only when ClawHub is unavailable), offers the gateway restart, and finishes with a doctor health check.
445
+ Security, integrity, and policy failures stop the update without trying npm.
417
446
  (`npm i -g @drakon-systems/multi-clawd` once, and it's just `multi-clawd
418
447
  update` — with `multi-clawd setup`, `multi-clawd explain` (your setup in
419
448
  plain English), and `multi-clawd doctor` alongside.)
@@ -556,7 +585,9 @@ error.
556
585
  "pool": {
557
586
  "id": "clawd",
558
587
  "accounts": ["claw1", "claw2"], // preference order; first = home for ordered strategy
559
- "strategy": "ordered", // optional: "reset-aware" drains healthy weekly quota that resets soonest
588
+ "strategy": "ordered", // optional: "reset-aware" drains healthy weekly quota that resets soonest;
589
+ // "spread" starts each new session on the least-served healthy account
590
+ // "spreadWindowMs": 3600000, // optional, spread only: how far back served launches count (default 1h)
560
591
  "utilizationThreshold": 0.85, // default hand-over; minute/hour and overage windows use this
561
592
  "weeklyUtilizationThreshold": 0.97, // optional: day/week/month quota windows at 97%
562
593
  "minDwellMs": 600000, // v0.3: anti-flap dwell before returning home
@@ -602,7 +633,14 @@ a gateway reload. A turn already admitted or running continues on its selected
602
633
  account; the command does not interrupt it.
603
634
 
604
635
  `pool.strategy` defaults to `"ordered"`, which retains the configured account
605
- order. Set it to `"reset-aware"` to use healthy weekly quota before its
636
+ order. In every strategy, when no `ok` or `no_data` account is left but some
637
+ are near their limit, the launch goes to the near-limit account with the most
638
+ headroom: the lowest fresh utilization, taking the newer of the stream and
639
+ usage-poll records for each window. Equal figures keep pool order, an account
640
+ with a figure outranks one without, and two without keep pool order.
641
+ Exhausted and credential-failed accounts are never chosen. `explain` says
642
+ "all near limit, picked X at N%" and the direct Anthropic route orders its
643
+ near-limit profiles the same way. Set it to `"reset-aware"` to use healthy weekly quota before its
606
644
  `seven_day` window resets: among `ok` accounts with live seven-day reset data,
607
645
  the soonest reset wins. Accounts without that data follow in pool order;
608
646
  near-limit or exhausted accounts never outrank a healthy one. Equal reset
@@ -610,6 +648,25 @@ times keep pool order. The existing `minDwellMs` delay still holds an account
610
648
  that remains usable. Both CLI launches and the direct Anthropic route use the
611
649
  same preference. Other values are rejected by the plugin config schema.
612
650
 
651
+ Set `pool.strategy` to `"spread"` when several agent lanes start sessions at
652
+ once and you want them on different accounts instead of queueing behind one
653
+ account's concurrency ceiling. Each **new** session starts on the healthy
654
+ (`ok` or `no_data`) account that served the fewest launches in the last
655
+ `spreadWindowMs` (default one hour; clamped between one minute and seven
656
+ days), counted from the served-launch journal `explain --history` reads. On a
657
+ tie the account served least recently wins, then pool order, so equally idle
658
+ accounts are used in turn. Near-limit, exhausted and credential-failed
659
+ accounts are never chosen; when no healthy account is left the pool answers
660
+ exactly as `ordered` does. Spread keeps every session on the account that
661
+ first served it: it switches `sessionAffinity` on for the pool (an explicit
662
+ `sessionAffinity: false` is overridden with a warning, because without it a
663
+ conversation would move on every turn). A pinned session moves only when its
664
+ account stops being healthy, exactly as under `sessionAffinity`. Launches
665
+ without a session id take the spread pick. There is no home account, no
666
+ dwell and no rotation alert under spread; `explain`, `usage` and `doctor`
667
+ show the strategy, the next placement and the served counts behind it. The
668
+ direct Anthropic route orders its profiles by the same counts.
669
+
613
670
  `pool.sessionAffinity: true` remembers the account that successfully served each
614
671
  Claude CLI session (`--session-id` or `--resume`) and prefers it on later turns
615
672
  while its health is `ok` or `no_data`. A near-limit or otherwise unhealthy
@@ -637,6 +694,7 @@ How it decides, per launch (all data from each account's live
637
694
  | `allowed_warning` and **no** number, on a short window | skipped (v1.7.2) — see below |
638
695
  | `allowed_warning` and no number, on the weekly window | still used — weekly warns from ~30% and would flap |
639
696
  | no data / stale data | used — never rotate on missing evidence |
697
+ | every usable account near its limit | the near-limit account with the most headroom (lowest fresh utilization) serves; equal figures and accounts without a number keep pool order |
640
698
  | whole pool exhausted | home account anyway → real limit error → your chain drops provider |
641
699
 
642
700
  **Why the short-window rule exists (v1.7.2).** (v1.10's live usage poll now
@@ -892,15 +950,16 @@ event, and no plugin API can force a rebuild. See `DESIGN.md`.
892
950
  after the assessment or 24 hours after that turn, whichever comes first. A
893
951
  CLI login failure recorded on a real turn removes it at once.
894
952
 
895
- - **Shim window persistence is sequential-safe, not concurrent-writer-safe.**
896
- `persistState()` in `src/shim.ts` does a read-merge-write on every save so a
897
- turn that only reports one window type (say `five_hour`) doesn't clobber
898
- the last-seen `seven_day` data — but the read, merge, and rename aren't
899
- atomic together. Two truly concurrent shim processes for the *same*
900
- account can still race each other and drop an event on last-rename-wins.
901
- In practice this needs two in-flight turns on one account at once, which
902
- is rare, but it's a real gap. A per-account lock/retry protocol is tracked
903
- as a v0.3.x follow-up.
953
+ - **Shim state and retry coordination is best effort under lock contention.**
954
+ Concurrent shims for one account use a short advisory lock around each
955
+ state merge and rename. A second short lock covers recording a refusal and
956
+ reading sibling health for the in-turn retry. Later decisions can therefore
957
+ see rejections already recorded by another shim. If a lock stays busy beyond
958
+ 150 ms or filesystem locking fails, the shim proceeds without it so the
959
+ turn still runs; concurrent writes in that fallback can lose an observation.
960
+ A replacement between the lock's final ownership check and removal can
961
+ still remove another live holder's lock because those steps are not atomic.
962
+ The guard does not pace requests sent to a newly chosen sibling.
904
963
 
905
964
  ## Security
906
965
 
@@ -915,7 +974,7 @@ callsite (the ones static scanners flag) and why each exists.
915
974
  | Network | The gateway plugin polls `https://api.anthropic.com/api/oauth/usage` for eligible pooled accounts on an interval (default 120000 ms / 2 minutes); `pool.usagePoll.enabled: false` disables that poll. `multi-clawd usage` requests the same endpoint on demand. `doctor` runs `npm view` when its registry cache is stale; install and update commands contact the package registry. Claude Code subprocesses make their own provider requests. No plugin telemetry or analytics endpoint. |
916
975
  | Credentials | Reads declared setup-token files or gateway-resolved `oauthTokenRef` values for Claude child launches; reads each eligible account's `.credentials.json` access token for the usage poll; reads Claude login credentials from files or the macOS Keychain via the `security` CLI for login checks and runtime auth; reads account identity for file-backed logins from `.claude.json` (`~/.claude.json` for a native login, or the account config dir). Setup tokens are passed to Claude through the child environment. |
917
976
  | Files read | `~/.openclaw/openclaw.json`, declared account config dirs and token files, Claude `.credentials.json` and `.claude.json` files as described above, and local state files. |
918
- | Files written | Under `~/.openclaw/state/multi-clawd/` by default: `<id>.json`, `<id>.usage.json`, `pool-<id>.sticky.json`, `pool-<id>.sessions.json` (when affinity is enabled), `direct-order.sticky.json`, `known-models.json`, `alerts-spool.jsonl`, `watchdog.json`, `registry-check.json`, `auth-profiles-<agent>.json`, `runtime/<hash>/` (copied runtime files), and `watchdog-launcher.mjs`. The watchdog state file can be redirected with `MULTI_CLAWD_WATCHDOG_STATE`. Setup writes `~/.openclaw/openclaw.json` and a dated backup. On session handover, transcript `.jsonl` files are copied between account config dirs. The optional watchdog installs `~/.config/systemd/user/multi-clawd-eviction-watchdog.service` and `.timer` on Linux, or `~/Library/LaunchAgents/com.multi-clawd.eviction-watchdog.plist` on macOS; the unit points at the state-dir launcher. |
977
+ | Files written | Under `~/.openclaw/state/multi-clawd/` by default: `<id>.json`, `<id>.usage.json`, `history.jsonl` (bounded account activity), `pool-<id>.sticky.json`, `pool-<id>.sessions.json` (when affinity is enabled), `direct-order.sticky.json`, `known-models.json`, `alerts-spool.jsonl`, `watchdog.json`, `registry-check.json`, `auth-profiles-<agent>.json`, `runtime/<hash>/` (copied runtime files), and `watchdog-launcher.mjs`. The watchdog state file can be redirected with `MULTI_CLAWD_WATCHDOG_STATE`. Setup writes `~/.openclaw/openclaw.json` and a dated backup. On session handover, transcript `.jsonl` files are copied between account config dirs. The optional watchdog installs `~/.config/systemd/user/multi-clawd-eviction-watchdog.service` and `.timer` on Linux, or `~/Library/LaunchAgents/com.multi-clawd.eviction-watchdog.plist` on macOS; the unit points at the state-dir launcher. |
919
978
  | Processes | Launches Claude Code and the node + shim transport; CLI, setup and doctor run npm, OpenClaw, Claude and OS scheduler commands; the optional watchdog checks OpenClaw logs and may restart the gateway; Hermes integration runs its Python bridge. See [SECURITY.md](SECURITY.md) for the process-call table. |
920
979
  | Consent | The wizard asks before every write, merges non-destructively, and never overwrites an existing account entry. `--dry-run` previews everything. |
921
980
 
package/SECURITY.md CHANGED
@@ -37,6 +37,9 @@ credit you (unless you'd rather we didn't), and note it in the CHANGELOG.
37
37
  - **Per-account health and usage files are written `0600`,** using temporary
38
38
  files and rename, and contain rate-limit data rather than credentials. Other
39
39
  state files include model, alert, registry and watchdog data.
40
+ - **`history.jsonl` is a bounded `0600` account activity file** in the same
41
+ state directory. It contains timestamps, account/model IDs and short reason
42
+ codes, with no prompt, transcript, token, cost, credential, or environment values.
40
43
 
41
44
  ## Permissions
42
45
 
@@ -1,33 +1,17 @@
1
- import { readFileSync, writeFileSync, mkdirSync, renameSync } from "node:fs";
2
1
  import { homedir } from "node:os";
3
- import { dirname, join } from "node:path";
4
- import { parseStoredState, mergeHealthStates, clearCredentialFailure, } from "./shim-core.js";
2
+ import { join } from "node:path";
3
+ import { persistHealthState, readHealthStateFile } from "./health-journal.js";
4
+ import { clearCredentialFailure, } from "./shim-core.js";
5
5
  export function healthStateFile(accountId) {
6
6
  return join(homedir(), ".openclaw", "state", "multi-clawd", `${accountId}.json`);
7
7
  }
8
8
  export function usageStateFile(accountId) {
9
9
  return join(homedir(), ".openclaw", "state", "multi-clawd", `${accountId}.usage.json`);
10
10
  }
11
- export function clearAccountCredentialFailure(accountId) {
12
- const file = healthStateFile(accountId);
13
- let state;
14
- try {
15
- state = parseStoredState(readFileSync(file, "utf8")) ?? { accountId, windows: {} };
16
- }
17
- catch {
11
+ export function clearAccountCredentialFailure(accountId, file = healthStateFile(accountId)) {
12
+ const state = readHealthStateFile(file);
13
+ if (state?.credential?.status !== "failed")
18
14
  return false;
19
- }
20
- if (state.credential?.status !== "failed")
21
- return false;
22
- const cleared = mergeHealthStates(state, clearCredentialFailure(state, Date.now()), Date.now());
23
- try {
24
- mkdirSync(dirname(file), { recursive: true });
25
- const tmp = `${file}.tmp-${process.pid}`;
26
- writeFileSync(tmp, JSON.stringify(cleared, null, 2), { mode: 0o600 });
27
- renameSync(tmp, file);
28
- return true;
29
- }
30
- catch {
31
- return false;
32
- }
15
+ persistHealthState(file, clearCredentialFailure(state, Date.now()));
16
+ return readHealthStateFile(file)?.credential?.status === "ok";
33
17
  }
@@ -1,4 +1,5 @@
1
1
  import { classifyAccountHealth, sevenDayReset } from "./health.js";
2
+ import { spreadStatsFor, spreadStatsFromHistory, spreadWindowMs } from "./spread.js";
2
3
  import { DIRECT_PROVIDER, collectDirectMembers, } from "./direct-route.js";
3
4
  import { parseUnusableProfiles, planAgentOrder, readDirectRouteSnapshot, safeCliError, } from "./direct-sync.js";
4
5
  export function describeDirectSource(member) {
@@ -21,16 +22,22 @@ export async function gatherDirectStatus(params) {
21
22
  if (members.length === 0 && problems.length === 0)
22
23
  return undefined;
23
24
  const errors = [];
25
+ const spread = params.strategy === "spread"
26
+ ? spreadStatsFromHistory(params.readHistory?.() ?? [], params.nowMs, spreadWindowMs(params.spreadWindowMs))
27
+ : undefined;
24
28
  const verdicts = members.map((m) => {
25
29
  const state = params.readHealth(m.accountId);
30
+ const health = classifyAccountHealth(state, params.healthOptions, params.nowMs);
26
31
  return {
27
32
  accountId: m.accountId,
28
33
  profileId: m.profileId,
29
34
  disabled: m.disabled,
30
- verdict: classifyAccountHealth(state, params.healthOptions, params.nowMs).verdict,
35
+ verdict: health.verdict,
36
+ utilization: health.utilization,
31
37
  sevenDayResetsAt: params.strategy === "reset-aware"
32
38
  ? sevenDayReset(state, params.healthOptions, params.nowMs)
33
39
  : undefined,
40
+ ...(spread ? spreadStatsFor(spread, m.accountId) : {}),
34
41
  };
35
42
  });
36
43
  const read = members.length > 0 ? await readDirectRouteSnapshot(params.runner, params.agentId) : {};
@@ -1,5 +1,7 @@
1
1
  import { isSecretRefShape } from "./token-resolution.js";
2
+ import { headroomBefore } from "./health.js";
2
3
  import { decideStickySelection } from "./sticky.js";
4
+ import { spreadBefore } from "./spread.js";
3
5
  export const DIRECT_PROVIDER = "anthropic";
4
6
  export const DIRECT_PROFILE_PREFIX = `${DIRECT_PROVIDER}:`;
5
7
  export const DIRECT_SETUP_TOKEN_GUIDANCE = "a native or config-dir Claude login is a rotating single-use OAuth grant — copying it into " +
@@ -134,7 +136,7 @@ export function planDirectOrder(params) {
134
136
  if (members.length === 0)
135
137
  throw new Error("pool.accounts[].disabled: every direct member is disabled");
136
138
  const decision = decideStickySelection({
137
- verdicts: members.map((m) => ({ id: m.accountId, verdict: m.verdict, sevenDayResetsAt: m.sevenDayResetsAt })),
139
+ verdicts: members.map((m) => ({ id: m.accountId, verdict: m.verdict, sevenDayResetsAt: m.sevenDayResetsAt, utilization: m.utilization, servedInWindow: m.servedInWindow, lastServedAt: m.lastServedAt })),
138
140
  sticky: params.sticky,
139
141
  nowMs: params.nowMs,
140
142
  minDwellMs: params.minDwellMs,
@@ -147,6 +149,12 @@ export function planDirectOrder(params) {
147
149
  .sort((a, b) => VERDICT_RANK[a.m.verdict] - VERDICT_RANK[b.m.verdict] ||
148
150
  (params.strategy === "reset-aware" && a.m.verdict === "ok"
149
151
  ? (a.m.sevenDayResetsAt ?? Infinity) - (b.m.sevenDayResetsAt ?? Infinity)
152
+ : 0) ||
153
+ (params.strategy === "spread" && VERDICT_RANK[a.m.verdict] === 0
154
+ ? (spreadBefore(a.m, b.m) ? -1 : spreadBefore(b.m, a.m) ? 1 : 0)
155
+ : 0) ||
156
+ (a.m.verdict === "near_limit" && b.m.verdict === "near_limit"
157
+ ? (headroomBefore(a.m, b.m) ? -1 : headroomBefore(b.m, a.m) ? 1 : 0)
150
158
  : 0) || a.index - b.index)
151
159
  .map(({ m }) => m.profileId);
152
160
  const managed = new Set(params.members.map((m) => m.profileId));
@@ -1,4 +1,5 @@
1
1
  import { classifyAccountHealth, sevenDayReset } from "./health.js";
2
+ import { spreadStatsFor, spreadStatsFromHistory, spreadWindowMs } from "./spread.js";
2
3
  import { DIRECT_PROVIDER, planDirectOrder } from "./direct-route.js";
3
4
  import { parseClaudeSetupToken } from "./hermes-core.js";
4
5
  export function safeCliError(result) {
@@ -247,16 +248,22 @@ export function createDirectOrderController(deps) {
247
248
  return { managedOrder: [], agents: [] };
248
249
  running = true;
249
250
  try {
251
+ const spread = deps.strategy === "spread"
252
+ ? spreadStatsFromHistory(deps.readHistory?.() ?? [], nowMs, spreadWindowMs(deps.spreadWindowMs))
253
+ : undefined;
250
254
  const verdicts = deps.members.map((m) => {
251
255
  const state = deps.readHealth(m.accountId);
256
+ const health = classifyAccountHealth(state, deps.healthOptions, nowMs);
252
257
  return {
253
258
  accountId: m.accountId,
254
259
  profileId: m.profileId,
255
260
  disabled: m.disabled,
256
- verdict: classifyAccountHealth(state, deps.healthOptions, nowMs).verdict,
261
+ verdict: health.verdict,
262
+ utilization: health.utilization,
257
263
  sevenDayResetsAt: deps.strategy === "reset-aware"
258
264
  ? sevenDayReset(state, deps.healthOptions, nowMs)
259
265
  : undefined,
266
+ ...(spread ? spreadStatsFor(spread, m.accountId) : {}),
260
267
  };
261
268
  });
262
269
  const sticky = deps.readSticky();
@@ -1,5 +1,6 @@
1
1
  import { DEFAULT_UTILIZATION_THRESHOLD, choosePoolAccount, poolStrategy } from "./health.js";
2
2
  import { decideStickySelection } from "./sticky.js";
3
+ import { describeSpreadPick, describeWindow, spreadStatsFromHistory, spreadWindowMs, withSpreadStats } from "./spread.js";
3
4
  const WINDOW_LABELS = {
4
5
  five_hour: "5-hour",
5
6
  seven_day: "weekly",
@@ -16,6 +17,9 @@ export function relativeUntil(ms, nowMs) {
16
17
  return `~${hours}h`;
17
18
  return `~${Math.round(hours / 24)}d`;
18
19
  }
20
+ export function relativeSince(ms, nowMs) {
21
+ return relativeUntil(nowMs + Math.max(0, nowMs - ms), nowMs);
22
+ }
19
23
  export function renderUsageLine(usage, nowMs) {
20
24
  return usage
21
25
  .map((u) => {
@@ -93,6 +97,12 @@ export function renderExplanation(model) {
93
97
  if (poolStrategy(model.pool.strategy) === "reset-aware") {
94
98
  lines.push(" Healthy accounts with a live seven-day reset are preferred by soonest reset.");
95
99
  }
100
+ else if (poolStrategy(model.pool.strategy) === "spread") {
101
+ lines.push(` Each new session starts on the healthy account with the fewest served launches in the last ${describeWindow(spreadWindowMs(model.pool.spreadWindowMs))};`);
102
+ lines.push(" a session then stays on its account while that account is healthy (session affinity is on).");
103
+ if (model.pool.sessionAffinity === false)
104
+ lines.push(" sessionAffinity: false is ignored under spread — without it a conversation would move every turn.");
105
+ }
96
106
  else {
97
107
  lines.push(" Every Claude launch runs on the first account below its threshold;");
98
108
  lines.push(" the home account reclaims work");
@@ -138,15 +148,19 @@ export function renderExplanation(model) {
138
148
  }
139
149
  if (model.pool) {
140
150
  const strategy = poolStrategy(model.pool.strategy);
141
- const verdicts = model.pool.accounts.filter((id) => !model.accounts.find((a) => a.id === id)?.disabled).map((id) => {
151
+ const windowMs = spreadWindowMs(model.pool.spreadWindowMs);
152
+ const spread = strategy === "spread" ? spreadStatsFromHistory(model.history ?? [], nowMs, windowMs) : undefined;
153
+ const rawVerdicts = model.pool.accounts.filter((id) => !model.accounts.find((a) => a.id === id)?.disabled).map((id) => {
142
154
  const h = model.health.find((row) => row.id === id);
143
155
  return {
144
156
  id,
145
157
  verdict: h?.verdict ?? "no_data",
158
+ utilization: h?.utilization,
146
159
  sevenDayResetsAt: (h?.usage?.find((u) => u.window === "usage:seven_day" && u.resetsAt !== undefined && u.resetsAt > nowMs)
147
160
  ?? h?.usage?.find((u) => u.window === "seven_day" && u.resetsAt !== undefined && u.resetsAt > nowMs))?.resetsAt,
148
161
  };
149
162
  });
163
+ const verdicts = spread ? withSpreadStats(rawVerdicts, spread) : rawVerdicts;
150
164
  if (verdicts.length) {
151
165
  const preferred = choosePoolAccount(verdicts, strategy);
152
166
  const decision = decideStickySelection({ verdicts, sticky: model.stickyEntry, nowMs, minDwellMs: model.pool.minDwellMs, strategy });
@@ -154,8 +168,12 @@ export function renderExplanation(model) {
154
168
  let reason;
155
169
  if (preferred && decision.account !== preferred)
156
170
  reason = "sticky dwell keeps this usable account";
171
+ else if (picked?.verdict === "near_limit")
172
+ reason = describeNearLimitPick(verdicts, picked);
157
173
  else if (strategy === "ordered")
158
- reason = picked?.verdict === "near_limit" ? "first near-limit member in pool order" : "first healthy member in pool order";
174
+ reason = "first healthy member in pool order";
175
+ else if (strategy === "spread")
176
+ reason = picked && (picked.verdict === "ok" || picked.verdict === "no_data") ? describeSpreadPick(picked, windowMs) : "no healthy member; using the existing fallback";
159
177
  else if (picked?.verdict === "ok" && picked.sevenDayResetsAt !== undefined)
160
178
  reason = `seven-day window resets soonest (in ${relativeUntil(picked.sevenDayResetsAt, nowMs)})`;
161
179
  else if (picked?.verdict === "ok")
@@ -164,20 +182,43 @@ export function renderExplanation(model) {
164
182
  reason = "first member without live health data";
165
183
  else
166
184
  reason = "no healthy member; using the existing fallback";
167
- lines.push(` strategy: ${strategy} — picked ${decision.account}: ${reason}`);
185
+ lines.push(` strategy: ${strategy} — ${strategy === "spread" ? "next new session on" : "picked"} ${decision.account}: ${reason}`);
186
+ if (spread) {
187
+ for (const v of verdicts) {
188
+ lines.push(` ${v.id}: ${v.servedInWindow ?? 0} served in the last ${describeWindow(windowMs)}${v.lastServedAt !== undefined ? `, last ${relativeSince(v.lastServedAt, nowMs)} ago` : ""}`);
189
+ }
190
+ }
168
191
  }
169
192
  else {
170
193
  lines.push(` strategy: ${strategy} — pool.accounts[].disabled: every pool member is disabled`);
171
194
  }
172
- lines.push(model.stickyAccount
173
- ? strategy === "reset-aware"
174
- ? ` pool is on ${model.stickyAccount} (dwell applies while it remains usable)`
175
- : ` pool is rotated onto ${model.stickyAccount} (returns home when the home window resets)`
176
- : ` pool is on its home account (no rotation active)`);
195
+ if (strategy === "spread")
196
+ lines.push(" pool has no home account under spread; existing sessions keep the account that served them");
197
+ else
198
+ lines.push(model.stickyAccount
199
+ ? strategy === "reset-aware"
200
+ ? ` pool is on ${model.stickyAccount} (dwell applies while it remains usable)`
201
+ : ` pool is rotated onto ${model.stickyAccount} (returns home when the home window resets)`
202
+ : ` pool is on its home account (no rotation active)`);
177
203
  }
178
204
  }
179
205
  return lines.join("\n");
180
206
  }
207
+ export function describeNearLimitPick(verdicts, picked) {
208
+ const nearLimit = verdicts.filter((v) => v.verdict === "near_limit");
209
+ const scope = nearLimit.length === verdicts.length ? "all near limit" : "no healthy member, the rest near limit";
210
+ const others = nearLimit.filter((v) => v.id !== picked.id);
211
+ if (picked.utilization === undefined) {
212
+ return `${scope}, picked ${picked.id} (no utilization number; pool order)`;
213
+ }
214
+ const pct = `${Math.round(picked.utilization * 100)}%`;
215
+ if (others.length === 0)
216
+ return `${scope}, picked ${picked.id} at ${pct}`;
217
+ const shown = others
218
+ .map((v) => `${v.id} ${v.utilization === undefined ? "no number" : `${Math.round(v.utilization * 100)}%`}`)
219
+ .join(", ");
220
+ return `${scope}, picked ${picked.id} at ${pct} — most headroom (${shown})`;
221
+ }
181
222
  export function renderDirectSection(direct, nowMs) {
182
223
  const lines = [];
183
224
  lines.push("DIRECT ROUTE anthropic/* (gateway → Anthropic API with each account's setup-token)");
@@ -0,0 +1,123 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { mkdirSync, readFileSync, readdirSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
3
+ import { basename, dirname, join } from "node:path";
4
+ import { acquireShimLock } from "./shim-lock.js";
5
+ import { mergeHealthStates, parseStoredState } from "./shim-core.js";
6
+ function pendingFiles(file) {
7
+ const prefix = `${basename(file)}.pending-`;
8
+ try {
9
+ return readdirSync(dirname(file)).filter((name) => name.startsWith(prefix) && /^\d+-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name.slice(prefix.length))).map((name) => join(dirname(file), name));
10
+ }
11
+ catch {
12
+ return [];
13
+ }
14
+ }
15
+ function snapshot(file, onError, strict = false) {
16
+ let raw;
17
+ try {
18
+ raw = readFileSync(file, "utf8");
19
+ }
20
+ catch (err) {
21
+ if (err.code !== "ENOENT") {
22
+ onError?.(undefined);
23
+ if (strict)
24
+ throw err;
25
+ }
26
+ return undefined;
27
+ }
28
+ const state = parseStoredState(raw);
29
+ if (!state) {
30
+ onError?.(raw);
31
+ if (strict)
32
+ throw new Error("not valid health-state JSON");
33
+ }
34
+ return state;
35
+ }
36
+ export function readHealthStateFile(file, opts = {}) {
37
+ for (let attempt = 0; attempt < 3; attempt++) {
38
+ let state = snapshot(file, opts.onSnapshotError, opts.strictSnapshot);
39
+ const files = pendingFiles(file);
40
+ opts.afterList?.(files);
41
+ let missing = false;
42
+ for (const pending of files) {
43
+ let raw;
44
+ try {
45
+ raw = readFileSync(pending, "utf8");
46
+ }
47
+ catch (err) {
48
+ if (err.code === "ENOENT")
49
+ missing = true;
50
+ continue;
51
+ }
52
+ const delta = parseStoredState(raw);
53
+ if (delta)
54
+ state = state ? mergeHealthStates(state, delta) : delta;
55
+ }
56
+ if (!missing || attempt === 2)
57
+ return state;
58
+ }
59
+ return undefined;
60
+ }
61
+ function publish(file, state) {
62
+ mkdirSync(dirname(file), { recursive: true });
63
+ const tmp = `${file}.tmp-${process.pid}-${randomUUID()}`;
64
+ try {
65
+ writeFileSync(tmp, JSON.stringify(state, null, 2), { mode: 0o600 });
66
+ renameSync(tmp, file);
67
+ }
68
+ catch (err) {
69
+ try {
70
+ unlinkSync(tmp);
71
+ }
72
+ catch { }
73
+ throw err;
74
+ }
75
+ }
76
+ export function persistHealthState(file, state, opts = {}) {
77
+ const release = acquireShimLock(file);
78
+ try {
79
+ if (!release) {
80
+ publish(`${file}.pending-${process.pid}-${randomUUID()}`, state);
81
+ return state;
82
+ }
83
+ let merged = snapshot(file, opts.onSnapshotError);
84
+ const files = pendingFiles(file);
85
+ opts.afterList?.(files);
86
+ const folded = [];
87
+ for (const pending of files) {
88
+ let raw;
89
+ try {
90
+ raw = readFileSync(pending, "utf8");
91
+ }
92
+ catch {
93
+ continue;
94
+ }
95
+ const delta = parseStoredState(raw);
96
+ if (!delta) {
97
+ try {
98
+ renameSync(pending, `${pending}.corrupt-${Date.now()}`);
99
+ }
100
+ catch { }
101
+ continue;
102
+ }
103
+ merged = merged ? mergeHealthStates(merged, delta) : delta;
104
+ folded.push(pending);
105
+ }
106
+ merged = merged ? mergeHealthStates(merged, state, Date.now()) : mergeHealthStates(state, state, Date.now());
107
+ publish(file, merged);
108
+ for (const pending of folded) {
109
+ try {
110
+ unlinkSync(pending);
111
+ }
112
+ catch { }
113
+ }
114
+ return merged;
115
+ }
116
+ catch (err) {
117
+ process.stderr.write(`[multi-clawd shim] state write failed: ${String(err)}\n`);
118
+ return state;
119
+ }
120
+ finally {
121
+ release?.();
122
+ }
123
+ }