@drakon-systems/multi-clawd 2.1.3 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -3
- package/dist/direct-report.js +13 -6
- package/dist/direct-route.js +6 -2
- package/dist/direct-sync.js +15 -7
- package/dist/explain-core.js +50 -5
- package/dist/health.js +31 -5
- package/dist/index.js +11 -2
- package/dist/sticky.js +11 -1
- package/openclaw.plugin.json +13 -2
- package/package.json +1 -1
- package/scripts/cli.mjs +28 -6
- package/scripts/doctor.mjs +15 -6
package/README.md
CHANGED
|
@@ -386,7 +386,7 @@ openclaw plugins install (Get-Location).Path
|
|
|
386
386
|
**Or let your agent install it.** Running an OpenClaw assistant or Claude
|
|
387
387
|
Code on the target machine already? Paste it this and go make coffee:
|
|
388
388
|
|
|
389
|
-
> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v2.
|
|
389
|
+
> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v2.3.0/SETUP-AGENT.md
|
|
390
390
|
> and follow it to set up multi-clawd on this machine. I own a second
|
|
391
391
|
> Claude account — ask me when you need me to log in.
|
|
392
392
|
|
|
@@ -553,8 +553,10 @@ error.
|
|
|
553
553
|
],
|
|
554
554
|
"pool": {
|
|
555
555
|
"id": "clawd",
|
|
556
|
-
"accounts": ["claw1", "claw2"], // preference order; first = home
|
|
557
|
-
"
|
|
556
|
+
"accounts": ["claw1", "claw2"], // preference order; first = home for ordered strategy
|
|
557
|
+
"strategy": "ordered", // optional: "reset-aware" drains healthy weekly quota that resets soonest
|
|
558
|
+
"utilizationThreshold": 0.85, // default hand-over; minute/hour and overage windows use this
|
|
559
|
+
"weeklyUtilizationThreshold": 0.97, // optional: day/week/month quota windows at 97%
|
|
558
560
|
"minDwellMs": 600000, // v0.3: anti-flap dwell before returning home
|
|
559
561
|
"degrade": { // v0.3.5: last step before provider drop
|
|
560
562
|
"ladder": ["claude-opus-4-8"], // whole pool exhausted → same account, lower tier
|
|
@@ -576,6 +578,15 @@ error.
|
|
|
576
578
|
} }
|
|
577
579
|
```
|
|
578
580
|
|
|
581
|
+
`pool.strategy` defaults to `"ordered"`, which retains the configured account
|
|
582
|
+
order. Set it to `"reset-aware"` to use healthy weekly quota before its
|
|
583
|
+
`seven_day` window resets: among `ok` accounts with live seven-day reset data,
|
|
584
|
+
the soonest reset wins. Accounts without that data follow in pool order;
|
|
585
|
+
near-limit or exhausted accounts never outrank a healthy one. Equal reset
|
|
586
|
+
times keep pool order. The existing `minDwellMs` delay still holds an account
|
|
587
|
+
that remains usable. Both CLI launches and the direct Anthropic route use the
|
|
588
|
+
same preference. Other values are rejected by the plugin config schema.
|
|
589
|
+
|
|
579
590
|
How it decides, per launch (all data from each account's live
|
|
580
591
|
`rate_limit_event` stream, captured by the shim):
|
|
581
592
|
|
|
@@ -630,6 +641,7 @@ from the provider** on a timer — the same figures the Claude CLI's own
|
|
|
630
641
|
"id": "clawd",
|
|
631
642
|
"accounts": ["claw1", "claw2"],
|
|
632
643
|
"utilizationThreshold": 0.85, // hand over here (unchanged)
|
|
644
|
+
"weeklyUtilizationThreshold": 0.97, // optional: day/week/month quota windows at 97%; overage stays at 85%
|
|
633
645
|
"usagePoll": { // v1.10 — on by default
|
|
634
646
|
"intervalMs": 120000, // every 2 min (minimum 60000)
|
|
635
647
|
"warnThreshold": 0.95 // operator alert from here
|
|
@@ -830,6 +842,18 @@ event, and no plugin API can force a rebuild. See `DESIGN.md`.
|
|
|
830
842
|
a usable login now reports as available, a dead one still reports
|
|
831
843
|
"missing-auth", and the direct API transport keeps refusing CLI-login
|
|
832
844
|
accounts as before. Update with `multi-clawd update`.
|
|
845
|
+
If an account's stored login is deleted or becomes unreadable, the account
|
|
846
|
+
can stay available until the 30-second cache window that began at its last
|
|
847
|
+
assessment ends; the window runs from that assessment, not from the loss.
|
|
848
|
+
The first availability check after the window ends starts a background
|
|
849
|
+
re-check, and the account stays available only while that re-check is
|
|
850
|
+
pending; once it finishes it marks the account unavailable. Even if the
|
|
851
|
+
re-check stalls, a login backed by a valid access token is refused at its
|
|
852
|
+
access-token expiry minus 60 seconds, including at that exact moment and
|
|
853
|
+
even inside the first window. A login proven only by a recent successful
|
|
854
|
+
turn (its stored token can already be expired) is refused at two windows
|
|
855
|
+
after the assessment or 24 hours after that turn, whichever comes first. A
|
|
856
|
+
CLI login failure recorded on a real turn removes it at once.
|
|
833
857
|
|
|
834
858
|
- **Shim window persistence is sequential-safe, not concurrent-writer-safe.**
|
|
835
859
|
`persistState()` in `src/shim.ts` does a read-merge-write on every save so a
|
package/dist/direct-report.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { classifyAccountHealth } from "./health.js";
|
|
1
|
+
import { classifyAccountHealth, sevenDayReset } from "./health.js";
|
|
2
2
|
import { DIRECT_PROVIDER, collectDirectMembers, } from "./direct-route.js";
|
|
3
3
|
import { parseUnusableProfiles, planAgentOrder, readDirectRouteSnapshot, safeCliError, } from "./direct-sync.js";
|
|
4
4
|
export function describeDirectSource(member) {
|
|
@@ -19,11 +19,17 @@ export async function gatherDirectStatus(params) {
|
|
|
19
19
|
if (members.length === 0 && problems.length === 0)
|
|
20
20
|
return undefined;
|
|
21
21
|
const errors = [];
|
|
22
|
-
const verdicts = members.map((m) =>
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
22
|
+
const verdicts = members.map((m) => {
|
|
23
|
+
const state = params.readHealth(m.accountId);
|
|
24
|
+
return {
|
|
25
|
+
accountId: m.accountId,
|
|
26
|
+
profileId: m.profileId,
|
|
27
|
+
verdict: classifyAccountHealth(state, params.healthOptions, params.nowMs).verdict,
|
|
28
|
+
sevenDayResetsAt: params.strategy === "reset-aware"
|
|
29
|
+
? sevenDayReset(state, params.healthOptions, params.nowMs)
|
|
30
|
+
: undefined,
|
|
31
|
+
};
|
|
32
|
+
});
|
|
27
33
|
const read = members.length > 0 ? await readDirectRouteSnapshot(params.runner, params.agentId) : {};
|
|
28
34
|
if ("error" in read && read.error)
|
|
29
35
|
errors.push(read.error);
|
|
@@ -46,6 +52,7 @@ export async function gatherDirectStatus(params) {
|
|
|
46
52
|
sticky: params.sticky,
|
|
47
53
|
nowMs: params.nowMs,
|
|
48
54
|
minDwellMs: params.minDwellMs,
|
|
55
|
+
strategy: params.strategy,
|
|
49
56
|
}).order;
|
|
50
57
|
}
|
|
51
58
|
const byProfile = new Map(unusable.map((u) => [u.profileId, u]));
|
package/dist/direct-route.js
CHANGED
|
@@ -132,16 +132,20 @@ export function planDirectOrder(params) {
|
|
|
132
132
|
if (members.length === 0)
|
|
133
133
|
return undefined;
|
|
134
134
|
const decision = decideStickySelection({
|
|
135
|
-
verdicts: members.map((m) => ({ id: m.accountId, verdict: m.verdict })),
|
|
135
|
+
verdicts: members.map((m) => ({ id: m.accountId, verdict: m.verdict, sevenDayResetsAt: m.sevenDayResetsAt })),
|
|
136
136
|
sticky: params.sticky,
|
|
137
137
|
nowMs: params.nowMs,
|
|
138
138
|
minDwellMs: params.minDwellMs,
|
|
139
|
+
strategy: params.strategy,
|
|
139
140
|
});
|
|
140
141
|
const first = members.find((m) => m.accountId === decision.account) ?? members[0];
|
|
141
142
|
const rest = members
|
|
142
143
|
.filter((m) => m !== first)
|
|
143
144
|
.map((m, index) => ({ m, index }))
|
|
144
|
-
.sort((a, b) => VERDICT_RANK[a.m.verdict] - VERDICT_RANK[b.m.verdict] ||
|
|
145
|
+
.sort((a, b) => VERDICT_RANK[a.m.verdict] - VERDICT_RANK[b.m.verdict] ||
|
|
146
|
+
(params.strategy === "reset-aware" && a.m.verdict === "ok"
|
|
147
|
+
? (a.m.sevenDayResetsAt ?? Infinity) - (b.m.sevenDayResetsAt ?? Infinity)
|
|
148
|
+
: 0) || a.index - b.index)
|
|
145
149
|
.map(({ m }) => m.profileId);
|
|
146
150
|
const managed = new Set(members.map((m) => m.profileId));
|
|
147
151
|
const unmanaged = (params.currentOrder ?? []).filter((id, i, all) => typeof id === "string" && !managed.has(id) && all.indexOf(id) === i);
|
package/dist/direct-sync.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { classifyAccountHealth } from "./health.js";
|
|
1
|
+
import { classifyAccountHealth, sevenDayReset } from "./health.js";
|
|
2
2
|
import { DIRECT_PROVIDER, planDirectOrder } from "./direct-route.js";
|
|
3
3
|
import { parseClaudeSetupToken } from "./hermes-core.js";
|
|
4
4
|
export function safeCliError(result) {
|
|
@@ -224,6 +224,7 @@ export function planAgentOrder(params) {
|
|
|
224
224
|
sticky: params.sticky,
|
|
225
225
|
nowMs: params.nowMs,
|
|
226
226
|
minDwellMs: params.minDwellMs,
|
|
227
|
+
strategy: params.strategy,
|
|
227
228
|
});
|
|
228
229
|
if (!plan)
|
|
229
230
|
return { missing };
|
|
@@ -244,13 +245,19 @@ export function createDirectOrderController(deps) {
|
|
|
244
245
|
return { managedOrder: [], agents: [] };
|
|
245
246
|
running = true;
|
|
246
247
|
try {
|
|
247
|
-
const verdicts = deps.members.map((m) =>
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
248
|
+
const verdicts = deps.members.map((m) => {
|
|
249
|
+
const state = deps.readHealth(m.accountId);
|
|
250
|
+
return {
|
|
251
|
+
accountId: m.accountId,
|
|
252
|
+
profileId: m.profileId,
|
|
253
|
+
verdict: classifyAccountHealth(state, deps.healthOptions, nowMs).verdict,
|
|
254
|
+
sevenDayResetsAt: deps.strategy === "reset-aware"
|
|
255
|
+
? sevenDayReset(state, deps.healthOptions, nowMs)
|
|
256
|
+
: undefined,
|
|
257
|
+
};
|
|
258
|
+
});
|
|
252
259
|
const sticky = deps.readSticky();
|
|
253
|
-
const managed = planDirectOrder({ members: verdicts, sticky, nowMs, minDwellMs: deps.minDwellMs });
|
|
260
|
+
const managed = planDirectOrder({ members: verdicts, sticky, nowMs, minDwellMs: deps.minDwellMs, strategy: deps.strategy });
|
|
254
261
|
if (!managed)
|
|
255
262
|
return { managedOrder: [], agents: [] };
|
|
256
263
|
if (JSON.stringify(managed.sticky) !== JSON.stringify(sticky))
|
|
@@ -285,6 +292,7 @@ export function createDirectOrderController(deps) {
|
|
|
285
292
|
sticky,
|
|
286
293
|
nowMs,
|
|
287
294
|
minDwellMs: deps.minDwellMs,
|
|
295
|
+
strategy: deps.strategy,
|
|
288
296
|
});
|
|
289
297
|
for (const id of plan.missing) {
|
|
290
298
|
const k = `${agentId}:${id}`;
|
package/dist/explain-core.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { DEFAULT_UTILIZATION_THRESHOLD, choosePoolAccount, poolStrategy } from "./health.js";
|
|
2
|
+
import { decideStickySelection } from "./sticky.js";
|
|
1
3
|
const WINDOW_LABELS = {
|
|
2
4
|
five_hour: "5-hour",
|
|
3
5
|
seven_day: "weekly",
|
|
@@ -84,11 +86,20 @@ export function renderExplanation(model) {
|
|
|
84
86
|
}
|
|
85
87
|
lines.push("");
|
|
86
88
|
if (model.pool) {
|
|
87
|
-
const
|
|
89
|
+
const generalThreshold = model.pool.utilizationThreshold ?? DEFAULT_UTILIZATION_THRESHOLD;
|
|
90
|
+
const weeklyThreshold = model.pool.weeklyUtilizationThreshold ?? generalThreshold;
|
|
88
91
|
lines.push(`POOL ${model.pool.id} (${model.pool.accounts.join(" → ")})`);
|
|
89
|
-
lines.push(`
|
|
90
|
-
|
|
91
|
-
|
|
92
|
+
lines.push(` Strategy: ${poolStrategy(model.pool.strategy)}.`);
|
|
93
|
+
if (poolStrategy(model.pool.strategy) === "reset-aware") {
|
|
94
|
+
lines.push(" Healthy accounts with a live seven-day reset are preferred by soonest reset.");
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
lines.push(" Every Claude launch runs on the first account below its threshold;");
|
|
98
|
+
lines.push(" the home account reclaims work");
|
|
99
|
+
lines.push(" automatically once its window resets.");
|
|
100
|
+
}
|
|
101
|
+
lines.push(` Thresholds: general ${Math.round(generalThreshold * 100)}%, weekly ${Math.round(weeklyThreshold * 100)}%${model.pool.weeklyUtilizationThreshold === undefined ? " (follows general)" : ""}.`);
|
|
102
|
+
lines.push(` The weekly threshold applies to day/week/month quota windows; overage uses the general threshold.`);
|
|
92
103
|
const ladder = model.pool.degrade?.ladder ?? [];
|
|
93
104
|
if (ladder.length > 0) {
|
|
94
105
|
lines.push(` If the WHOLE pool is exhausted: step down to ${ladder.join(" → ")} first.`);
|
|
@@ -126,8 +137,42 @@ export function renderExplanation(model) {
|
|
|
126
137
|
lines.push(` usage: ${h.usage?.length ? renderUsageLine(h.usage, nowMs) : "no live telemetry"}`);
|
|
127
138
|
}
|
|
128
139
|
if (model.pool) {
|
|
140
|
+
const strategy = poolStrategy(model.pool.strategy);
|
|
141
|
+
const verdicts = model.pool.accounts.map((id) => {
|
|
142
|
+
const h = model.health.find((row) => row.id === id);
|
|
143
|
+
return {
|
|
144
|
+
id,
|
|
145
|
+
verdict: h?.verdict ?? "no_data",
|
|
146
|
+
sevenDayResetsAt: (h?.usage?.find((u) => u.window === "usage:seven_day" && u.resetsAt !== undefined && u.resetsAt > nowMs)
|
|
147
|
+
?? h?.usage?.find((u) => u.window === "seven_day" && u.resetsAt !== undefined && u.resetsAt > nowMs))?.resetsAt,
|
|
148
|
+
};
|
|
149
|
+
});
|
|
150
|
+
if (verdicts.length) {
|
|
151
|
+
const preferred = choosePoolAccount(verdicts, strategy);
|
|
152
|
+
const decision = decideStickySelection({ verdicts, sticky: model.stickyEntry, nowMs, minDwellMs: model.pool.minDwellMs, strategy });
|
|
153
|
+
const picked = verdicts.find((v) => v.id === decision.account);
|
|
154
|
+
let reason;
|
|
155
|
+
if (preferred && decision.account !== preferred)
|
|
156
|
+
reason = "sticky dwell keeps this usable account";
|
|
157
|
+
else if (strategy === "ordered")
|
|
158
|
+
reason = picked?.verdict === "near_limit" ? "first near-limit member in pool order" : "first healthy member in pool order";
|
|
159
|
+
else if (picked?.verdict === "ok" && picked.sevenDayResetsAt !== undefined)
|
|
160
|
+
reason = `seven-day window resets soonest (in ${relativeUntil(picked.sevenDayResetsAt, nowMs)})`;
|
|
161
|
+
else if (picked?.verdict === "ok")
|
|
162
|
+
reason = "first healthy member without live seven-day reset data";
|
|
163
|
+
else if (picked?.verdict === "no_data")
|
|
164
|
+
reason = "first member without live health data";
|
|
165
|
+
else
|
|
166
|
+
reason = "no healthy member; using the existing fallback";
|
|
167
|
+
lines.push(` strategy: ${strategy} — picked ${decision.account}: ${reason}`);
|
|
168
|
+
}
|
|
169
|
+
else {
|
|
170
|
+
lines.push(` strategy: ${strategy}`);
|
|
171
|
+
}
|
|
129
172
|
lines.push(model.stickyAccount
|
|
130
|
-
?
|
|
173
|
+
? strategy === "reset-aware"
|
|
174
|
+
? ` pool is on ${model.stickyAccount} (dwell applies while it remains usable)`
|
|
175
|
+
: ` pool is rotated onto ${model.stickyAccount} (returns home when the home window resets)`
|
|
131
176
|
: ` pool is on its home account (no rotation active)`);
|
|
132
177
|
}
|
|
133
178
|
}
|
package/dist/health.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { modelWindowApplies, modelWindowKey } from "./shim-core.js";
|
|
2
|
-
const DEFAULT_UTILIZATION_THRESHOLD = 0.85;
|
|
2
|
+
export const DEFAULT_UTILIZATION_THRESHOLD = 0.85;
|
|
3
3
|
const DEFAULT_STALE_AFTER_MS = 6 * 60 * 60 * 1000;
|
|
4
4
|
export const MODEL_REJECTED_TTL_MS = 60 * 60 * 1000;
|
|
5
5
|
export const CREDENTIAL_FAILED_TTL_MS = 15 * 60 * 1000;
|
|
@@ -19,6 +19,9 @@ export function isPeriodWindow(window) {
|
|
|
19
19
|
export function isOverageWindow(window) {
|
|
20
20
|
return /overage/i.test(window);
|
|
21
21
|
}
|
|
22
|
+
export function isLongQuotaWindow(window) {
|
|
23
|
+
return /(^|_)(days?|weeks?|months?)(_|$)/.test(window) && !isOverageWindow(window);
|
|
24
|
+
}
|
|
22
25
|
export function isWarningStatus(status) {
|
|
23
26
|
return /warning/i.test(status);
|
|
24
27
|
}
|
|
@@ -75,7 +78,7 @@ export function lastAuthSuccessAt(state) {
|
|
|
75
78
|
}
|
|
76
79
|
export function classifyAccountHealth(state, options, nowMs, requestedModel) {
|
|
77
80
|
const staleAfterMs = options.staleAfterMs ?? DEFAULT_STALE_AFTER_MS;
|
|
78
|
-
const
|
|
81
|
+
const generalThreshold = options.utilizationThreshold ?? DEFAULT_UTILIZATION_THRESHOLD;
|
|
79
82
|
if (!state)
|
|
80
83
|
return { verdict: "no_data" };
|
|
81
84
|
const credentialFailure = credentialFailureFor(state, nowMs);
|
|
@@ -153,6 +156,9 @@ export function classifyAccountHealth(state, options, nowMs, requestedModel) {
|
|
|
153
156
|
reason: `${window} rejected ${Math.round((nowMs - w.seenAt) / 60000)}m ago (no reset time; ${Math.round(resetLessBlockMs(window) / 3600000)}h block)`,
|
|
154
157
|
};
|
|
155
158
|
}
|
|
159
|
+
const threshold = isLongQuotaWindow(window)
|
|
160
|
+
? options.weeklyUtilizationThreshold ?? generalThreshold
|
|
161
|
+
: generalThreshold;
|
|
156
162
|
if (typeof w.utilization === "number" &&
|
|
157
163
|
w.utilization >= threshold &&
|
|
158
164
|
(resetBearing || resetMs === undefined)) {
|
|
@@ -216,7 +222,27 @@ export function summarizeWindowUsage(state, options, nowMs) {
|
|
|
216
222
|
}
|
|
217
223
|
return usage.sort((a, b) => (b.resetsAt ?? 0) - (a.resetsAt ?? 0));
|
|
218
224
|
}
|
|
219
|
-
export function
|
|
225
|
+
export function poolStrategy(value) {
|
|
226
|
+
return value === "reset-aware" ? "reset-aware" : "ordered";
|
|
227
|
+
}
|
|
228
|
+
export function sevenDayReset(state, options, nowMs) {
|
|
229
|
+
const usage = summarizeWindowUsage(state, options, nowMs);
|
|
230
|
+
return (usage.find((w) => w.window === "usage:seven_day" && w.resetsAt !== undefined)
|
|
231
|
+
?? usage.find((w) => w.window === "seven_day" && w.resetsAt !== undefined))?.resetsAt;
|
|
232
|
+
}
|
|
233
|
+
export function choosePoolAccount(pool, strategy = "ordered") {
|
|
234
|
+
if (strategy === "reset-aware") {
|
|
235
|
+
let best;
|
|
236
|
+
for (const member of pool) {
|
|
237
|
+
if (member.verdict !== "ok")
|
|
238
|
+
continue;
|
|
239
|
+
if (!best || (member.sevenDayResetsAt !== undefined &&
|
|
240
|
+
(best.sevenDayResetsAt === undefined || member.sevenDayResetsAt < best.sevenDayResetsAt)))
|
|
241
|
+
best = member;
|
|
242
|
+
}
|
|
243
|
+
if (best?.sevenDayResetsAt !== undefined)
|
|
244
|
+
return best.id;
|
|
245
|
+
}
|
|
220
246
|
const usable = pool.find((a) => a.verdict === "ok" || a.verdict === "no_data");
|
|
221
247
|
if (usable)
|
|
222
248
|
return usable.id;
|
|
@@ -246,6 +272,6 @@ export function fallbackPoolAccount(pool, nowMs) {
|
|
|
246
272
|
return retest.id;
|
|
247
273
|
return (pool.find((a) => a.verdict !== "credential_failed") ?? pool[0]).id;
|
|
248
274
|
}
|
|
249
|
-
export function pickPoolAccountForLaunch(pool, nowMs) {
|
|
250
|
-
return choosePoolAccount(pool) ?? fallbackPoolAccount(pool, nowMs);
|
|
275
|
+
export function pickPoolAccountForLaunch(pool, nowMs, strategy = "ordered") {
|
|
276
|
+
return choosePoolAccount(pool, strategy) ?? fallbackPoolAccount(pool, nowMs);
|
|
251
277
|
}
|
package/dist/index.js
CHANGED
|
@@ -7,7 +7,7 @@ import { MODEL_ALIASES, buildCatalogEntries, canonicalModelId, isModernClaudeMod
|
|
|
7
7
|
import { decideDegradation, matchesPin } from "./degrade.js";
|
|
8
8
|
import { resolveExecMode, permissionModeArgs } from "./exec-policy.js";
|
|
9
9
|
import { resolveBaseModelIds } from "./catalog-source.js";
|
|
10
|
-
import { allCredentialFailed, classifyAccountHealth, credentialFailureFor, lastAuthSuccessAt, overdueProbeAccount, } from "./health.js";
|
|
10
|
+
import { allCredentialFailed, classifyAccountHealth, credentialFailureFor, lastAuthSuccessAt, overdueProbeAccount, sevenDayReset, } from "./health.js";
|
|
11
11
|
import { decideStickySelection } from "./sticky.js";
|
|
12
12
|
import { mergeHealthStates, parseStoredState, recordCredentialFailure, } from "./shim-core.js";
|
|
13
13
|
import { createTokenRefResolver, isSecretRefShape, } from "./token-resolution.js";
|
|
@@ -805,6 +805,7 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
805
805
|
}
|
|
806
806
|
const options = {
|
|
807
807
|
utilizationThreshold: pool.utilizationThreshold,
|
|
808
|
+
weeklyUtilizationThreshold: pool.weeklyUtilizationThreshold,
|
|
808
809
|
staleAfterMs: pool.staleAfterMs,
|
|
809
810
|
rotateOnOverage: pool.rotateOnOverage,
|
|
810
811
|
};
|
|
@@ -842,12 +843,16 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
842
843
|
id: v.id,
|
|
843
844
|
verdict: v.health.verdict,
|
|
844
845
|
observedAt: v.health.observedAt,
|
|
846
|
+
sevenDayResetsAt: pool.strategy === "reset-aware"
|
|
847
|
+
? sevenDayReset(states.find((s) => s.id === v.id)?.state, options, now)
|
|
848
|
+
: undefined,
|
|
845
849
|
}));
|
|
846
850
|
const decision = decideStickySelection({
|
|
847
851
|
verdicts: poolVerdicts,
|
|
848
852
|
sticky: previousSticky,
|
|
849
853
|
nowMs: now,
|
|
850
854
|
minDwellMs,
|
|
855
|
+
strategy: pool.strategy,
|
|
851
856
|
});
|
|
852
857
|
const chosen = members.find((a) => a.id === decision.account) ?? members[0];
|
|
853
858
|
const previousAccount = previousSticky?.account ?? members[0].id;
|
|
@@ -1040,6 +1045,7 @@ export function startDirectOrderSync(params) {
|
|
|
1040
1045
|
const intervalMs = Math.max(15_000, params.directRoute?.intervalMs ?? 60_000);
|
|
1041
1046
|
const healthOptions = {
|
|
1042
1047
|
utilizationThreshold: params.pool?.utilizationThreshold,
|
|
1048
|
+
weeklyUtilizationThreshold: params.pool?.weeklyUtilizationThreshold,
|
|
1043
1049
|
staleAfterMs: params.pool?.staleAfterMs,
|
|
1044
1050
|
rotateOnOverage: params.pool?.rotateOnOverage,
|
|
1045
1051
|
};
|
|
@@ -1049,6 +1055,7 @@ export function startDirectOrderSync(params) {
|
|
|
1049
1055
|
intervalMs,
|
|
1050
1056
|
healthOptions,
|
|
1051
1057
|
minDwellMs: params.pool?.minDwellMs,
|
|
1058
|
+
strategy: params.pool?.strategy,
|
|
1052
1059
|
command: params.directRoute?.openclawCommand,
|
|
1053
1060
|
});
|
|
1054
1061
|
if (directController?.signature === signature && directTimer) {
|
|
@@ -1063,6 +1070,7 @@ export function startDirectOrderSync(params) {
|
|
|
1063
1070
|
readHealth: (id) => readHealthState(id),
|
|
1064
1071
|
healthOptions,
|
|
1065
1072
|
minDwellMs: params.pool?.minDwellMs,
|
|
1073
|
+
strategy: params.pool?.strategy,
|
|
1066
1074
|
configOrder: params.configOrder,
|
|
1067
1075
|
readSticky: () => readStickyEntry(stickyFile),
|
|
1068
1076
|
writeSticky: (entry) => writeStickyEntry(stickyFile, entry, logger),
|
|
@@ -1137,6 +1145,7 @@ export function startUsagePoll(params) {
|
|
|
1137
1145
|
const warnThreshold = effectiveUsageWarnThreshold(pool.usagePoll);
|
|
1138
1146
|
const healthOptions = {
|
|
1139
1147
|
utilizationThreshold: pool.utilizationThreshold,
|
|
1148
|
+
weeklyUtilizationThreshold: pool.weeklyUtilizationThreshold,
|
|
1140
1149
|
staleAfterMs: pool.staleAfterMs,
|
|
1141
1150
|
rotateOnOverage: pool.rotateOnOverage,
|
|
1142
1151
|
};
|
|
@@ -1205,7 +1214,7 @@ export function startUsagePoll(params) {
|
|
|
1205
1214
|
usageInitial.unref?.();
|
|
1206
1215
|
usageTimer = setInterval(tick, intervalMs);
|
|
1207
1216
|
usageTimer.unref?.();
|
|
1208
|
-
logger.info(`[multi-clawd] usage poll: reading live usage for ${members.map((m) => m.id).join(", ")} every ${Math.round(intervalMs / 1000)}s — rotate at ${Math.round((pool.utilizationThreshold ?? 0.85) * 100)}
|
|
1217
|
+
logger.info(`[multi-clawd] usage poll: reading live usage for ${members.map((m) => m.id).join(", ")} every ${Math.round(intervalMs / 1000)}s — rotate at ${Math.round((pool.utilizationThreshold ?? 0.85) * 100)}%${pool.weeklyUtilizationThreshold !== undefined ? ` (weekly ${Math.round(pool.weeklyUtilizationThreshold * 100)}%)` : ""}, warn at ${Math.round(warnThreshold * 100)}%${skipped.length > 0 ? `; not polled: ${skipped.join("; ")}` : ""}`);
|
|
1209
1218
|
return { active: true, members: members.map((m) => m.id) };
|
|
1210
1219
|
}
|
|
1211
1220
|
export async function runUsagePollTickNow() {
|
package/dist/sticky.js
CHANGED
|
@@ -4,9 +4,19 @@ export function decideStickySelection(params) {
|
|
|
4
4
|
const { verdicts, sticky, nowMs } = params;
|
|
5
5
|
const minDwellMs = params.minDwellMs ?? DEFAULT_MIN_DWELL_MS;
|
|
6
6
|
const home = verdicts[0];
|
|
7
|
-
const healthChoice = choosePoolAccount(verdicts);
|
|
7
|
+
const healthChoice = choosePoolAccount(verdicts, params.strategy);
|
|
8
8
|
if (!healthChoice)
|
|
9
9
|
return { account: fallbackPoolAccount(verdicts, nowMs) };
|
|
10
|
+
if (params.strategy === "reset-aware") {
|
|
11
|
+
const current = sticky && verdicts.find((v) => v.id === sticky.account);
|
|
12
|
+
const stillUsable = current?.verdict === "ok" || current?.verdict === "no_data";
|
|
13
|
+
if (sticky && stillUsable && nowMs - sticky.since < minDwellMs) {
|
|
14
|
+
return { account: sticky.account, sticky };
|
|
15
|
+
}
|
|
16
|
+
if (sticky?.account === healthChoice)
|
|
17
|
+
return { account: healthChoice, sticky };
|
|
18
|
+
return { account: healthChoice, sticky: { account: healthChoice, since: nowMs } };
|
|
19
|
+
}
|
|
10
20
|
const stickyVerdict = sticky
|
|
11
21
|
? verdicts.find((v) => v.id === sticky.account)?.verdict
|
|
12
22
|
: undefined;
|
package/openclaw.plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "multi-clawd",
|
|
3
3
|
"name": "multi-clawd",
|
|
4
|
-
"version": "2.
|
|
4
|
+
"version": "2.3.0",
|
|
5
5
|
"description": "Register additional Claude Code logins (Max/Pro accounts) as first-class OpenClaw CLI backends for cross-account failover, keeping the full skills/MCP harness on every account.",
|
|
6
6
|
"cliBackends": [
|
|
7
7
|
"claw1",
|
|
@@ -41,12 +41,23 @@
|
|
|
41
41
|
"items": {
|
|
42
42
|
"type": "string"
|
|
43
43
|
},
|
|
44
|
-
"description": "Account ids (from accounts[]) in preference order; the first is the home account
|
|
44
|
+
"description": "Account ids (from accounts[]) in preference order; the first is the home account in the default ordered strategy."
|
|
45
|
+
},
|
|
46
|
+
"strategy": {
|
|
47
|
+
"type": "string",
|
|
48
|
+
"enum": ["ordered", "reset-aware"],
|
|
49
|
+
"description": "Healthy account selection: ordered (default) keeps pool order; reset-aware prefers the healthy account whose live seven-day quota window resets soonest."
|
|
45
50
|
},
|
|
46
51
|
"utilizationThreshold": {
|
|
47
52
|
"type": "number",
|
|
48
53
|
"description": "Hand over when any rate-limit window's utilization reaches this fraction. Default 0.85."
|
|
49
54
|
},
|
|
55
|
+
"weeklyUtilizationThreshold": {
|
|
56
|
+
"type": "number",
|
|
57
|
+
"minimum": 0,
|
|
58
|
+
"maximum": 1,
|
|
59
|
+
"description": "Hand over when a day/week/month-scale quota window (such as seven_day) reaches this fraction. Defaults to utilizationThreshold. Minute/hour and overage windows keep using utilizationThreshold; overage rotation still requires rotateOnOverage."
|
|
60
|
+
},
|
|
50
61
|
"staleAfterMs": {
|
|
51
62
|
"type": "number",
|
|
52
63
|
"description": "Ignore health data older than this. Default 21600000 (6h)."
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@drakon-systems/multi-clawd",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.3.0",
|
|
4
4
|
"description": "Multi-account Claude Code failover for OpenClaw — register additional Claude (Max/Pro) logins as first-class CLI backends and keep the full skills/MCP harness across every account. Also imports those accounts' setup tokens into Hermes Agent's Anthropic credential pool.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/scripts/cli.mjs
CHANGED
|
@@ -200,6 +200,7 @@ async function loadDirect(config) {
|
|
|
200
200
|
const runner = run.createOpenclawRunner(directRoute.openclawCommand ?? "openclaw");
|
|
201
201
|
const healthOptions = {
|
|
202
202
|
utilizationThreshold: pool?.utilizationThreshold,
|
|
203
|
+
weeklyUtilizationThreshold: pool?.weeklyUtilizationThreshold,
|
|
203
204
|
staleAfterMs: pool?.staleAfterMs,
|
|
204
205
|
rotateOnOverage: pool?.rotateOnOverage,
|
|
205
206
|
};
|
|
@@ -214,6 +215,7 @@ async function loadDirect(config) {
|
|
|
214
215
|
configOrder,
|
|
215
216
|
sticky,
|
|
216
217
|
minDwellMs: pool?.minDwellMs,
|
|
218
|
+
strategy: pool?.strategy,
|
|
217
219
|
nowMs: Date.now(),
|
|
218
220
|
...opts,
|
|
219
221
|
});
|
|
@@ -676,6 +678,7 @@ async function explain() {
|
|
|
676
678
|
}
|
|
677
679
|
const h = health.classifyAccountHealth(state, {
|
|
678
680
|
utilizationThreshold: pool?.utilizationThreshold,
|
|
681
|
+
weeklyUtilizationThreshold: pool?.weeklyUtilizationThreshold,
|
|
679
682
|
staleAfterMs: pool?.staleAfterMs,
|
|
680
683
|
}, now);
|
|
681
684
|
let detail = h.reason;
|
|
@@ -684,14 +687,17 @@ async function explain() {
|
|
|
684
687
|
}
|
|
685
688
|
const usage = health.summarizeWindowUsage(state, {
|
|
686
689
|
utilizationThreshold: pool?.utilizationThreshold,
|
|
690
|
+
weeklyUtilizationThreshold: pool?.weeklyUtilizationThreshold,
|
|
687
691
|
staleAfterMs: pool?.staleAfterMs,
|
|
688
692
|
}, now);
|
|
689
693
|
return { id: a.id, verdict: h.verdict, detail, usage };
|
|
690
694
|
});
|
|
691
695
|
let stickyAccount;
|
|
696
|
+
let stickyEntry;
|
|
692
697
|
if (pool) {
|
|
693
698
|
try {
|
|
694
699
|
const sticky = JSON.parse(rf(join(stateDir, `pool-${pool.id}.sticky.json`), "utf8"));
|
|
700
|
+
stickyEntry = sticky;
|
|
695
701
|
if (sticky?.account && sticky.account !== pool.accounts[0]) stickyAccount = sticky.account;
|
|
696
702
|
} catch {
|
|
697
703
|
/* no sticky state */
|
|
@@ -707,7 +713,7 @@ async function explain() {
|
|
|
707
713
|
}
|
|
708
714
|
console.log(`\n${BOLD}🦞 multi-clawd — your setup, in plain English${RESET}\n`);
|
|
709
715
|
console.log(
|
|
710
|
-
ec.renderExplanation({ accounts, pool, chain, health: healthRows, stickyAccount, nowMs: now, direct }),
|
|
716
|
+
ec.renderExplanation({ accounts, pool, chain, health: healthRows, stickyAccount, stickyEntry, nowMs: now, direct }),
|
|
711
717
|
);
|
|
712
718
|
console.log(`\n${DIM}(health checks: multi-clawd doctor · change things: multi-clawd setup)${RESET}`);
|
|
713
719
|
}
|
|
@@ -742,14 +748,17 @@ async function usageCommand(args) {
|
|
|
742
748
|
const pool = pc.pool ? { ...pc.pool, id: pc.pool.id?.trim() || "clawd", accounts: pc.pool.accounts ?? [] } : undefined;
|
|
743
749
|
const healthOptions = {
|
|
744
750
|
utilizationThreshold: pool?.utilizationThreshold,
|
|
751
|
+
weeklyUtilizationThreshold: pool?.weeklyUtilizationThreshold,
|
|
745
752
|
staleAfterMs: pool?.staleAfterMs,
|
|
746
753
|
rotateOnOverage: pool?.rotateOnOverage,
|
|
747
754
|
};
|
|
748
755
|
const warnThreshold = up.effectiveUsageWarnThreshold(pool?.usagePoll);
|
|
749
756
|
const rotateAt = pool?.utilizationThreshold ?? 0.85;
|
|
757
|
+
const weeklyRotateAt = pool?.weeklyUtilizationThreshold;
|
|
750
758
|
const stateDir = join(homedir(), ".openclaw", "state", "multi-clawd");
|
|
751
759
|
const now = Date.now();
|
|
752
760
|
const rows = [];
|
|
761
|
+
const resetById = new Map();
|
|
753
762
|
for (const a of accounts) {
|
|
754
763
|
const row = { id: a.id, label: a.label };
|
|
755
764
|
if (a.oauthTokenFile || a.oauthTokenRef) {
|
|
@@ -780,12 +789,21 @@ async function usageCommand(args) {
|
|
|
780
789
|
const h = health.classifyAccountHealth(state, healthOptions, now);
|
|
781
790
|
row.verdict = h.verdict;
|
|
782
791
|
row.reason = h.reason;
|
|
792
|
+
if (pool?.strategy === "reset-aware") resetById.set(a.id, health.sevenDayReset(state, healthOptions, now));
|
|
783
793
|
rows.push(row);
|
|
784
794
|
}
|
|
785
795
|
let next;
|
|
786
796
|
if (pool) {
|
|
787
|
-
const verdicts = pool.accounts.map((id) => ({ id, verdict: rows.find((r) => r.id === id)?.verdict ?? "no_data"
|
|
788
|
-
|
|
797
|
+
const verdicts = pool.accounts.map((id) => ({ id, verdict: rows.find((r) => r.id === id)?.verdict ?? "no_data",
|
|
798
|
+
sevenDayResetsAt: resetById.get(id) }));
|
|
799
|
+
if (pool.strategy === "reset-aware") {
|
|
800
|
+
const stickySelection = await import(resolve(__dirname, "..", "dist", "sticky.js"));
|
|
801
|
+
let sticky;
|
|
802
|
+
try { sticky = JSON.parse(rf(join(stateDir, `pool-${pool.id}.sticky.json`), "utf8")); } catch { /* none */ }
|
|
803
|
+
next = stickySelection.decideStickySelection({ verdicts, sticky, nowMs: now, minDwellMs: pool.minDwellMs, strategy: pool.strategy }).account;
|
|
804
|
+
} else {
|
|
805
|
+
next = health.choosePoolAccount(verdicts);
|
|
806
|
+
}
|
|
789
807
|
}
|
|
790
808
|
if (args.includes("--json")) {
|
|
791
809
|
console.log(JSON.stringify({ at: new Date(now).toISOString(), rotateAt, warnThreshold, pool: pool?.id, nextLaunch: next, accounts: rows }, null, 2));
|
|
@@ -797,13 +815,17 @@ async function usageCommand(args) {
|
|
|
797
815
|
const mins = Math.max(0, Math.round((resetsAt * 1000 - now) / 60000));
|
|
798
816
|
return mins < 90 ? ` (resets in ~${mins}m)` : mins < 36 * 60 ? ` (resets in ~${Math.round(mins / 60)}h)` : ` (resets in ~${Math.round(mins / 1440)}d)`;
|
|
799
817
|
};
|
|
800
|
-
const mark = (u) =>
|
|
801
|
-
|
|
818
|
+
const mark = (u, window) => {
|
|
819
|
+
const weekly = window && health.isLongQuotaWindow(window);
|
|
820
|
+
const threshold = weekly ? weeklyRotateAt ?? rotateAt : rotateAt;
|
|
821
|
+
return u >= 1 ? "⛔" : u >= warnThreshold ? "🔴" : u >= threshold ? "🟠" : "🟢";
|
|
822
|
+
};
|
|
823
|
+
console.log(`\n${BOLD}🦞 multi-clawd — live usage${RESET} ${DIM}(rotate at ${pct(rotateAt)}${weeklyRotateAt !== undefined ? `, weekly ${pct(weeklyRotateAt)}` : ""}, warn at ${pct(warnThreshold)})${RESET}\n`);
|
|
802
824
|
for (const r of rows) {
|
|
803
825
|
console.log(` ${BOLD}${r.id}${RESET}${r.label ? ` ${DIM}— ${r.label}${RESET}` : ""} pool verdict: ${r.verdict}${r.reason ? ` ${DIM}(${r.reason})${RESET}` : ""}`);
|
|
804
826
|
if (r.snapshot) {
|
|
805
827
|
for (const [w, u] of Object.entries(r.snapshot.windows)) {
|
|
806
|
-
console.log(` ${mark(u.utilization)} ${up.usageWindowLabel(w).padEnd(12)} ${pct(u.utilization).padStart(4)}${until(u.resetsAt)}`);
|
|
828
|
+
console.log(` ${mark(u.utilization, w)} ${up.usageWindowLabel(w).padEnd(12)} ${pct(u.utilization).padStart(4)}${until(u.resetsAt)}`);
|
|
807
829
|
}
|
|
808
830
|
for (const s of r.snapshot.scoped) {
|
|
809
831
|
console.log(` ${mark(s.utilization)} ${`${s.label} model`.padEnd(12)} ${pct(s.utilization).padStart(4)}${until(s.resetsAt)}`);
|
package/scripts/doctor.mjs
CHANGED
|
@@ -294,13 +294,15 @@ const { createSystemCredentialIo, findCredentialCopies, describeSplitStore } = a
|
|
|
294
294
|
["createSystemCredentialIo", "findCredentialCopies", "describeSplitStore"],
|
|
295
295
|
"cli",
|
|
296
296
|
);
|
|
297
|
-
const { summarizeWindowUsage, classifyAccountHealth, overdueProbeAccount, lastAuthSuccessAt } =
|
|
297
|
+
const { summarizeWindowUsage, classifyAccountHealth, overdueProbeAccount, lastAuthSuccessAt, sevenDayReset } =
|
|
298
298
|
await importDist("health.js", [
|
|
299
299
|
"summarizeWindowUsage",
|
|
300
300
|
"classifyAccountHealth",
|
|
301
301
|
"overdueProbeAccount",
|
|
302
302
|
"lastAuthSuccessAt",
|
|
303
|
+
"sevenDayReset",
|
|
303
304
|
]);
|
|
305
|
+
const { DEFAULT_UTILIZATION_THRESHOLD } = await import(join(REPO_DIR, "dist", "health.js"));
|
|
304
306
|
// "cli", like the chain audits: resolving WHICH login backs an account is
|
|
305
307
|
// doctor's own diagnostic, not a description of the running plugin's
|
|
306
308
|
// behaviour. Asking the installed copy meant a fix to the resolver could not
|
|
@@ -495,6 +497,9 @@ else {
|
|
|
495
497
|
const members = (pool.accounts ?? []).filter((id) => accounts.some((a) => a.id === id));
|
|
496
498
|
if (members.length < 2) bad(`pool "${pool.id ?? "clawd"}" has ${members.length} valid member(s); needs ≥ 2`);
|
|
497
499
|
else ok(`pool "${pool.id ?? "clawd"}": ${members.join(" → ")}`);
|
|
500
|
+
const generalThreshold = pool.utilizationThreshold ?? DEFAULT_UTILIZATION_THRESHOLD;
|
|
501
|
+
const weeklyThreshold = pool.weeklyUtilizationThreshold ?? generalThreshold;
|
|
502
|
+
note(`thresholds: general ${Math.round(generalThreshold * 100)}%, weekly ${Math.round(weeklyThreshold * 100)}%${pool.weeklyUtilizationThreshold === undefined ? " (follows general)" : ""}`);
|
|
498
503
|
// Setup-token members: what they do and do not get, said once, plainly.
|
|
499
504
|
const tokenMembers = members
|
|
500
505
|
.map((id) => accounts.find((a) => a.id === id))
|
|
@@ -529,18 +534,20 @@ else {
|
|
|
529
534
|
const healthOptions = {
|
|
530
535
|
staleAfterMs: pool.staleAfterMs,
|
|
531
536
|
utilizationThreshold: pool.utilizationThreshold,
|
|
537
|
+
weeklyUtilizationThreshold: pool.weeklyUtilizationThreshold,
|
|
532
538
|
rotateOnOverage: pool.rotateOnOverage,
|
|
533
539
|
};
|
|
534
|
-
const verdicts = members.map((id) =>
|
|
535
|
-
id
|
|
536
|
-
verdict: classifyAccountHealth(
|
|
537
|
-
.
|
|
538
|
-
})
|
|
540
|
+
const verdicts = members.map((id) => {
|
|
541
|
+
const state = readJson(join(STATE_DIR, `${id}.json`));
|
|
542
|
+
return { id, verdict: classifyAccountHealth(state, healthOptions, now).verdict,
|
|
543
|
+
sevenDayResetsAt: pool.strategy === "reset-aware" ? sevenDayReset(state, healthOptions, now) : undefined };
|
|
544
|
+
});
|
|
539
545
|
const decision = decideStickySelection({
|
|
540
546
|
verdicts,
|
|
541
547
|
sticky,
|
|
542
548
|
nowMs: now,
|
|
543
549
|
minDwellMs: pool.minDwellMs,
|
|
550
|
+
strategy: pool.strategy,
|
|
544
551
|
});
|
|
545
552
|
const identity = identities.find((i) => i.accountId === decision.account);
|
|
546
553
|
const who =
|
|
@@ -876,12 +883,14 @@ if (accounts.some((a) => a && a.direct !== undefined && a.direct !== false)) {
|
|
|
876
883
|
readHealth,
|
|
877
884
|
healthOptions: {
|
|
878
885
|
utilizationThreshold: pool?.utilizationThreshold,
|
|
886
|
+
weeklyUtilizationThreshold: pool?.weeklyUtilizationThreshold,
|
|
879
887
|
staleAfterMs: pool?.staleAfterMs,
|
|
880
888
|
rotateOnOverage: pool?.rotateOnOverage,
|
|
881
889
|
},
|
|
882
890
|
configOrder: Array.isArray(config?.auth?.order?.anthropic) ? config.auth.order.anthropic : undefined,
|
|
883
891
|
sticky: readJson(join(STATE_DIR, "direct-order.sticky.json")),
|
|
884
892
|
minDwellMs: pool?.minDwellMs,
|
|
893
|
+
strategy: pool?.strategy,
|
|
885
894
|
nowMs: Date.now(),
|
|
886
895
|
});
|
|
887
896
|
if (!status) continue;
|