@drakon-systems/multi-clawd 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +452 -0
- package/dist/account-env.js +35 -0
- package/dist/alerts.js +28 -0
- package/dist/catalog-source.js +34 -0
- package/dist/chain-audit.js +202 -0
- package/dist/degrade.js +32 -0
- package/dist/exec-policy.js +15 -0
- package/dist/health.js +83 -0
- package/dist/index.js +508 -0
- package/dist/login-health.js +91 -0
- package/dist/models.js +69 -0
- package/dist/setup-core.js +140 -0
- package/dist/shim-core.js +218 -0
- package/dist/shim.js +151 -0
- package/dist/sticky.js +24 -0
- package/dist/token-resolution.js +62 -0
- package/dist/watchdog-core.js +53 -0
- package/openclaw.plugin.json +134 -0
- package/package.json +65 -0
- package/scripts/doctor.mjs +340 -0
- package/scripts/eviction-watchdog.mjs +178 -0
- package/scripts/setup.mjs +231 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { isSecretRefShape } from "./token-resolution.js";
|
|
2
|
+
const PLUGIN_ID = "multi-clawd";
|
|
3
|
+
function assertAccountId(id) {
|
|
4
|
+
const trimmed = id?.trim();
|
|
5
|
+
if (!trimmed)
|
|
6
|
+
throw new Error("account id must be non-empty");
|
|
7
|
+
if (trimmed === "claude-cli") {
|
|
8
|
+
throw new Error(`account id "${trimmed}" collides with the bundled backend id`);
|
|
9
|
+
}
|
|
10
|
+
return trimmed;
|
|
11
|
+
}
|
|
12
|
+
export function buildMainAccount(opts) {
|
|
13
|
+
return { id: assertAccountId(opts.id), label: opts.label, native: true };
|
|
14
|
+
}
|
|
15
|
+
export function buildSecondAccount(opts) {
|
|
16
|
+
const id = assertAccountId(opts.id);
|
|
17
|
+
const dirError = validateSecondConfigDir(opts.configDir);
|
|
18
|
+
if (dirError)
|
|
19
|
+
throw new Error(dirError);
|
|
20
|
+
const account = { id, label: opts.label, configDir: opts.configDir };
|
|
21
|
+
switch (opts.tokenSource.kind) {
|
|
22
|
+
case "ref":
|
|
23
|
+
if (!isSecretRefShape(opts.tokenSource.ref)) {
|
|
24
|
+
throw new Error('secret ref must be { "source": "...", "provider": "...", "id": "..." } — e.g. { "source": "exec", "provider": "onepassword", "id": "op://Vault/Item/field" }');
|
|
25
|
+
}
|
|
26
|
+
account.oauthTokenRef = opts.tokenSource.ref;
|
|
27
|
+
break;
|
|
28
|
+
case "file":
|
|
29
|
+
account.oauthTokenFile = opts.tokenSource.path;
|
|
30
|
+
break;
|
|
31
|
+
case "dir-login":
|
|
32
|
+
break;
|
|
33
|
+
}
|
|
34
|
+
return account;
|
|
35
|
+
}
|
|
36
|
+
export function validateSecondConfigDir(dir) {
|
|
37
|
+
const d = (dir ?? "").trim().replace(/\/+$/, "");
|
|
38
|
+
if (!d.startsWith("~/") && !d.startsWith("/")) {
|
|
39
|
+
return `config dir must be an absolute path or start with ~/ (got "${dir}")`;
|
|
40
|
+
}
|
|
41
|
+
if (d === "~/.claude" || /\/\.claude$/.test(d)) {
|
|
42
|
+
return "that's the DEFAULT Claude config dir — it belongs to your main (native) login; pick an isolated dir like ~/.claw2";
|
|
43
|
+
}
|
|
44
|
+
if (d.startsWith("~/.claude/") || d.includes("/.claude/")) {
|
|
45
|
+
return "the second account's dir must not live inside ~/.claude — it must be fully isolated from the main login";
|
|
46
|
+
}
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
49
|
+
export function buildPool(accountIds, opts) {
|
|
50
|
+
const pool = {
|
|
51
|
+
id: opts?.id?.trim() || "clawd",
|
|
52
|
+
accounts: accountIds,
|
|
53
|
+
};
|
|
54
|
+
if (opts?.label)
|
|
55
|
+
pool.label = opts.label;
|
|
56
|
+
return pool;
|
|
57
|
+
}
|
|
58
|
+
function asRecord(v) {
|
|
59
|
+
return typeof v === "object" && v !== null && !Array.isArray(v)
|
|
60
|
+
? v
|
|
61
|
+
: undefined;
|
|
62
|
+
}
|
|
63
|
+
export function planFromExisting(config) {
|
|
64
|
+
const cfg = asRecord(config) ?? {};
|
|
65
|
+
const plugins = asRecord(cfg.plugins);
|
|
66
|
+
const allow = Array.isArray(plugins?.allow) ? plugins?.allow : undefined;
|
|
67
|
+
const entries = asRecord(plugins?.entries);
|
|
68
|
+
const entry = asRecord(entries?.[PLUGIN_ID]);
|
|
69
|
+
const entryConfig = asRecord(entry?.config);
|
|
70
|
+
const accounts = Array.isArray(entryConfig?.accounts)
|
|
71
|
+
? entryConfig?.accounts
|
|
72
|
+
: [];
|
|
73
|
+
return {
|
|
74
|
+
hasPluginEntry: entry !== undefined,
|
|
75
|
+
accountIds: accounts
|
|
76
|
+
.map((a) => asRecord(a)?.id)
|
|
77
|
+
.filter((id) => typeof id === "string"),
|
|
78
|
+
hasPool: asRecord(entryConfig?.pool) !== undefined,
|
|
79
|
+
hasAllowList: allow !== undefined,
|
|
80
|
+
allowListed: allow?.includes(PLUGIN_ID) ?? false,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
export function mergeSetupIntoConfig(existing, plan) {
|
|
84
|
+
const config = JSON.parse(JSON.stringify(asRecord(existing) ?? {}));
|
|
85
|
+
const changes = [];
|
|
86
|
+
const plugins = (config.plugins = asRecord(config.plugins) ?? {});
|
|
87
|
+
const entries = (plugins.entries = asRecord(plugins.entries) ?? {});
|
|
88
|
+
const entry = (entries[PLUGIN_ID] = asRecord(entries[PLUGIN_ID]) ?? {});
|
|
89
|
+
if (entry.enabled !== true) {
|
|
90
|
+
entry.enabled = true;
|
|
91
|
+
changes.push(`enable plugins.entries["${PLUGIN_ID}"]`);
|
|
92
|
+
}
|
|
93
|
+
const entryConfig = (entry.config = asRecord(entry.config) ?? {});
|
|
94
|
+
const accounts = (entryConfig.accounts = Array.isArray(entryConfig.accounts)
|
|
95
|
+
? entryConfig.accounts
|
|
96
|
+
: []);
|
|
97
|
+
for (const planned of plan.accounts) {
|
|
98
|
+
const existingIdx = accounts.findIndex((a) => asRecord(a)?.id === planned.id);
|
|
99
|
+
const plannedClean = Object.fromEntries(Object.entries(planned).filter(([, v]) => v !== undefined));
|
|
100
|
+
if (existingIdx === -1) {
|
|
101
|
+
accounts.push(plannedClean);
|
|
102
|
+
changes.push(`add account "${planned.id}"`);
|
|
103
|
+
}
|
|
104
|
+
else {
|
|
105
|
+
const current = asRecord(accounts[existingIdx]) ?? {};
|
|
106
|
+
const merged = { ...current, ...plannedClean };
|
|
107
|
+
if (JSON.stringify(merged) !== JSON.stringify(current)) {
|
|
108
|
+
accounts[existingIdx] = merged;
|
|
109
|
+
changes.push(`update account "${planned.id}"`);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
if (plan.pool) {
|
|
114
|
+
if (asRecord(entryConfig.pool)) {
|
|
115
|
+
const samePool = JSON.stringify(entryConfig.pool) === JSON.stringify(plan.pool);
|
|
116
|
+
if (!samePool)
|
|
117
|
+
changes.push(`pool already exists — skipped (edit it by hand if you want changes)`);
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
120
|
+
entryConfig.pool = plan.pool;
|
|
121
|
+
changes.push(`add pool "${plan.pool.id}" over [${plan.pool.accounts.join(", ")}]`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (Array.isArray(plugins.allow) && !plugins.allow.includes(PLUGIN_ID)) {
|
|
125
|
+
plugins.allow.push(PLUGIN_ID);
|
|
126
|
+
changes.push(`append "${PLUGIN_ID}" to plugins.allow`);
|
|
127
|
+
}
|
|
128
|
+
if (plan.modelRungs.length > 0) {
|
|
129
|
+
const agents = (config.agents = asRecord(config.agents) ?? {});
|
|
130
|
+
const defaults = (agents.defaults = asRecord(agents.defaults) ?? {});
|
|
131
|
+
const models = (defaults.models = asRecord(defaults.models) ?? {});
|
|
132
|
+
for (const rung of plan.modelRungs) {
|
|
133
|
+
if (!(rung in models)) {
|
|
134
|
+
models[rung] = {};
|
|
135
|
+
changes.push(`register model "${rung}"`);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return { config, changes };
|
|
140
|
+
}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
export const PRUNE_AFTER_MS = 14 * 24 * 60 * 60 * 1000;
|
|
2
|
+
const RAW_INFO_MAX_CHARS = 512;
|
|
3
|
+
export function createLineScanner(onLine) {
|
|
4
|
+
let buffer = "";
|
|
5
|
+
return {
|
|
6
|
+
push(chunk) {
|
|
7
|
+
buffer += chunk;
|
|
8
|
+
let idx;
|
|
9
|
+
while ((idx = buffer.indexOf("\n")) !== -1) {
|
|
10
|
+
onLine(buffer.slice(0, idx));
|
|
11
|
+
buffer = buffer.slice(idx + 1);
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
flush() {
|
|
15
|
+
if (buffer.length > 0) {
|
|
16
|
+
onLine(buffer);
|
|
17
|
+
buffer = "";
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
export function parseRateLimitEvent(line) {
|
|
23
|
+
if (!line.includes('"rate_limit_event"'))
|
|
24
|
+
return undefined;
|
|
25
|
+
let record;
|
|
26
|
+
try {
|
|
27
|
+
record = JSON.parse(line);
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
if (typeof record !== "object" || record === null)
|
|
33
|
+
return undefined;
|
|
34
|
+
const r = record;
|
|
35
|
+
if (r.type !== "rate_limit_event")
|
|
36
|
+
return undefined;
|
|
37
|
+
const info = r.rate_limit_info;
|
|
38
|
+
if (typeof info !== "object" || info === null)
|
|
39
|
+
return undefined;
|
|
40
|
+
const i = info;
|
|
41
|
+
if (typeof i.status !== "string" || i.status.length === 0)
|
|
42
|
+
return undefined;
|
|
43
|
+
const rateLimitType = typeof i.rateLimitType === "string" ? i.rateLimitType : undefined;
|
|
44
|
+
const event = {
|
|
45
|
+
status: i.status,
|
|
46
|
+
rateLimitType,
|
|
47
|
+
resetsAt: typeof i.resetsAt === "number" ? i.resetsAt : undefined,
|
|
48
|
+
utilization: typeof i.utilization === "number" ? i.utilization : undefined,
|
|
49
|
+
isUsingOverage: typeof i.isUsingOverage === "boolean" ? i.isUsingOverage : undefined,
|
|
50
|
+
};
|
|
51
|
+
if (rateLimitType === undefined) {
|
|
52
|
+
event.rawInfo = JSON.stringify(info).slice(0, RAW_INFO_MAX_CHARS);
|
|
53
|
+
}
|
|
54
|
+
return event;
|
|
55
|
+
}
|
|
56
|
+
export function classifyStateReadFailure(err) {
|
|
57
|
+
if (typeof err === "object" &&
|
|
58
|
+
err !== null &&
|
|
59
|
+
err.code === "ENOENT") {
|
|
60
|
+
return "absent";
|
|
61
|
+
}
|
|
62
|
+
return "unreadable";
|
|
63
|
+
}
|
|
64
|
+
export function parseStoredState(raw) {
|
|
65
|
+
let parsed;
|
|
66
|
+
try {
|
|
67
|
+
parsed = JSON.parse(raw);
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
if (typeof parsed !== "object" || parsed === null)
|
|
73
|
+
return undefined;
|
|
74
|
+
const p = parsed;
|
|
75
|
+
if (typeof p.windows !== "object" || p.windows === null)
|
|
76
|
+
return undefined;
|
|
77
|
+
const windows = {};
|
|
78
|
+
for (const [key, value] of Object.entries(p.windows)) {
|
|
79
|
+
if (typeof value !== "object" || value === null)
|
|
80
|
+
continue;
|
|
81
|
+
const w = value;
|
|
82
|
+
if (typeof w.status !== "string" || typeof w.seenAt !== "number")
|
|
83
|
+
continue;
|
|
84
|
+
windows[key] = {
|
|
85
|
+
status: w.status,
|
|
86
|
+
resetsAt: typeof w.resetsAt === "number" ? w.resetsAt : undefined,
|
|
87
|
+
utilization: typeof w.utilization === "number" ? w.utilization : undefined,
|
|
88
|
+
isUsingOverage: typeof w.isUsingOverage === "boolean" ? w.isUsingOverage : undefined,
|
|
89
|
+
seenAt: w.seenAt,
|
|
90
|
+
rawInfo: typeof w.rawInfo === "string" ? w.rawInfo : undefined,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
accountId: typeof p.accountId === "string" ? p.accountId : "unknown",
|
|
95
|
+
updatedAt: typeof p.updatedAt === "number" ? p.updatedAt : undefined,
|
|
96
|
+
windows,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
export function mergeHealthStates(disk, live, now, pruneAfterMs = PRUNE_AFTER_MS) {
|
|
100
|
+
const merged = { ...disk.windows };
|
|
101
|
+
for (const [key, w] of Object.entries(live.windows)) {
|
|
102
|
+
const existing = merged[key];
|
|
103
|
+
if (!existing || w.seenAt >= existing.seenAt)
|
|
104
|
+
merged[key] = w;
|
|
105
|
+
}
|
|
106
|
+
const windows = canonicalizeModelWindowKeys(merged);
|
|
107
|
+
if (now !== undefined) {
|
|
108
|
+
for (const [key, w] of Object.entries(windows)) {
|
|
109
|
+
if (now - w.seenAt > pruneAfterMs)
|
|
110
|
+
delete windows[key];
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
const updatedAt = Math.max(disk.updatedAt ?? 0, live.updatedAt ?? 0);
|
|
114
|
+
return {
|
|
115
|
+
accountId: live.accountId,
|
|
116
|
+
updatedAt: updatedAt > 0 ? updatedAt : undefined,
|
|
117
|
+
windows,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
const MODEL_ID_PROVIDER_PREFIXES = [
|
|
121
|
+
"clawd/",
|
|
122
|
+
"claude-cli/",
|
|
123
|
+
"anthropic/",
|
|
124
|
+
];
|
|
125
|
+
const CLAW_ACCOUNT_PREFIX_RE = /^claw\d+\//;
|
|
126
|
+
export function canonicalizeModelIdForWindow(modelId) {
|
|
127
|
+
const clawMatch = CLAW_ACCOUNT_PREFIX_RE.exec(modelId);
|
|
128
|
+
if (clawMatch)
|
|
129
|
+
return modelId.slice(clawMatch[0].length);
|
|
130
|
+
for (const prefix of MODEL_ID_PROVIDER_PREFIXES) {
|
|
131
|
+
if (modelId.startsWith(prefix))
|
|
132
|
+
return modelId.slice(prefix.length);
|
|
133
|
+
}
|
|
134
|
+
return modelId;
|
|
135
|
+
}
|
|
136
|
+
export function modelWindowKey(modelId) {
|
|
137
|
+
return `model:${canonicalizeModelIdForWindow(modelId)}`;
|
|
138
|
+
}
|
|
139
|
+
export function canonicalizeModelWindowKeys(windows) {
|
|
140
|
+
const out = {};
|
|
141
|
+
for (const [key, w] of Object.entries(windows)) {
|
|
142
|
+
const canonicalKey = key.startsWith("model:")
|
|
143
|
+
? modelWindowKey(key.slice("model:".length))
|
|
144
|
+
: key;
|
|
145
|
+
const existing = out[canonicalKey];
|
|
146
|
+
if (!existing || w.seenAt >= existing.seenAt)
|
|
147
|
+
out[canonicalKey] = w;
|
|
148
|
+
}
|
|
149
|
+
return out;
|
|
150
|
+
}
|
|
151
|
+
const LIMIT_TEXT_RE = /reached your (.{1,40}?) limit/i;
|
|
152
|
+
export function parseModelLimitError(line) {
|
|
153
|
+
if (!/reached your /i.test(line))
|
|
154
|
+
return undefined;
|
|
155
|
+
let record;
|
|
156
|
+
try {
|
|
157
|
+
record = JSON.parse(line);
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
return undefined;
|
|
161
|
+
}
|
|
162
|
+
if (typeof record !== "object" || record === null)
|
|
163
|
+
return undefined;
|
|
164
|
+
const r = record;
|
|
165
|
+
const isErrorRecord = r.type === "error" ||
|
|
166
|
+
r.is_error === true ||
|
|
167
|
+
(typeof r.subtype === "string" && r.subtype.startsWith("error"));
|
|
168
|
+
if (!isErrorRecord)
|
|
169
|
+
return undefined;
|
|
170
|
+
const texts = [];
|
|
171
|
+
if (typeof r.result === "string")
|
|
172
|
+
texts.push(r.result);
|
|
173
|
+
if (typeof r.error === "string")
|
|
174
|
+
texts.push(r.error);
|
|
175
|
+
if (typeof r.error === "object" && r.error !== null) {
|
|
176
|
+
const msg = r.error.message;
|
|
177
|
+
if (typeof msg === "string")
|
|
178
|
+
texts.push(msg);
|
|
179
|
+
}
|
|
180
|
+
for (const text of texts) {
|
|
181
|
+
const match = LIMIT_TEXT_RE.exec(text);
|
|
182
|
+
if (match)
|
|
183
|
+
return { displayName: match[1].trim() };
|
|
184
|
+
}
|
|
185
|
+
return undefined;
|
|
186
|
+
}
|
|
187
|
+
export function recordModelLimit(state, modelId, now, resetsAt) {
|
|
188
|
+
return {
|
|
189
|
+
...state,
|
|
190
|
+
updatedAt: now,
|
|
191
|
+
windows: {
|
|
192
|
+
...state.windows,
|
|
193
|
+
[modelWindowKey(modelId)]: {
|
|
194
|
+
status: "rejected",
|
|
195
|
+
resetsAt,
|
|
196
|
+
seenAt: now,
|
|
197
|
+
},
|
|
198
|
+
},
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
export function updateHealthState(state, event, now) {
|
|
202
|
+
const key = event.rateLimitType ?? "unknown";
|
|
203
|
+
return {
|
|
204
|
+
...state,
|
|
205
|
+
updatedAt: now,
|
|
206
|
+
windows: {
|
|
207
|
+
...state.windows,
|
|
208
|
+
[key]: {
|
|
209
|
+
status: event.status,
|
|
210
|
+
resetsAt: event.resetsAt,
|
|
211
|
+
utilization: event.utilization,
|
|
212
|
+
isUsingOverage: event.isUsingOverage,
|
|
213
|
+
seenAt: now,
|
|
214
|
+
rawInfo: event.rawInfo,
|
|
215
|
+
},
|
|
216
|
+
},
|
|
217
|
+
};
|
|
218
|
+
}
|
package/dist/shim.js
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import { classifyStateReadFailure, createLineScanner, mergeHealthStates, parseRateLimitEvent, parseStoredState, updateHealthState, } from "./shim-core.js";
|
|
5
|
+
import { rewriteModelArg } from "./degrade.js";
|
|
6
|
+
import { parseModelLimitError, recordModelLimit } from "./shim-core.js";
|
|
7
|
+
import { canonicalModelId } from "./models.js";
|
|
8
|
+
function resolveClaudeCommand() {
|
|
9
|
+
const override = process.env.MULTI_CLAWD_CLAUDE_BIN;
|
|
10
|
+
if (override) {
|
|
11
|
+
try {
|
|
12
|
+
const parsed = JSON.parse(override);
|
|
13
|
+
if (Array.isArray(parsed) && parsed.length > 0 && parsed.every((p) => typeof p === "string")) {
|
|
14
|
+
return { command: parsed[0], prependArgs: parsed.slice(1) };
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
}
|
|
19
|
+
return { command: override, prependArgs: [] };
|
|
20
|
+
}
|
|
21
|
+
return { command: "claude", prependArgs: [] };
|
|
22
|
+
}
|
|
23
|
+
const stateFile = process.env.MULTI_CLAWD_STATE_FILE;
|
|
24
|
+
const accountId = process.env.MULTI_CLAWD_ACCOUNT_ID ?? "unknown";
|
|
25
|
+
let state = { accountId, windows: {} };
|
|
26
|
+
function preserveCorruptState(raw) {
|
|
27
|
+
if (!stateFile)
|
|
28
|
+
return;
|
|
29
|
+
const preservedPath = `${stateFile}.corrupt-${Date.now()}`;
|
|
30
|
+
let preserved = false;
|
|
31
|
+
if (raw !== undefined) {
|
|
32
|
+
try {
|
|
33
|
+
writeFileSync(preservedPath, raw, { mode: 0o600 });
|
|
34
|
+
preserved = true;
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
const note = preserved
|
|
40
|
+
? `preserved copy: ${preservedPath}`
|
|
41
|
+
: "original bytes could not be preserved";
|
|
42
|
+
process.stderr.write(`[multi-clawd shim] state file unreadable/corrupt — starting fresh (${note})\n`);
|
|
43
|
+
}
|
|
44
|
+
function readPersistedState() {
|
|
45
|
+
if (!stateFile)
|
|
46
|
+
return undefined;
|
|
47
|
+
let raw;
|
|
48
|
+
try {
|
|
49
|
+
raw = readFileSync(stateFile, "utf8");
|
|
50
|
+
}
|
|
51
|
+
catch (err) {
|
|
52
|
+
if (classifyStateReadFailure(err) === "absent")
|
|
53
|
+
return undefined;
|
|
54
|
+
preserveCorruptState(undefined);
|
|
55
|
+
return undefined;
|
|
56
|
+
}
|
|
57
|
+
const parsed = parseStoredState(raw);
|
|
58
|
+
if (parsed === undefined) {
|
|
59
|
+
preserveCorruptState(raw);
|
|
60
|
+
return undefined;
|
|
61
|
+
}
|
|
62
|
+
return parsed;
|
|
63
|
+
}
|
|
64
|
+
function persistState() {
|
|
65
|
+
if (!stateFile)
|
|
66
|
+
return;
|
|
67
|
+
try {
|
|
68
|
+
const disk = readPersistedState();
|
|
69
|
+
if (disk)
|
|
70
|
+
state = mergeHealthStates(disk, state, Date.now());
|
|
71
|
+
mkdirSync(dirname(stateFile), { recursive: true });
|
|
72
|
+
const tmp = `${stateFile}.tmp-${process.pid}`;
|
|
73
|
+
writeFileSync(tmp, JSON.stringify(state, null, 2), { mode: 0o600 });
|
|
74
|
+
renameSync(tmp, stateFile);
|
|
75
|
+
}
|
|
76
|
+
catch (err) {
|
|
77
|
+
process.stderr.write(`[multi-clawd shim] state write failed: ${String(err)}\n`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
const { command, prependArgs } = resolveClaudeCommand();
|
|
81
|
+
let childArgs = [...prependArgs, ...process.argv.slice(2)];
|
|
82
|
+
const modelOverride = process.env.MULTI_CLAWD_MODEL_OVERRIDE;
|
|
83
|
+
if (modelOverride) {
|
|
84
|
+
const before = childArgs.join(" ");
|
|
85
|
+
childArgs = rewriteModelArg(childArgs, modelOverride);
|
|
86
|
+
if (childArgs.join(" ") !== before) {
|
|
87
|
+
process.stderr.write(`[multi-clawd shim] degrading model for this launch → ${modelOverride}\n`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const child = spawn(command, childArgs, {
|
|
91
|
+
stdio: ["pipe", "pipe", "inherit"],
|
|
92
|
+
env: process.env,
|
|
93
|
+
});
|
|
94
|
+
process.stdin.pipe(child.stdin);
|
|
95
|
+
function effectiveModelId() {
|
|
96
|
+
const idx = childArgs.indexOf("--model");
|
|
97
|
+
if (idx < 0 || idx + 1 >= childArgs.length)
|
|
98
|
+
return undefined;
|
|
99
|
+
const raw = childArgs[idx + 1];
|
|
100
|
+
return canonicalModelId(raw) ?? raw;
|
|
101
|
+
}
|
|
102
|
+
function guessLimitResetsAt() {
|
|
103
|
+
const nowS = Date.now() / 1000;
|
|
104
|
+
for (const [key, w] of Object.entries(state.windows)) {
|
|
105
|
+
if (key.includes("seven_day") && typeof w.resetsAt === "number" && w.resetsAt > nowS) {
|
|
106
|
+
return w.resetsAt;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return undefined;
|
|
110
|
+
}
|
|
111
|
+
const scanner = createLineScanner((line) => {
|
|
112
|
+
try {
|
|
113
|
+
const event = parseRateLimitEvent(line);
|
|
114
|
+
if (event) {
|
|
115
|
+
state = updateHealthState(state, event, Date.now());
|
|
116
|
+
persistState();
|
|
117
|
+
}
|
|
118
|
+
const limitHit = parseModelLimitError(line);
|
|
119
|
+
if (limitHit) {
|
|
120
|
+
const model = effectiveModelId();
|
|
121
|
+
if (model) {
|
|
122
|
+
state = recordModelLimit(state, model, Date.now(), guessLimitResetsAt());
|
|
123
|
+
persistState();
|
|
124
|
+
process.stderr.write(`[multi-clawd shim] model limit hit recorded: ${model} (reported as "${limitHit.displayName}")\n`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
child.stdout.on("data", (chunk) => {
|
|
132
|
+
process.stdout.write(chunk);
|
|
133
|
+
scanner.push(chunk.toString("utf8"));
|
|
134
|
+
});
|
|
135
|
+
child.stdout.on("end", () => scanner.flush());
|
|
136
|
+
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
|
|
137
|
+
process.on(signal, () => {
|
|
138
|
+
child.kill(signal);
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
child.on("close", (code, signal) => {
|
|
142
|
+
if (signal) {
|
|
143
|
+
process.kill(process.pid, signal);
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
process.exit(code ?? 0);
|
|
147
|
+
});
|
|
148
|
+
child.on("error", (err) => {
|
|
149
|
+
process.stderr.write(`[multi-clawd shim] failed to spawn claude: ${String(err)}\n`);
|
|
150
|
+
process.exit(127);
|
|
151
|
+
});
|
package/dist/sticky.js
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { choosePoolAccount } from "./health.js";
|
|
2
|
+
export const DEFAULT_MIN_DWELL_MS = 10 * 60 * 1000;
|
|
3
|
+
export function decideStickySelection(params) {
|
|
4
|
+
const { verdicts, sticky, nowMs } = params;
|
|
5
|
+
const minDwellMs = params.minDwellMs ?? DEFAULT_MIN_DWELL_MS;
|
|
6
|
+
const home = verdicts[0];
|
|
7
|
+
const healthChoice = choosePoolAccount(verdicts);
|
|
8
|
+
if (!healthChoice)
|
|
9
|
+
return { account: home.id };
|
|
10
|
+
const stickyVerdict = sticky
|
|
11
|
+
? verdicts.find((v) => v.id === sticky.account)?.verdict
|
|
12
|
+
: undefined;
|
|
13
|
+
const stickyUsable = stickyVerdict === "ok" || stickyVerdict === "no_data";
|
|
14
|
+
if (sticky && sticky.account !== home.id && stickyUsable) {
|
|
15
|
+
const homeUsable = home.verdict === "ok" || home.verdict === "no_data";
|
|
16
|
+
if (homeUsable && nowMs - sticky.since >= minDwellMs) {
|
|
17
|
+
return { account: home.id };
|
|
18
|
+
}
|
|
19
|
+
return { account: sticky.account, sticky };
|
|
20
|
+
}
|
|
21
|
+
if (healthChoice === home.id)
|
|
22
|
+
return { account: home.id };
|
|
23
|
+
return { account: healthChoice, sticky: { account: healthChoice, since: nowMs } };
|
|
24
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
export function redactRefError(_ref, error) {
|
|
2
|
+
const errorClass = error instanceof Error ? error.constructor.name : typeof error;
|
|
3
|
+
return `credential_resolution_failed (${errorClass})`;
|
|
4
|
+
}
|
|
5
|
+
export function isSecretRefShape(value) {
|
|
6
|
+
if (typeof value !== "object" || value === null)
|
|
7
|
+
return false;
|
|
8
|
+
const v = value;
|
|
9
|
+
return (typeof v.source === "string" &&
|
|
10
|
+
v.source.length > 0 &&
|
|
11
|
+
typeof v.provider === "string" &&
|
|
12
|
+
v.provider.length > 0 &&
|
|
13
|
+
typeof v.id === "string" &&
|
|
14
|
+
v.id.length > 0);
|
|
15
|
+
}
|
|
16
|
+
const DEFAULT_TTL_MS = 5 * 60 * 1000;
|
|
17
|
+
function cacheKey(ref) {
|
|
18
|
+
return `${ref.source}\u0000${ref.provider}\u0000${ref.id}`;
|
|
19
|
+
}
|
|
20
|
+
export function createTokenRefResolver(options) {
|
|
21
|
+
const reportError = (ref, error) => {
|
|
22
|
+
options.onError?.(ref, options.redact ? redactRefError(ref, error) : error);
|
|
23
|
+
};
|
|
24
|
+
const ttlMs = options.ttlMs ?? DEFAULT_TTL_MS;
|
|
25
|
+
const cache = new Map();
|
|
26
|
+
const fromCache = (ref, nowMs) => {
|
|
27
|
+
const hit = cache.get(cacheKey(ref));
|
|
28
|
+
if (hit && hit.expiresAt > nowMs)
|
|
29
|
+
return hit.value;
|
|
30
|
+
return undefined;
|
|
31
|
+
};
|
|
32
|
+
return {
|
|
33
|
+
peek(ref, nowMs = Date.now()) {
|
|
34
|
+
return fromCache(ref, nowMs);
|
|
35
|
+
},
|
|
36
|
+
async resolveDetailed(ref, nowMs = Date.now()) {
|
|
37
|
+
const cached = fromCache(ref, nowMs);
|
|
38
|
+
if (cached !== undefined)
|
|
39
|
+
return { value: cached };
|
|
40
|
+
let resolved;
|
|
41
|
+
try {
|
|
42
|
+
const values = await options.resolveRefs([ref]);
|
|
43
|
+
resolved =
|
|
44
|
+
values.get(`${ref.source}:${ref.provider}:${ref.id}`) ?? values.get(ref.id);
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
reportError(ref, error);
|
|
48
|
+
return { failure: "provider_error" };
|
|
49
|
+
}
|
|
50
|
+
if (typeof resolved !== "string" || resolved.trim().length === 0) {
|
|
51
|
+
reportError(ref, new Error(`secret ref resolved to ${resolved === undefined ? "nothing" : typeof resolved}`));
|
|
52
|
+
return { failure: "empty_result" };
|
|
53
|
+
}
|
|
54
|
+
const value = resolved.trim();
|
|
55
|
+
cache.set(cacheKey(ref), { value, expiresAt: nowMs + ttlMs });
|
|
56
|
+
return { value };
|
|
57
|
+
},
|
|
58
|
+
async resolve(ref, nowMs = Date.now()) {
|
|
59
|
+
return (await this.resolveDetailed(ref, nowMs)).value;
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
export function decideWatchdogAction(input) {
|
|
2
|
+
const { state, nowMs } = input;
|
|
3
|
+
const fresh = input.evictionTimestamp !== undefined &&
|
|
4
|
+
(state.lastHandled === undefined || input.evictionTimestamp > state.lastHandled);
|
|
5
|
+
const pending = state.pendingEviction;
|
|
6
|
+
if (!fresh && !pending) {
|
|
7
|
+
return { action: "none", reason: "no unhandled eviction", nextState: state };
|
|
8
|
+
}
|
|
9
|
+
const logTimestamp = pending?.logTimestamp ?? input.evictionTimestamp;
|
|
10
|
+
const firstDeferredAt = pending?.firstDeferredAt;
|
|
11
|
+
const deferCapExceeded = firstDeferredAt !== undefined && nowMs - firstDeferredAt >= input.maxDeferMs;
|
|
12
|
+
if (input.inFlight && !deferCapExceeded) {
|
|
13
|
+
return {
|
|
14
|
+
action: "defer",
|
|
15
|
+
reason: "turn in-flight — deferring restart",
|
|
16
|
+
nextState: {
|
|
17
|
+
...state,
|
|
18
|
+
pendingEviction: {
|
|
19
|
+
logTimestamp,
|
|
20
|
+
firstDeferredAt: firstDeferredAt ?? nowMs,
|
|
21
|
+
defers: (pending?.defers ?? 0) + 1,
|
|
22
|
+
},
|
|
23
|
+
},
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
const sinceRestart = nowMs - (state.lastRestartAt ?? 0);
|
|
27
|
+
if (sinceRestart < input.restartCooldownMs) {
|
|
28
|
+
return {
|
|
29
|
+
action: "defer",
|
|
30
|
+
reason: "restart cooldown active",
|
|
31
|
+
nextState: {
|
|
32
|
+
...state,
|
|
33
|
+
pendingEviction: {
|
|
34
|
+
logTimestamp,
|
|
35
|
+
firstDeferredAt: firstDeferredAt ?? nowMs,
|
|
36
|
+
defers: (pending?.defers ?? 0) + 1,
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
return {
|
|
42
|
+
action: "restart",
|
|
43
|
+
reason: deferCapExceeded
|
|
44
|
+
? `defer cap exceeded after ${pending?.defers ?? 0} deferral(s) — restarting despite in-flight turn`
|
|
45
|
+
: pending
|
|
46
|
+
? `deferred eviction now safe to handle (${pending.defers} deferral(s))`
|
|
47
|
+
: "eviction detected, no turn in flight",
|
|
48
|
+
nextState: {
|
|
49
|
+
lastHandled: logTimestamp,
|
|
50
|
+
lastRestartAt: nowMs,
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|