agent-dealer 1.2.2 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle/server/dist/capacity/claude-events.js +27 -10
- package/bundle/server/dist/capacity/claude-local-cache.js +794 -0
- package/bundle/server/dist/capacity/claude-local-cache.test.js +524 -0
- package/bundle/server/dist/capacity/cursor-individual-credentials.js +203 -16
- package/bundle/server/dist/capacity/cursor-individual-credentials.test.js +186 -2
- package/bundle/server/dist/capacity/cursor-individual.js +69 -2
- package/bundle/server/dist/capacity/cursor-individual.test.js +268 -2
- package/bundle/server/dist/capacity/muse-host.js +778 -0
- package/bundle/server/dist/capacity/muse-host.test.js +480 -0
- package/bundle/server/dist/capacity/muse-lifecycle.test.js +96 -0
- package/bundle/server/dist/capacity/muse-probe.js +250 -0
- package/bundle/server/dist/capacity/muse-probe.test.js +183 -0
- package/bundle/server/dist/capacity/muse.js +320 -63
- package/bundle/server/dist/capacity/muse.test.js +8 -51
- package/bundle/server/dist/coordinator/muse-spawn.js +143 -1
- package/bundle/server/dist/index.js +15 -2
- package/bundle/server/dist/routes/index.js +25 -9
- package/bundle/server/dist/routes/runtime-capacity.test.js +20 -7
- package/bundle/server/dist/runners/muse-code-jsonl.js +7 -1
- package/bundle/server/dist/runners/muse-serve-session.js +314 -0
- package/bundle/server/dist/runners/muse-serve-session.test.js +325 -0
- package/bundle/server/package.json +2 -2
- package/bundle/server/static-ui/assets/codex-YZSpL-SB.png +0 -0
- package/bundle/server/static-ui/assets/{index-DUYzboWh.css → index-CYqRh_-S.css} +1 -1
- package/bundle/server/static-ui/assets/index-UO4lHZw4.js +60 -0
- package/bundle/server/static-ui/assets/muse-CtdcC6ve.svg +33 -0
- package/bundle/server/static-ui/index.html +2 -2
- package/bundle/shared/package.json +1 -1
- package/dist/doctor.d.ts +72 -0
- package/dist/doctor.js +235 -1
- package/dist/doctor.test.d.ts +1 -0
- package/dist/doctor.test.js +276 -0
- package/package.json +1 -1
- package/bundle/server/static-ui/assets/index-0GDYfbaM.js +0 -60
|
@@ -0,0 +1,778 @@
|
|
|
1
|
+
// packages/server/src/capacity/muse-host.ts
|
|
2
|
+
//
|
|
3
|
+
// NOT-270: server-owned long-lived `muse serve` host for Muse 5H/1W capacity.
|
|
4
|
+
//
|
|
5
|
+
// NOT-269 proved the only supported acquisition path: a host that actually
|
|
6
|
+
// observed the account's provider traffic (a real session's `turn/start`
|
|
7
|
+
// flow on that same host), whose `usage/changed` notifications and
|
|
8
|
+
// `usage/read` answers then carry the `window` + `weekly` pair. A fresh
|
|
9
|
+
// host is structurally unobserved — the retired one-shot poll (spawn →
|
|
10
|
+
// `usage/read` → exit) could only ever read `missing` — and
|
|
11
|
+
// `session/resume` carries no usage, so neither is used here.
|
|
12
|
+
//
|
|
13
|
+
// Ownership (exactly the NOT-269 design):
|
|
14
|
+
// - One `muse serve` host per server process, started at first demand.
|
|
15
|
+
// Concurrent readers share it: connection startup and `usage/read` are
|
|
16
|
+
// both single-flight, so there is never a second Muse execution host
|
|
17
|
+
// per concurrent request.
|
|
18
|
+
// - The host is held open only once it has observed provider traffic
|
|
19
|
+
// (state worth preserving across restarts is process-local to the
|
|
20
|
+
// host). A host that has observed nothing is released right after the
|
|
21
|
+
// read — an unobserved host holds no state, so releasing it loses
|
|
22
|
+
// nothing, and production keeps no lifetime child that can only answer
|
|
23
|
+
// `missing`. The next throttled refresh transparently respawns it.
|
|
24
|
+
// The release never fires while an execution turn is using the
|
|
25
|
+
// connection (`hasInflightExecution`): a capacity read must not SIGTERM
|
|
26
|
+
// an admitted turn.
|
|
27
|
+
// - The host launches under a server-owned XDG home
|
|
28
|
+
// (`prepareMuseServeHome`): the same worker posture as the exec lane's
|
|
29
|
+
// per-attempt settings (no MCP servers, no subagents, no workflows, no
|
|
30
|
+
// reminders) plus a symlink to the ambient login, so serve-lane turns
|
|
31
|
+
// never inherit the operator's ambient config.
|
|
32
|
+
// - `usage/changed` is ingested as soon as received; the throttled
|
|
33
|
+
// on-demand `usage/read` on the same host is the final read.
|
|
34
|
+
// - Restart/crash: usage state is process-local to the host, so a dead host
|
|
35
|
+
// restarts empty — `missing` until fresh provider traffic is observed on
|
|
36
|
+
// it. Never backfill from another host. No leaked child processes:
|
|
37
|
+
// every failure path kills the child and the next read restarts it;
|
|
38
|
+
// `shutdownMuseCapacityHost()` releases the host for clean shutdown.
|
|
39
|
+
// - Ingest is newest-`observedAtMs`-wins (`ingestMuseUsagePayload`): an
|
|
40
|
+
// older finishing read cannot clobber newer rows, and a fresh-host
|
|
41
|
+
// `missing` never overwrites or deletes a newer known pair.
|
|
42
|
+
// - Failures preserve last-good rows (`noteMuseCapacityFailure`): a failure
|
|
43
|
+
// diagnostic never sits beside valid windows, and a successful recovery
|
|
44
|
+
// clears the sentinel.
|
|
45
|
+
// - Capacity stays independent from `runtime_availability` hard-cap
|
|
46
|
+
// admission: this module never reads or writes health rows.
|
|
47
|
+
// - The free read path enforces the read-only allowlist (`initialize`,
|
|
48
|
+
// `initialized`, `usage/read`) and never starts billable work. The separate
|
|
49
|
+
// default-on fallback in `muse-probe.ts` may create a dedicated restricted
|
|
50
|
+
// host for one fixed, bounded turn after the complete pair has been absent
|
|
51
|
+
// for an hour.
|
|
52
|
+
//
|
|
53
|
+
// Execution precondition (NOT-269): only a host that observes the account's
|
|
54
|
+
// provider traffic can answer `usage/read`. Real Dealer Muse turns run
|
|
55
|
+
// through THIS host (`session/start` + `turn/start` via the execution lane
|
|
56
|
+
// below, driven by runners/muse-serve-session.ts) — that traffic is the
|
|
57
|
+
// observation, so the session-boundary refresh hook
|
|
58
|
+
// (`refreshMuseCapacityAfterSession` in coordinator/muse-spawn.ts) is the
|
|
59
|
+
// final `usage/read` that populates 5H/1W. If no genuine turn has populated
|
|
60
|
+
// a complete current pair for an hour, `muse-probe.ts` may use its own host
|
|
61
|
+
// for the bounded paid fallback. When the normal serve execution lane cannot
|
|
62
|
+
// admit a developer turn, that session still falls back to the legacy
|
|
63
|
+
// `muse exec` subprocess before any model work starts; last-good capacity
|
|
64
|
+
// rows remain preserved.
|
|
65
|
+
//
|
|
66
|
+
// PRODUCT DECISION (2026-09-26, resolved via human_action on this ticket):
|
|
67
|
+
// the runner migration is in scope for NOT-270, in this same PR, not a
|
|
68
|
+
// follow-up ticket. Facts supporting that decision:
|
|
69
|
+
// - Sandbox-network posture is a fixed constant across every existing
|
|
70
|
+
// developer/reviewer invocation today (`--sandbox-network restricted`,
|
|
71
|
+
// see runners/muse-code-args.ts) — there is no per-session variation to
|
|
72
|
+
// lose. `muse serve --sandbox-network restricted` at host startup
|
|
73
|
+
// reproduces it exactly; this is a one-line host-launch change, not a
|
|
74
|
+
// rewrite.
|
|
75
|
+
// - Approval mode is already wire-selectable per session
|
|
76
|
+
// (`SessionStartParams.approvalMode`), matching today's
|
|
77
|
+
// `--approval-mode never`.
|
|
78
|
+
// - Re-verify the NOT-177/179/181 sandbox contracts (shell/write/network
|
|
79
|
+
// restriction actually holds under `serve`) using a free
|
|
80
|
+
// `--provider echo` session/turn on the host — no live paid Meta turn is
|
|
81
|
+
// needed for this verification.
|
|
82
|
+
// - Accepted, explicit tradeoff (not an oversight): `muse serve` has no
|
|
83
|
+
// wire- or host-level equivalent to `--disable-web-tools` or
|
|
84
|
+
// `--no-foreign-personal-context` in this Muse version (1.4.0). Real
|
|
85
|
+
// turns routed through the owned host run without those two specific
|
|
86
|
+
// restrictions. Everything else (network sandbox, approval mode, write/
|
|
87
|
+
// shell restriction) is unaffected.
|
|
88
|
+
// This question was escalated twice on this ticket (product_scope_decision,
|
|
89
|
+
// resolved both times); the decision above is final for NOT-270's scope.
|
|
90
|
+
import { spawn } from "node:child_process";
|
|
91
|
+
import fs from "node:fs";
|
|
92
|
+
import os from "node:os";
|
|
93
|
+
import path from "node:path";
|
|
94
|
+
import { MUSE_CLI_ENV, resolveMuseAuthFile, resolveMuseBin } from "../cli-env.js";
|
|
95
|
+
import { buildMuseDeveloperSettings } from "../runners/muse-code-settings.js";
|
|
96
|
+
import { MSP_USAGE_CHANGED, MUSE_CLIENT_INFO, MUSE_SERVE_ARGV, assertMuseExecMethod, assertMuseReadOnlyMethod, hasMuseCredential, ingestMuseUsagePayload, museCapacityTimeoutMs, museClassifyRpcError, museEvidenceRef, museRefreshThrottleMs, museUuidv7, noteMuseCapacityFailure, parseMuseRpcLine, } from "./muse.js";
|
|
97
|
+
/**
|
|
98
|
+
* NOT-270 serve lane: the owned host must run workers under the same
|
|
99
|
+
* posture as the exec lane's per-attempt settings — no MCP servers, no
|
|
100
|
+
* subagent delegation, no workflows, no reminder child runs — so serve-lane
|
|
101
|
+
* turns never inherit the operator's ambient MCP/subagent config. Builds a
|
|
102
|
+
* server-owned config home carrying `buildMuseDeveloperSettings()` plus a
|
|
103
|
+
* symlink to the ambient login (linked, never read or copied; absent when
|
|
104
|
+
* the operator authenticates with META_API_KEY), and an isolated data home
|
|
105
|
+
* for the host's own session/state. Mirrors the exec lane's per-attempt
|
|
106
|
+
* dirs (`coordinator/muse-spawn.ts`), but process-scoped: one home per host
|
|
107
|
+
* instance, reused across restarts, removed on shutdown.
|
|
108
|
+
*/
|
|
109
|
+
export function prepareMuseServeHome(ambientAuthFile = resolveMuseAuthFile()) {
|
|
110
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-muse-serve-"));
|
|
111
|
+
const configHome = path.join(root, "config");
|
|
112
|
+
const dataHome = path.join(root, "data");
|
|
113
|
+
fs.mkdirSync(path.join(configHome, "muse"), { recursive: true, mode: 0o700 });
|
|
114
|
+
fs.mkdirSync(dataHome, { recursive: true, mode: 0o700 });
|
|
115
|
+
// mkdir's mode is masked by the umask; the home holds a login symlink.
|
|
116
|
+
fs.chmodSync(root, 0o700);
|
|
117
|
+
fs.chmodSync(configHome, 0o700);
|
|
118
|
+
fs.chmodSync(dataHome, 0o700);
|
|
119
|
+
fs.writeFileSync(path.join(configHome, "muse", "settings.json"), `${JSON.stringify(buildMuseDeveloperSettings(), null, 2)}\n`, { mode: 0o600 });
|
|
120
|
+
let authLinked = false;
|
|
121
|
+
try {
|
|
122
|
+
if (fs.existsSync(ambientAuthFile)) {
|
|
123
|
+
fs.symlinkSync(ambientAuthFile, path.join(configHome, "muse", "auth.json"));
|
|
124
|
+
authLinked = true;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
// Best-effort: the credential check in start() decides admission, and
|
|
129
|
+
// API-key auth needs no link at all.
|
|
130
|
+
}
|
|
131
|
+
return { configHome, dataHome, authLinked };
|
|
132
|
+
}
|
|
133
|
+
const KEYCHAIN_UNREADABLE_RE = /keychain item .* unreadable|OSStatus\s+-?\d+/i;
|
|
134
|
+
const AUTH_FAILURE_RE = /credential|auth|login|sign[ -]?in|\b401\b|\b403\b/i;
|
|
135
|
+
function readOnlyRequest(id, method, params) {
|
|
136
|
+
assertMuseReadOnlyMethod(method);
|
|
137
|
+
return `${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`;
|
|
138
|
+
}
|
|
139
|
+
function readOnlyNotification(method, params) {
|
|
140
|
+
assertMuseReadOnlyMethod(method);
|
|
141
|
+
return `${JSON.stringify({ jsonrpc: "2.0", method, params })}\n`;
|
|
142
|
+
}
|
|
143
|
+
function execRequestLine(id, method, params) {
|
|
144
|
+
assertMuseExecMethod(method);
|
|
145
|
+
return `${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* One long-lived connection. A new instance is created per process (plus
|
|
149
|
+
* per test reset); the module singleton below guarantees at most one live
|
|
150
|
+
* instance per process at a time.
|
|
151
|
+
*/
|
|
152
|
+
export class MuseCapacityHost {
|
|
153
|
+
opts;
|
|
154
|
+
child = null;
|
|
155
|
+
dead = true;
|
|
156
|
+
starting = null;
|
|
157
|
+
inflightRead = null;
|
|
158
|
+
pending = new Map();
|
|
159
|
+
/** Live execution-turn subscribers (turn/completed, item/*, ...). */
|
|
160
|
+
notifWaiters = new Set();
|
|
161
|
+
buffer = "";
|
|
162
|
+
stderrTail = "";
|
|
163
|
+
readSeq = 0;
|
|
164
|
+
/**
|
|
165
|
+
* True once this connection has observed provider traffic (a `usage/read`
|
|
166
|
+
* or `usage/changed` ingest that wrote rows). Only an observed host is
|
|
167
|
+
* held open — an unobserved one is released after the read. Reset on
|
|
168
|
+
* every (re)spawn: usage state is process-local, so a restarted host
|
|
169
|
+
* starts unobserved.
|
|
170
|
+
*/
|
|
171
|
+
observedOnConnection = false;
|
|
172
|
+
/** Increments on every spawn — tests use it to prove host reuse/restart. */
|
|
173
|
+
connectionEpoch = 0;
|
|
174
|
+
/**
|
|
175
|
+
* Server-owned XDG home for the host (worker settings + auth link).
|
|
176
|
+
* Built lazily on first spawn, reused across restarts, removed on
|
|
177
|
+
* shutdown.
|
|
178
|
+
*/
|
|
179
|
+
serveHome = null;
|
|
180
|
+
constructor(opts = {}) {
|
|
181
|
+
this.opts = { ...opts };
|
|
182
|
+
}
|
|
183
|
+
/** True while a live child backs this host. */
|
|
184
|
+
isConnected() {
|
|
185
|
+
return !this.dead && this.child !== null && this.child.exitCode === null;
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* True while a real execution turn (or its admission) is using the shared
|
|
189
|
+
* connection: an unanswered execution request id, or a live
|
|
190
|
+
* `turn/completed` subscriber. A capacity read must never tear the host
|
|
191
|
+
* down while this is true — on a fresh host the first turn is still
|
|
192
|
+
* unobserved, so an unconditional unobserved-host release would SIGTERM
|
|
193
|
+
* the admitted billable turn and fail it with 'no turn completion
|
|
194
|
+
* observed'.
|
|
195
|
+
*/
|
|
196
|
+
hasInflightExecution() {
|
|
197
|
+
return this.pending.size > 0 || this.notifWaiters.size > 0;
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Env for the host subprocess: the server-owned XDG home always wins over
|
|
201
|
+
* inherited ambient config (the server process itself may run under an
|
|
202
|
+
* operator XDG home), so serve-lane workers never inherit ambient MCP
|
|
203
|
+
* servers, subagents, or workflows. Only an explicit XDG override in
|
|
204
|
+
* `opts.env` (tests) still wins.
|
|
205
|
+
*/
|
|
206
|
+
mergedEnv() {
|
|
207
|
+
const home = this.ensureServeHome();
|
|
208
|
+
const env = { ...process.env, ...this.opts.env };
|
|
209
|
+
if (this.opts.env?.XDG_CONFIG_HOME === undefined)
|
|
210
|
+
env.XDG_CONFIG_HOME = home.configHome;
|
|
211
|
+
if (this.opts.env?.XDG_DATA_HOME === undefined)
|
|
212
|
+
env.XDG_DATA_HOME = home.dataHome;
|
|
213
|
+
return env;
|
|
214
|
+
}
|
|
215
|
+
/** Lazily built server-owned home; rebuilt if removed (e.g. after shutdown). */
|
|
216
|
+
ensureServeHome() {
|
|
217
|
+
const home = this.serveHome;
|
|
218
|
+
if (home) {
|
|
219
|
+
try {
|
|
220
|
+
if (fs.existsSync(path.join(home.configHome, "muse", "settings.json")))
|
|
221
|
+
return home;
|
|
222
|
+
}
|
|
223
|
+
catch {
|
|
224
|
+
// Fall through and rebuild.
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
this.serveHome = prepareMuseServeHome();
|
|
228
|
+
return this.serveHome;
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* SIGTERM the child and release it. A wedged host can ignore SIGTERM, so
|
|
232
|
+
* every kill arms a one-shot 2 s SIGKILL escalation against that exact
|
|
233
|
+
* child — the timer holds only the old handle, so it can never signal a
|
|
234
|
+
* replacement host spawned later. SIGKILL is the backstop for wedged
|
|
235
|
+
* hosts only: a host that exits on SIGTERM is never signalled again.
|
|
236
|
+
*
|
|
237
|
+
* Notification waiters resolve null here so execution turns fail fast on
|
|
238
|
+
* host death instead of hanging until their own timeout.
|
|
239
|
+
*/
|
|
240
|
+
killChild() {
|
|
241
|
+
const child = this.child;
|
|
242
|
+
this.child = null;
|
|
243
|
+
this.dead = true;
|
|
244
|
+
for (const [, p] of this.pending) {
|
|
245
|
+
try {
|
|
246
|
+
p.resolve({});
|
|
247
|
+
}
|
|
248
|
+
catch {
|
|
249
|
+
// Never throw out of teardown.
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
this.pending.clear();
|
|
253
|
+
for (const w of this.notifWaiters) {
|
|
254
|
+
clearTimeout(w.timer);
|
|
255
|
+
try {
|
|
256
|
+
w.resolve(null);
|
|
257
|
+
}
|
|
258
|
+
catch {
|
|
259
|
+
// Never throw out of teardown.
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
this.notifWaiters.clear();
|
|
263
|
+
this.buffer = "";
|
|
264
|
+
try {
|
|
265
|
+
child?.kill();
|
|
266
|
+
}
|
|
267
|
+
catch {
|
|
268
|
+
// Already exited — nothing to signal.
|
|
269
|
+
}
|
|
270
|
+
if (child && child.exitCode === null) {
|
|
271
|
+
const killer = setTimeout(() => {
|
|
272
|
+
try {
|
|
273
|
+
if (child.exitCode === null)
|
|
274
|
+
child.kill("SIGKILL");
|
|
275
|
+
}
|
|
276
|
+
catch {
|
|
277
|
+
// Already exited — nothing to signal.
|
|
278
|
+
}
|
|
279
|
+
}, 2000);
|
|
280
|
+
killer.unref?.();
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
/** True when `child` is still the live backing process for this host. */
|
|
284
|
+
isCurrentChild(child) {
|
|
285
|
+
return this.child === child && !this.dead;
|
|
286
|
+
}
|
|
287
|
+
onLine(line) {
|
|
288
|
+
const msg = parseMuseRpcLine(line);
|
|
289
|
+
if (!msg)
|
|
290
|
+
return;
|
|
291
|
+
if (typeof msg.method === "string" && msg.id === undefined) {
|
|
292
|
+
if (msg.method === MSP_USAGE_CHANGED) {
|
|
293
|
+
// Passive observation from real provider traffic on this host —
|
|
294
|
+
// ingest immediately, newest-observedAtMs-wins.
|
|
295
|
+
void ingestMuseUsagePayload(msg.params ?? null, {
|
|
296
|
+
evidenceRef: museEvidenceRef("changed"),
|
|
297
|
+
})
|
|
298
|
+
.then((outcome) => {
|
|
299
|
+
// A changed-notification that wrote rows means this host has
|
|
300
|
+
// observed traffic: it is worth holding open.
|
|
301
|
+
if (outcome.written.length > 0)
|
|
302
|
+
this.observedOnConnection = true;
|
|
303
|
+
})
|
|
304
|
+
.catch(() => {
|
|
305
|
+
// Best-effort: a failed ingest never breaks the connection.
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
// Execution-turn subscribers (turn/completed, item/*, ...). A waiter
|
|
309
|
+
// that fires is removed; waiters never see each other's messages.
|
|
310
|
+
for (const w of [...this.notifWaiters]) {
|
|
311
|
+
let match = false;
|
|
312
|
+
try {
|
|
313
|
+
match = w.predicate(msg);
|
|
314
|
+
}
|
|
315
|
+
catch {
|
|
316
|
+
match = false;
|
|
317
|
+
}
|
|
318
|
+
if (match) {
|
|
319
|
+
this.notifWaiters.delete(w);
|
|
320
|
+
clearTimeout(w.timer);
|
|
321
|
+
try {
|
|
322
|
+
w.resolve(msg);
|
|
323
|
+
}
|
|
324
|
+
catch {
|
|
325
|
+
// Never throw out of the read loop.
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
if (typeof msg.id === "string") {
|
|
332
|
+
const p = this.pending.get(msg.id);
|
|
333
|
+
if (p) {
|
|
334
|
+
this.pending.delete(msg.id);
|
|
335
|
+
p.resolve({ result: msg.result, error: msg.error });
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* NOT-270 execution lane: one JSON-RPC request for REAL Dealer work
|
|
341
|
+
* (`session/start`, `turn/start`, `turn/cancel`, `session/read` only —
|
|
342
|
+
* anything else throws before write). Multiplexed over the single shared
|
|
343
|
+
* connection by request id, so concurrent sessions share the one owned
|
|
344
|
+
* host and there is never a second Muse execution host per request.
|
|
345
|
+
* Resolves null on timeout, dead host, or unwritable stdin.
|
|
346
|
+
*/
|
|
347
|
+
async execRequest(id, method, params, timeoutMs) {
|
|
348
|
+
assertMuseExecMethod(method);
|
|
349
|
+
if (this.dead || !this.child || this.child.exitCode !== null)
|
|
350
|
+
return null;
|
|
351
|
+
return new Promise((resolve) => {
|
|
352
|
+
const timer = setTimeout(() => {
|
|
353
|
+
this.pending.delete(id);
|
|
354
|
+
resolve(null);
|
|
355
|
+
}, timeoutMs);
|
|
356
|
+
timer.unref?.();
|
|
357
|
+
this.pending.set(id, {
|
|
358
|
+
resolve: (value) => {
|
|
359
|
+
clearTimeout(timer);
|
|
360
|
+
resolve(value);
|
|
361
|
+
},
|
|
362
|
+
});
|
|
363
|
+
try {
|
|
364
|
+
this.child?.stdin?.write(execRequestLine(id, method, params));
|
|
365
|
+
}
|
|
366
|
+
catch {
|
|
367
|
+
this.pending.delete(id);
|
|
368
|
+
clearTimeout(timer);
|
|
369
|
+
resolve(null);
|
|
370
|
+
}
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
/**
|
|
374
|
+
* Subscribe to one host notification (e.g. `turn/completed` for a session).
|
|
375
|
+
* Resolves with the first matching message, null on timeout or host death.
|
|
376
|
+
* Concurrent turns each hold their own waiter on the shared connection.
|
|
377
|
+
*/
|
|
378
|
+
async waitForHostNotification(predicate, timeoutMs) {
|
|
379
|
+
if (this.dead)
|
|
380
|
+
return null;
|
|
381
|
+
return new Promise((resolve) => {
|
|
382
|
+
const waiter = {
|
|
383
|
+
predicate,
|
|
384
|
+
resolve: (msg) => {
|
|
385
|
+
clearTimeout(waiter.timer);
|
|
386
|
+
this.notifWaiters.delete(waiter);
|
|
387
|
+
resolve(msg);
|
|
388
|
+
},
|
|
389
|
+
timer: setTimeout(() => {
|
|
390
|
+
this.notifWaiters.delete(waiter);
|
|
391
|
+
resolve(null);
|
|
392
|
+
}, timeoutMs),
|
|
393
|
+
};
|
|
394
|
+
waiter.timer.unref?.();
|
|
395
|
+
this.notifWaiters.add(waiter);
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
/**
|
|
399
|
+
* Best-effort `turn/cancel` for a timed-out or aborted execution turn.
|
|
400
|
+
* Never throws; the caller owns the timeout/abort verdict regardless.
|
|
401
|
+
*/
|
|
402
|
+
async cancelExecTurn(sessionId, turnId) {
|
|
403
|
+
try {
|
|
404
|
+
if (this.dead)
|
|
405
|
+
return;
|
|
406
|
+
this.readSeq += 1;
|
|
407
|
+
await this.execRequest(`muse-cancel-${this.connectionEpoch}-${this.readSeq}-${Date.now()}`, "turn/cancel", {
|
|
408
|
+
commandId: museUuidv7(),
|
|
409
|
+
sessionId,
|
|
410
|
+
...(turnId ? { turnId } : {}),
|
|
411
|
+
}, 10_000);
|
|
412
|
+
}
|
|
413
|
+
catch {
|
|
414
|
+
// Best-effort: cancellation must never break the caller's verdict.
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
sendRequest(id, method, params, timeoutMs) {
|
|
418
|
+
return new Promise((resolve) => {
|
|
419
|
+
const timer = setTimeout(() => {
|
|
420
|
+
this.pending.delete(id);
|
|
421
|
+
resolve(null);
|
|
422
|
+
}, timeoutMs);
|
|
423
|
+
timer.unref?.();
|
|
424
|
+
this.pending.set(id, {
|
|
425
|
+
resolve: (value) => {
|
|
426
|
+
clearTimeout(timer);
|
|
427
|
+
resolve(value);
|
|
428
|
+
},
|
|
429
|
+
});
|
|
430
|
+
try {
|
|
431
|
+
this.child?.stdin?.write(readOnlyRequest(id, method, params));
|
|
432
|
+
}
|
|
433
|
+
catch {
|
|
434
|
+
this.pending.delete(id);
|
|
435
|
+
clearTimeout(timer);
|
|
436
|
+
resolve(null);
|
|
437
|
+
}
|
|
438
|
+
});
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* Start (or reuse) the connection. Single-flight across callers: a start
|
|
442
|
+
* already in flight is always joined first. (The child is assigned before
|
|
443
|
+
* the handshake completes, so checking `isConnected()` first would let a
|
|
444
|
+
* second caller send requests down a half-open connection — session/start
|
|
445
|
+
* arriving before `initialized` is rejected by the host.)
|
|
446
|
+
*/
|
|
447
|
+
async ensureStarted() {
|
|
448
|
+
if (this.starting)
|
|
449
|
+
return this.starting;
|
|
450
|
+
if (this.isConnected())
|
|
451
|
+
return true;
|
|
452
|
+
this.starting = this.start();
|
|
453
|
+
try {
|
|
454
|
+
return await this.starting;
|
|
455
|
+
}
|
|
456
|
+
finally {
|
|
457
|
+
this.starting = null;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
async start() {
|
|
461
|
+
this.killChild();
|
|
462
|
+
const timeoutMs = this.opts.timeoutMs ?? museCapacityTimeoutMs();
|
|
463
|
+
const mergedEnv = this.mergedEnv();
|
|
464
|
+
if (!hasMuseCredential(mergedEnv, this.opts.authFilePath ?? resolveMuseAuthFile())) {
|
|
465
|
+
this.startFailure = "credential_missing";
|
|
466
|
+
return false;
|
|
467
|
+
}
|
|
468
|
+
const command = this.opts.command ?? resolveMuseBin();
|
|
469
|
+
const args = this.opts.args ?? [...MUSE_SERVE_ARGV];
|
|
470
|
+
let child;
|
|
471
|
+
try {
|
|
472
|
+
child = spawn(command, args, {
|
|
473
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
474
|
+
env: { ...mergedEnv, ...MUSE_CLI_ENV },
|
|
475
|
+
});
|
|
476
|
+
}
|
|
477
|
+
catch (err) {
|
|
478
|
+
this.startFailure =
|
|
479
|
+
err?.code === "ENOENT" ? "binary_missing" : "spawn_failed";
|
|
480
|
+
return false;
|
|
481
|
+
}
|
|
482
|
+
this.child = child;
|
|
483
|
+
this.dead = false;
|
|
484
|
+
this.connectionEpoch += 1;
|
|
485
|
+
// Usage state is process-local: a (re)spawned host starts unobserved.
|
|
486
|
+
this.observedOnConnection = false;
|
|
487
|
+
this.stderrTail = "";
|
|
488
|
+
child.stderr?.on("data", (buf) => {
|
|
489
|
+
// Stderr is diagnostic-only; attribute it only while this child is live.
|
|
490
|
+
if (!this.isCurrentChild(child))
|
|
491
|
+
return;
|
|
492
|
+
this.stderrTail = `${this.stderrTail}${buf.toString()}`.slice(-2000);
|
|
493
|
+
});
|
|
494
|
+
child.stdout?.on("data", (buf) => {
|
|
495
|
+
// A previous child killed for restart may still flush output after its
|
|
496
|
+
// replacement spawns — never let a stale child feed the live buffer
|
|
497
|
+
// or tear the new connection down.
|
|
498
|
+
if (!this.isCurrentChild(child))
|
|
499
|
+
return;
|
|
500
|
+
this.buffer += buf.toString();
|
|
501
|
+
if (this.buffer.length > 10 * 1024 * 1024) {
|
|
502
|
+
this.killChild();
|
|
503
|
+
return;
|
|
504
|
+
}
|
|
505
|
+
let idx;
|
|
506
|
+
while ((idx = this.buffer.indexOf("\n")) >= 0) {
|
|
507
|
+
const line = this.buffer.slice(0, idx);
|
|
508
|
+
this.buffer = this.buffer.slice(idx + 1);
|
|
509
|
+
this.onLine(line);
|
|
510
|
+
if (this.dead)
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
513
|
+
});
|
|
514
|
+
child.on("error", (err) => {
|
|
515
|
+
if (!this.isCurrentChild(child))
|
|
516
|
+
return;
|
|
517
|
+
this.startFailure =
|
|
518
|
+
err?.code === "ENOENT" ? "binary_missing" : "spawn_failed";
|
|
519
|
+
this.killChild();
|
|
520
|
+
});
|
|
521
|
+
child.stdin?.on("error", () => {
|
|
522
|
+
if (!this.isCurrentChild(child))
|
|
523
|
+
return;
|
|
524
|
+
this.startFailure = "transport_error";
|
|
525
|
+
this.killChild();
|
|
526
|
+
});
|
|
527
|
+
child.on("close", () => {
|
|
528
|
+
// Process-local usage state dies with the host (NOT-269 restart rule).
|
|
529
|
+
// Guarded: a SIGTERMed hung child that exits after a restart spawned
|
|
530
|
+
// its replacement must not kill the new host.
|
|
531
|
+
if (!this.isCurrentChild(child))
|
|
532
|
+
return;
|
|
533
|
+
if (KEYCHAIN_UNREADABLE_RE.test(this.stderrTail)) {
|
|
534
|
+
this.startFailure = "keychain_unreadable";
|
|
535
|
+
}
|
|
536
|
+
else if (AUTH_FAILURE_RE.test(this.stderrTail)) {
|
|
537
|
+
this.startFailure = "auth_rejected";
|
|
538
|
+
}
|
|
539
|
+
else if (this.startFailure === null) {
|
|
540
|
+
this.startFailure = "host_exited";
|
|
541
|
+
}
|
|
542
|
+
this.killChild();
|
|
543
|
+
});
|
|
544
|
+
const initId = `muse-capacity-init-${this.connectionEpoch}`;
|
|
545
|
+
const init = await this.sendRequest(initId, "initialize", { clientInfo: { ...MUSE_CLIENT_INFO } }, timeoutMs);
|
|
546
|
+
if (this.dead || !init || init.error) {
|
|
547
|
+
if (init?.error) {
|
|
548
|
+
const kind = museClassifyRpcError(init.error);
|
|
549
|
+
this.startFailure =
|
|
550
|
+
kind === "auth"
|
|
551
|
+
? "auth_rejected"
|
|
552
|
+
: kind === "unsupported"
|
|
553
|
+
? "unsupported"
|
|
554
|
+
: "handshake_rejected";
|
|
555
|
+
}
|
|
556
|
+
else if (this.startFailure === null) {
|
|
557
|
+
this.startFailure = this.dead ? "host_exited" : "handshake_timeout";
|
|
558
|
+
}
|
|
559
|
+
this.killChild();
|
|
560
|
+
return false;
|
|
561
|
+
}
|
|
562
|
+
try {
|
|
563
|
+
child.stdin?.write(readOnlyNotification("initialized", {}));
|
|
564
|
+
}
|
|
565
|
+
catch {
|
|
566
|
+
this.killChild();
|
|
567
|
+
return false;
|
|
568
|
+
}
|
|
569
|
+
return true;
|
|
570
|
+
}
|
|
571
|
+
startFailure = null;
|
|
572
|
+
/**
|
|
573
|
+
* Final `usage/read` on the owned host. Single-flight: concurrent
|
|
574
|
+
* callers share one read on the one connection. Ingests newest-wins;
|
|
575
|
+
* a fresh-host `missing` preserves stored rows.
|
|
576
|
+
*/
|
|
577
|
+
async readUsage() {
|
|
578
|
+
if (this.inflightRead)
|
|
579
|
+
return this.inflightRead;
|
|
580
|
+
this.inflightRead = this.read();
|
|
581
|
+
try {
|
|
582
|
+
const outcome = await this.inflightRead;
|
|
583
|
+
if (outcome.status === "observed") {
|
|
584
|
+
this.observedOnConnection = true;
|
|
585
|
+
}
|
|
586
|
+
else if (!this.observedOnConnection && !this.hasInflightExecution()) {
|
|
587
|
+
// The host observed nothing and holds no state worth keeping:
|
|
588
|
+
// release the child instead of parking a lifetime process that
|
|
589
|
+
// can only answer `missing`. The next refresh transparently
|
|
590
|
+
// respawns it; last-good DB rows are untouched either way.
|
|
591
|
+
// Skipped while an execution turn is using the connection — the
|
|
592
|
+
// release must never SIGTERM an admitted turn (see
|
|
593
|
+
// hasInflightExecution).
|
|
594
|
+
this.killChild();
|
|
595
|
+
}
|
|
596
|
+
return outcome;
|
|
597
|
+
}
|
|
598
|
+
finally {
|
|
599
|
+
this.inflightRead = null;
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
async read() {
|
|
603
|
+
const timeoutMs = this.opts.timeoutMs ?? museCapacityTimeoutMs();
|
|
604
|
+
this.startFailure = null;
|
|
605
|
+
const started = await this.ensureStarted();
|
|
606
|
+
if (!started) {
|
|
607
|
+
// `ensureStarted` records the failure asynchronously. TypeScript does
|
|
608
|
+
// not model that mutation across the awaited call, so widen the field
|
|
609
|
+
// back to its declared type before selecting the fallback category.
|
|
610
|
+
const diagnostic = this.startFailure ?? "spawn_failed";
|
|
611
|
+
if (diagnostic === "unsupported" || diagnostic === "binary_missing") {
|
|
612
|
+
console.error(`[muse-capacity] host start failed: ${diagnostic}`);
|
|
613
|
+
await noteMuseCapacityFailure("unsupported");
|
|
614
|
+
return { status: "failure", reason: "unsupported", diagnostic };
|
|
615
|
+
}
|
|
616
|
+
// No credential, spawn failure, timeout, or bad exit before the
|
|
617
|
+
// handshake: honest `missing`, last-good rows preserved.
|
|
618
|
+
console.error(`[muse-capacity] host start failed: ${diagnostic}`);
|
|
619
|
+
await noteMuseCapacityFailure("missing");
|
|
620
|
+
return { status: "missing", diagnostic };
|
|
621
|
+
}
|
|
622
|
+
this.readSeq += 1;
|
|
623
|
+
const id = `muse-capacity-${this.connectionEpoch}-${this.readSeq}`;
|
|
624
|
+
const res = await this.sendRequest(id, "usage/read", {}, timeoutMs);
|
|
625
|
+
if (this.dead || !res) {
|
|
626
|
+
console.error("[muse-capacity] host read failed: timeout");
|
|
627
|
+
// Never tear down under an admitted turn: a capacity timeout must not
|
|
628
|
+
// become a turn failure. The turn's own timeout/cancel owns that
|
|
629
|
+
// verdict; the hung child is reaped when the turn settles or when a
|
|
630
|
+
// later read finds the connection idle.
|
|
631
|
+
if (!this.hasInflightExecution())
|
|
632
|
+
this.killChild();
|
|
633
|
+
await noteMuseCapacityFailure("missing");
|
|
634
|
+
return { status: "missing", diagnostic: "read_timeout" };
|
|
635
|
+
}
|
|
636
|
+
if (res.error) {
|
|
637
|
+
const kind = museClassifyRpcError(res.error);
|
|
638
|
+
if (kind === "auth") {
|
|
639
|
+
console.error("[muse-capacity] host read failed: unauthenticated");
|
|
640
|
+
await noteMuseCapacityFailure("missing");
|
|
641
|
+
return { status: "missing", diagnostic: "auth_rejected" };
|
|
642
|
+
}
|
|
643
|
+
if (kind === "unsupported") {
|
|
644
|
+
console.error("[muse-capacity] host read failed: unsupported");
|
|
645
|
+
await noteMuseCapacityFailure("unsupported");
|
|
646
|
+
return { status: "failure", reason: "unsupported", diagnostic: "unsupported" };
|
|
647
|
+
}
|
|
648
|
+
console.error("[muse-capacity] host read failed: malformed");
|
|
649
|
+
await noteMuseCapacityFailure("unparsable");
|
|
650
|
+
return { status: "failure", reason: "unparsable", diagnostic: "read_rejected" };
|
|
651
|
+
}
|
|
652
|
+
let payload = res.result;
|
|
653
|
+
if (payload &&
|
|
654
|
+
typeof payload === "object" &&
|
|
655
|
+
!("usage" in payload) &&
|
|
656
|
+
payload.result !== undefined) {
|
|
657
|
+
payload = payload.result;
|
|
658
|
+
}
|
|
659
|
+
const outcome = await ingestMuseUsagePayload(payload, {
|
|
660
|
+
evidenceRef: museEvidenceRef("read"),
|
|
661
|
+
});
|
|
662
|
+
if (outcome.written.length === 0) {
|
|
663
|
+
// Fresh/unobserved host: `usage` omitted — honest `missing`, never a
|
|
664
|
+
// write, so a newer known pair is never overwritten or deleted.
|
|
665
|
+
await noteMuseCapacityFailure("missing");
|
|
666
|
+
return { status: "missing", diagnostic: "unobserved" };
|
|
667
|
+
}
|
|
668
|
+
return { status: "observed", written: outcome.written, skippedStale: outcome.skippedStale };
|
|
669
|
+
}
|
|
670
|
+
/**
|
|
671
|
+
* Clean shutdown: release the child so no host process leaks.
|
|
672
|
+
* Graceful-first — a host that exits on SIGTERM is never signalled again
|
|
673
|
+
* (SIGKILL is the backstop for wedged hosts only). The close is awaited
|
|
674
|
+
* for a bounded grace window and a still-alive host is SIGKILLed before
|
|
675
|
+
* returning, so teardown never leaks the child and never depends on a
|
|
676
|
+
* timer the exiting event loop may never run.
|
|
677
|
+
*/
|
|
678
|
+
async shutdown() {
|
|
679
|
+
const child = this.child;
|
|
680
|
+
this.killChild();
|
|
681
|
+
this.starting = null;
|
|
682
|
+
this.inflightRead = null;
|
|
683
|
+
// The serve home is per-instance: remove it so restarts and test resets
|
|
684
|
+
// leave no litter. A later ensureStarted() transparently rebuilds it.
|
|
685
|
+
const home = this.serveHome;
|
|
686
|
+
this.serveHome = null;
|
|
687
|
+
if (home) {
|
|
688
|
+
try {
|
|
689
|
+
fs.rmSync(path.dirname(home.configHome), { recursive: true, force: true });
|
|
690
|
+
}
|
|
691
|
+
catch {
|
|
692
|
+
// Best-effort cleanup only.
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
if (!child || child.exitCode !== null)
|
|
696
|
+
return;
|
|
697
|
+
const closed = await new Promise((resolve) => {
|
|
698
|
+
if (child.exitCode !== null) {
|
|
699
|
+
resolve(true);
|
|
700
|
+
return;
|
|
701
|
+
}
|
|
702
|
+
const timer = setTimeout(() => resolve(false), 2000);
|
|
703
|
+
timer.unref?.();
|
|
704
|
+
child.once("close", () => {
|
|
705
|
+
clearTimeout(timer);
|
|
706
|
+
resolve(true);
|
|
707
|
+
});
|
|
708
|
+
});
|
|
709
|
+
if (!closed) {
|
|
710
|
+
try {
|
|
711
|
+
if (child.exitCode === null)
|
|
712
|
+
child.kill("SIGKILL");
|
|
713
|
+
}
|
|
714
|
+
catch {
|
|
715
|
+
// Already exited — nothing to signal.
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
// ---------------------------------------------------------------------------
|
|
721
|
+
// Process singleton + production refresh
|
|
722
|
+
// ---------------------------------------------------------------------------
|
|
723
|
+
let sharedHost = null;
|
|
724
|
+
let lastMuseHostRefreshMs = 0;
|
|
725
|
+
/** The server-owned host (created on first use). */
|
|
726
|
+
export function getMuseCapacityHost(opts = {}) {
|
|
727
|
+
if (!sharedHost)
|
|
728
|
+
sharedHost = new MuseCapacityHost(opts);
|
|
729
|
+
return sharedHost;
|
|
730
|
+
}
|
|
731
|
+
/** Test helper — shut the shared host down so the next use starts fresh. */
|
|
732
|
+
export async function resetMuseCapacityHostForTests() {
|
|
733
|
+
lastMuseHostRefreshMs = 0;
|
|
734
|
+
const host = sharedHost;
|
|
735
|
+
sharedHost = null;
|
|
736
|
+
await host?.shutdown();
|
|
737
|
+
}
|
|
738
|
+
/** Clean shutdown for server teardown. */
|
|
739
|
+
export async function shutdownMuseCapacityHost() {
|
|
740
|
+
const host = sharedHost;
|
|
741
|
+
sharedHost = null;
|
|
742
|
+
await host?.shutdown();
|
|
743
|
+
}
|
|
744
|
+
/** Test helper — reset the refresh throttle so the next refresh runs. */
|
|
745
|
+
export function resetMuseCapacityRefreshState() {
|
|
746
|
+
lastMuseHostRefreshMs = 0;
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
749
|
+
* Bounded refresh through the owned host: one final `usage/read` on the
|
|
750
|
+
* connection that observed the account's provider traffic, ingested
|
|
751
|
+
* newest-`observedAtMs`-wins. Reuses the shared snapshot path; failures
|
|
752
|
+
* preserve last-good rows as N/A, never as health rows.
|
|
753
|
+
*/
|
|
754
|
+
export async function refreshMuseCapacityFromHost(opts = {}) {
|
|
755
|
+
const { getRuntimeCapacitySnapshot } = await import("./service.js");
|
|
756
|
+
const { nowMs, ...hostOpts } = opts;
|
|
757
|
+
const host = getMuseCapacityHost(hostOpts);
|
|
758
|
+
await host.readUsage();
|
|
759
|
+
return getRuntimeCapacitySnapshot(nowMs ?? Date.now());
|
|
760
|
+
}
|
|
761
|
+
/**
|
|
762
|
+
* NOT-270 production trigger for Muse capacity: throttled, bounded,
|
|
763
|
+
* best-effort. Returns the refreshed snapshot, or null when
|
|
764
|
+
* throttled/disabled. Never throws — callers (routes) serve the
|
|
765
|
+
* last-known snapshot on null.
|
|
766
|
+
*/
|
|
767
|
+
export async function maybeRefreshMuseCapacityFromHost(opts = {}, nowMs = Date.now()) {
|
|
768
|
+
const throttle = museRefreshThrottleMs();
|
|
769
|
+
if (!Number.isFinite(throttle) || nowMs - lastMuseHostRefreshMs < throttle)
|
|
770
|
+
return null;
|
|
771
|
+
lastMuseHostRefreshMs = nowMs;
|
|
772
|
+
try {
|
|
773
|
+
return await refreshMuseCapacityFromHost({ ...opts, nowMs });
|
|
774
|
+
}
|
|
775
|
+
catch {
|
|
776
|
+
return null;
|
|
777
|
+
}
|
|
778
|
+
}
|