agent-dealer 1.2.3 → 1.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle/server/dist/adapters/agent-health.js +25 -2
- package/bundle/server/dist/adapters/agent-health.test.js +55 -2
- package/bundle/server/dist/adapters/github.js +110 -12
- package/bundle/server/dist/adapters/github.test.js +274 -3
- package/bundle/server/dist/adapters/muse-capability.js +330 -0
- package/bundle/server/dist/adapters/muse-capability.test.js +378 -0
- package/bundle/server/dist/capacity/claude-local-cache.js +222 -63
- package/bundle/server/dist/capacity/claude-local-cache.test.js +179 -30
- package/bundle/server/dist/capacity/muse-host.js +111 -23
- package/bundle/server/dist/capacity/muse-host.test.js +125 -1
- package/bundle/server/dist/capacity/muse-lifecycle.test.js +2 -3
- package/bundle/server/dist/capacity/muse-probe.js +250 -0
- package/bundle/server/dist/capacity/muse-probe.test.js +183 -0
- package/bundle/server/dist/capacity/muse.js +16 -6
- package/bundle/server/dist/coordinator/admission.test.js +85 -0
- package/bundle/server/dist/coordinator/commands.js +35 -8
- package/bundle/server/dist/coordinator/developer-effect.js +40 -6
- package/bundle/server/dist/coordinator/developer-effect.test.js +121 -12
- package/bundle/server/dist/coordinator/human-resolution.js +31 -1
- package/bundle/server/dist/coordinator/muse-spawn.js +12 -2
- package/bundle/server/dist/coordinator/prompts.js +15 -0
- package/bundle/server/dist/coordinator/prompts.test.js +20 -0
- package/bundle/server/dist/coordinator/spawn.js +7 -0
- package/bundle/server/dist/coordinator/worktree-cwd-guard.js +48 -0
- package/bundle/server/dist/coordinator/worktree-cwd-guard.test.js +83 -0
- package/bundle/server/dist/routes/human-actions.js +10 -1
- package/bundle/server/dist/routes/human-actions.test.js +117 -1
- package/bundle/server/dist/routes/index.js +17 -14
- package/bundle/server/dist/routes/runtime-capacity.test.js +11 -3
- package/bundle/server/dist/runners/muse-serve-session.js +7 -6
- package/bundle/server/package.json +2 -2
- package/bundle/server/static-ui/assets/{index-CYqRh_-S.css → index-BII-LgB8.css} +1 -1
- package/bundle/server/static-ui/assets/index-Bq8wWpZm.js +60 -0
- package/bundle/server/static-ui/index.html +2 -2
- package/bundle/shared/dist/agents.d.ts +15 -15
- package/bundle/shared/dist/agents.js +6 -0
- package/bundle/shared/dist/index.d.ts +7 -7
- package/bundle/shared/package.json +1 -1
- package/dist/doctor.d.ts +22 -3
- package/dist/doctor.js +46 -12
- package/dist/doctor.test.js +41 -10
- package/package.json +1 -1
- package/bundle/server/static-ui/assets/index-UO4lHZw4.js +0 -60
|
@@ -21,7 +21,7 @@ import fs from "node:fs";
|
|
|
21
21
|
import os from "node:os";
|
|
22
22
|
import path from "node:path";
|
|
23
23
|
process.env.AGENT_DEALER_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-muse-host-"));
|
|
24
|
-
import { getMuseCapacityHost, maybeRefreshMuseCapacityFromHost, MuseCapacityHost, prepareMuseServeHome, refreshMuseCapacityFromHost, resetMuseCapacityHostForTests, resetMuseCapacityRefreshState, shutdownMuseCapacityHost, } from "./muse-host.js";
|
|
24
|
+
import { getMuseCapacityHost, maybeRefreshMuseCapacityFromHost, MuseCapacityHost, prepareMuseServeHome, refreshMuseCapacityFromHost, resetMuseCapacityHostForTests, resetMuseCapacityRefreshState, resolveMuseHostCwd, shutdownMuseCapacityHost, } from "./muse-host.js";
|
|
25
25
|
import { ingestMuseUsagePayload, museRefreshThrottleMs, noteMuseCapacityFailure, } from "./muse.js";
|
|
26
26
|
const { migrate } = await import("../db/index.js");
|
|
27
27
|
const { clearAllCapacitySnapshots, listCapacitySnapshots } = await import("../repository/runtime-capacity.js");
|
|
@@ -342,6 +342,66 @@ test("a stale child's late close cannot kill its replacement", async () => {
|
|
|
342
342
|
await host.shutdown();
|
|
343
343
|
}
|
|
344
344
|
});
|
|
345
|
+
// NOT-277: developer work is only served on a host known to run the capability-checked build.
|
|
346
|
+
function versionedHost(versions) {
|
|
347
|
+
let i = 0;
|
|
348
|
+
const readVersion = async () => versions[Math.min(i++, versions.length - 1)] ?? null;
|
|
349
|
+
return new MuseCapacityHost({ ...hostOpts("persistent-full"), readVersion });
|
|
350
|
+
}
|
|
351
|
+
test("the host records the version it was spawned on; a matching host is reused", async () => {
|
|
352
|
+
const host = versionedHost(["1.4.0-R4161.1"]);
|
|
353
|
+
try {
|
|
354
|
+
assert.equal(await host.ensureStartedOnVersion("1.4.0-R4161.1"), true);
|
|
355
|
+
assert.equal(host.runningVersion, "1.4.0-R4161.1");
|
|
356
|
+
assert.equal(await host.ensureStartedOnVersion("1.4.0-R4161.1"), true);
|
|
357
|
+
assert.equal(host.connectionEpoch, 1, "no restart when the host already runs the version");
|
|
358
|
+
}
|
|
359
|
+
finally {
|
|
360
|
+
await host.shutdown();
|
|
361
|
+
}
|
|
362
|
+
});
|
|
363
|
+
test("an idle host on an older build is restarted onto the confirmed version", async () => {
|
|
364
|
+
// Spawned on B (before + after the handshake), then the binary is C for the restart.
|
|
365
|
+
const host = versionedHost(["1.4.0-R4161.1", "1.4.0-R4161.1", "1.4.0-R4302.1"]);
|
|
366
|
+
try {
|
|
367
|
+
assert.ok(await host.ensureStarted());
|
|
368
|
+
assert.equal(host.runningVersion, "1.4.0-R4161.1");
|
|
369
|
+
assert.equal(await host.ensureStartedOnVersion("1.4.0-R4302.1"), true);
|
|
370
|
+
assert.equal(host.connectionEpoch, 2, "restarted onto the checked build");
|
|
371
|
+
assert.equal(host.runningVersion, "1.4.0-R4302.1");
|
|
372
|
+
}
|
|
373
|
+
finally {
|
|
374
|
+
await host.shutdown();
|
|
375
|
+
}
|
|
376
|
+
});
|
|
377
|
+
test("a busy host on an older build is left alone and not used", async () => {
|
|
378
|
+
const host = versionedHost(["1.4.0-R4161.1", "1.4.0-R4161.1", "1.4.0-R4302.1"]);
|
|
379
|
+
try {
|
|
380
|
+
assert.ok(await host.ensureStarted());
|
|
381
|
+
// An admitted turn on the shared connection: never torn down under it.
|
|
382
|
+
const waiting = host.waitForHostNotification(() => false, 5_000);
|
|
383
|
+
assert.equal(await host.ensureStartedOnVersion("1.4.0-R4302.1"), false);
|
|
384
|
+
assert.equal(host.connectionEpoch, 1);
|
|
385
|
+
assert.ok(host.isConnected());
|
|
386
|
+
await host.shutdown();
|
|
387
|
+
await waiting;
|
|
388
|
+
}
|
|
389
|
+
finally {
|
|
390
|
+
await host.shutdown();
|
|
391
|
+
}
|
|
392
|
+
});
|
|
393
|
+
test("a binary that changed across the spawn leaves the host's version unknown (never served)", async () => {
|
|
394
|
+
// Changes across the first spawn and again across the restart.
|
|
395
|
+
const host = versionedHost(["1.4.0-R4161.1", "1.4.0-R4302.1", "1.4.0-R4302.1", "1.5.0-R5000.1"]);
|
|
396
|
+
try {
|
|
397
|
+
assert.equal(await host.ensureStartedOnVersion("1.4.0-R4302.1"), false);
|
|
398
|
+
assert.equal(host.connectionEpoch, 2);
|
|
399
|
+
assert.equal(host.runningVersion, null);
|
|
400
|
+
}
|
|
401
|
+
finally {
|
|
402
|
+
await host.shutdown();
|
|
403
|
+
}
|
|
404
|
+
});
|
|
345
405
|
test("clean shutdown releases the child process", async () => {
|
|
346
406
|
const host = new MuseCapacityHost(hostOpts("persistent-full"));
|
|
347
407
|
assert.equal((await host.readUsage()).status, "observed");
|
|
@@ -458,6 +518,70 @@ test("prepareMuseServeHome links the ambient login, or records API-key auth", as
|
|
|
458
518
|
}
|
|
459
519
|
fs.rmSync(ambientRoot, { recursive: true, force: true });
|
|
460
520
|
});
|
|
521
|
+
test("NOT-273: the serve child spawns with an explicit cwd under the managed data dir", async () => {
|
|
522
|
+
const { EventEmitter } = await import("node:events");
|
|
523
|
+
const { getDataDir } = await import("../db/index.js");
|
|
524
|
+
const seen = [];
|
|
525
|
+
const hangingChild = () => {
|
|
526
|
+
const events = new EventEmitter();
|
|
527
|
+
const fake = {
|
|
528
|
+
stdin: { write: () => true, on: () => ({}) },
|
|
529
|
+
stdout: { on: () => ({}) },
|
|
530
|
+
stderr: { on: () => ({}) },
|
|
531
|
+
on: events.on.bind(events),
|
|
532
|
+
// The handshake never answers, but a kill() must still settle the
|
|
533
|
+
// child like a real process would — otherwise shutdown()'s teardown
|
|
534
|
+
// has nothing but its 2s unref'd SIGKILL-escalation timer to resolve
|
|
535
|
+
// on, which races the test runner's own event-loop bookkeeping and
|
|
536
|
+
// can cancel the test outright (see the child-ticket playbook gotcha:
|
|
537
|
+
// never await an infinite fake handler).
|
|
538
|
+
kill: () => {
|
|
539
|
+
fake.exitCode = 0;
|
|
540
|
+
queueMicrotask(() => events.emit("close"));
|
|
541
|
+
return true;
|
|
542
|
+
},
|
|
543
|
+
exitCode: null,
|
|
544
|
+
};
|
|
545
|
+
return fake;
|
|
546
|
+
};
|
|
547
|
+
const host = new MuseCapacityHost({
|
|
548
|
+
command: process.execPath,
|
|
549
|
+
args: [FAKE],
|
|
550
|
+
env: { META_API_KEY: "test-fake-key" },
|
|
551
|
+
// Handshake never answers — start() still records the exact spawn options.
|
|
552
|
+
timeoutMs: 100,
|
|
553
|
+
spawnImpl: ((command, args, options) => {
|
|
554
|
+
seen.push({ command, args, options });
|
|
555
|
+
return hangingChild();
|
|
556
|
+
}),
|
|
557
|
+
});
|
|
558
|
+
// This test's fake spawn has no real child process, so nothing else keeps
|
|
559
|
+
// the event loop busy while the production code's handshake/kill timers
|
|
560
|
+
// (deliberately unref'd — a real server must never block on them) are
|
|
561
|
+
// pending. With zero other ref'd handles, some Node versions' test runner
|
|
562
|
+
// treats the loop as "drained" before those timers fire and cancels the
|
|
563
|
+
// test outright (cancelledByParent). A trivial ref'd keepalive avoids the
|
|
564
|
+
// race without touching the production unref'd-timer behavior being tested.
|
|
565
|
+
const keepalive = setInterval(() => { }, 10);
|
|
566
|
+
try {
|
|
567
|
+
// Missing (handshake timeout), not a spawn failure — the spawn happened.
|
|
568
|
+
assert.equal((await host.readUsage()).status, "missing");
|
|
569
|
+
assert.equal(seen.length, 1, "exactly one serve spawn");
|
|
570
|
+
const cwd = seen[0].options.cwd;
|
|
571
|
+
assert.equal(typeof cwd, "string", "spawn options include an explicit cwd");
|
|
572
|
+
const dir = cwd;
|
|
573
|
+
assert.ok(path.isAbsolute(dir), `cwd is absolute, got ${dir}`);
|
|
574
|
+
assert.notEqual(path.resolve(dir), path.resolve(process.cwd()), "cwd is not the ambient process cwd");
|
|
575
|
+
assert.ok(dir.startsWith(`${getDataDir()}${path.sep}`), `cwd lives under the managed data dir, got ${dir}`);
|
|
576
|
+
assert.equal(path.basename(dir), "muse-host-home");
|
|
577
|
+
assert.ok(fs.existsSync(dir), "the dedicated cwd is created if missing");
|
|
578
|
+
assert.equal(resolveMuseHostCwd(), dir, "default cwd is the dedicated data-dir home");
|
|
579
|
+
}
|
|
580
|
+
finally {
|
|
581
|
+
await host.shutdown();
|
|
582
|
+
clearInterval(keepalive);
|
|
583
|
+
}
|
|
584
|
+
});
|
|
461
585
|
test("no credential short-circuits to missing without spawning", async () => {
|
|
462
586
|
const noAuthDir = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-muse-host-noauth-"));
|
|
463
587
|
const env = { ...process.env, FAKE_MSP_MODE: "persistent-full" };
|
|
@@ -14,8 +14,7 @@
|
|
|
14
14
|
// - the first observation arrives as a `usage/changed` notification;
|
|
15
15
|
// - a fresh/unobserved host reads `missing` (restart is fresh — the client
|
|
16
16
|
// holds no cross-process usage state);
|
|
17
|
-
// - resume/turn methods are refused before write
|
|
18
|
-
// manufacture a billable call.
|
|
17
|
+
// - resume/turn methods are refused before write on the free adapter path.
|
|
19
18
|
import { test } from "node:test";
|
|
20
19
|
import assert from "node:assert/strict";
|
|
21
20
|
import { assertMuseReadOnlyMethod, museUsageToReadings, readMuseCapacity, requestMuseUsage, } from "./muse.js";
|
|
@@ -90,7 +89,7 @@ test("lifecycle: a fresh host with no observation reads missing — twice", asyn
|
|
|
90
89
|
assert.equal(result.unavailable[0].reason, "missing");
|
|
91
90
|
}
|
|
92
91
|
});
|
|
93
|
-
test("lifecycle:
|
|
92
|
+
test("lifecycle: the free read adapter cannot manufacture a session, turn, or prompt", () => {
|
|
94
93
|
for (const m of ["session/start", "session/resume", "session/prompt", "turn/start", "exec"]) {
|
|
95
94
|
assert.throws(() => assertMuseReadOnlyMethod(m), /refusing non-read method/);
|
|
96
95
|
}
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
// packages/server/src/capacity/muse-probe.ts
|
|
2
|
+
//
|
|
3
|
+
// Reliable Muse 5H/1W fallback. Muse's stable protocol only exposes
|
|
4
|
+
// subscription capacity after the same `muse serve` process has observed
|
|
5
|
+
// real provider traffic. A read-only poll, session resume, or exec log cannot
|
|
6
|
+
// populate it (NOT-269). When the free sources have no complete current pair
|
|
7
|
+
// for one hour, this module runs one tiny bounded turn on a dedicated owned
|
|
8
|
+
// host, then performs `usage/read` on that exact host.
|
|
9
|
+
//
|
|
10
|
+
// The fallback is default-on and can be disabled with
|
|
11
|
+
// `AGENT_DEALER_MUSE_CAPACITY_REFRESH=off`. It is single-flight, attempted at
|
|
12
|
+
// most once per hour, and backs off 1h -> 2h -> 4h -> 8h after failures. The
|
|
13
|
+
// host disables shell and workspace writes and keeps restricted network;
|
|
14
|
+
// Muse 1.4's serve protocol has no per-turn max-step or disable-web-tools
|
|
15
|
+
// switch, so the fixed one-word prompt plus wall-clock cancellation is the
|
|
16
|
+
// tightest supported bound. No transcript, prompt, credential, or account
|
|
17
|
+
// payload is logged.
|
|
18
|
+
import fs from "node:fs";
|
|
19
|
+
import os from "node:os";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import { MUSE_CODE_CONTRIBUTOR_MODEL } from "@agent-dealer/shared";
|
|
22
|
+
import { getDataDir } from "../db/index.js";
|
|
23
|
+
import { listCapacitySnapshots } from "../repository/runtime-capacity.js";
|
|
24
|
+
import { runMuseServeTurn } from "../runners/muse-serve-session.js";
|
|
25
|
+
import { configuredCapacityRuntimes } from "./service.js";
|
|
26
|
+
import { maybeRefreshMuseCapacityFromHost, MuseCapacityHost, } from "./muse-host.js";
|
|
27
|
+
import { MUSE_RUNTIME } from "./muse.js";
|
|
28
|
+
export const MUSE_CAPACITY_REFRESH_ENV = "AGENT_DEALER_MUSE_CAPACITY_REFRESH";
|
|
29
|
+
export const MUSE_CAPACITY_REFRESH_PAID_VALUE = "paid-after-1h";
|
|
30
|
+
export const MUSE_CAPACITY_REFRESH_OFF_VALUE = "off";
|
|
31
|
+
export const MUSE_CAPACITY_PROBE_TIMEOUT_ENV = "AGENT_DEALER_MUSE_CAPACITY_PROBE_TIMEOUT_MS";
|
|
32
|
+
export const MUSE_CAPACITY_PROBE_TIMEOUT_MS_DEFAULT = 60_000;
|
|
33
|
+
export const MUSE_CAPACITY_PROBE_STALE_AFTER_MS = 60 * 60 * 1000;
|
|
34
|
+
export const MUSE_CAPACITY_PROBE_ATTEMPT_COOLDOWN_MS = 60 * 60 * 1000;
|
|
35
|
+
export const MUSE_CAPACITY_PROBE_BACKOFF_CAP_MS = 8 * 60 * 60 * 1000;
|
|
36
|
+
export const MUSE_CAPACITY_PROBE_PROMPT = "Reply with exactly: OK";
|
|
37
|
+
export const MUSE_CAPACITY_PROBE_SERVE_ARGV = [
|
|
38
|
+
"serve",
|
|
39
|
+
"--sandbox-network",
|
|
40
|
+
"restricted",
|
|
41
|
+
"--disable-write",
|
|
42
|
+
"--disable-shell",
|
|
43
|
+
];
|
|
44
|
+
export function isMusePaidFallbackEnabled() {
|
|
45
|
+
const setting = process.env[MUSE_CAPACITY_REFRESH_ENV];
|
|
46
|
+
if (setting === MUSE_CAPACITY_REFRESH_PAID_VALUE)
|
|
47
|
+
return true;
|
|
48
|
+
if (setting !== undefined && setting !== "")
|
|
49
|
+
return false;
|
|
50
|
+
// Backward compatibility: this older switch used to disable every Muse
|
|
51
|
+
// capacity refresh. Do not turn an existing no-refresh deployment into a
|
|
52
|
+
// spender on upgrade. An explicit paid-after-1h above may override it.
|
|
53
|
+
if (process.env.AGENT_DEALER_MUSE_CAPACITY_REFRESH_MS === "off")
|
|
54
|
+
return false;
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
export function museCapacityProbeTimeoutMs() {
|
|
58
|
+
const raw = process.env[MUSE_CAPACITY_PROBE_TIMEOUT_ENV];
|
|
59
|
+
if (raw !== undefined && raw !== "") {
|
|
60
|
+
const n = Number(raw);
|
|
61
|
+
if (Number.isFinite(n) && n > 0)
|
|
62
|
+
return n;
|
|
63
|
+
}
|
|
64
|
+
return MUSE_CAPACITY_PROBE_TIMEOUT_MS_DEFAULT;
|
|
65
|
+
}
|
|
66
|
+
function validRoleObservedAtMs(role, nowMs) {
|
|
67
|
+
const row = listCapacitySnapshots(MUSE_RUNTIME).find((w) => w.criticalRole === role);
|
|
68
|
+
if (!row || row.remainingPercent === null || row.unavailableReason !== null || row.source === "unavailable") {
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
const observedMs = Date.parse(row.observedAt);
|
|
72
|
+
if (!Number.isFinite(observedMs) || observedMs > nowMs)
|
|
73
|
+
return null;
|
|
74
|
+
if (row.resetAt !== null) {
|
|
75
|
+
const resetMs = Date.parse(row.resetAt);
|
|
76
|
+
if (!Number.isFinite(resetMs) || resetMs <= nowMs)
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
return observedMs;
|
|
80
|
+
}
|
|
81
|
+
/** Both halves must be valid and under one hour old. A fresh 5H window may
|
|
82
|
+
* never hide a missing/stale 1W sibling (or vice versa). */
|
|
83
|
+
export function hasFreshMuseCapacityPair(nowMs = Date.now()) {
|
|
84
|
+
const fiveHour = validRoleObservedAtMs("five_hour", nowMs);
|
|
85
|
+
const weekly = validRoleObservedAtMs("weekly", nowMs);
|
|
86
|
+
return (fiveHour !== null &&
|
|
87
|
+
weekly !== null &&
|
|
88
|
+
nowMs - fiveHour < MUSE_CAPACITY_PROBE_STALE_AFTER_MS &&
|
|
89
|
+
nowMs - weekly < MUSE_CAPACITY_PROBE_STALE_AFTER_MS);
|
|
90
|
+
}
|
|
91
|
+
function museProbeDiagnosticLogPath() {
|
|
92
|
+
return path.join(getDataDir(), "capacity", "muse-probe.log");
|
|
93
|
+
}
|
|
94
|
+
function appendMuseProbeDiagnostic(nowMs, result) {
|
|
95
|
+
try {
|
|
96
|
+
const file = museProbeDiagnosticLogPath();
|
|
97
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
98
|
+
fs.appendFileSync(file, `${JSON.stringify({
|
|
99
|
+
ts: new Date(nowMs).toISOString(),
|
|
100
|
+
event: "muse_capacity_probe",
|
|
101
|
+
trigger: "stale_60m",
|
|
102
|
+
model: MUSE_CODE_CONTRIBUTOR_MODEL,
|
|
103
|
+
...result,
|
|
104
|
+
})}\n`);
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
// Diagnostics are advisory; capacity refresh must never fail on logging.
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
/** One bounded real turn followed by a read on the same restricted host. */
|
|
111
|
+
export async function runMuseCapacityProbe(nowMs = Date.now(), opts = {}) {
|
|
112
|
+
const startedRealMs = Date.now();
|
|
113
|
+
const timeoutMs = opts.timeoutMs ?? museCapacityProbeTimeoutMs();
|
|
114
|
+
const ownsHost = opts.host === undefined;
|
|
115
|
+
const host = opts.host ??
|
|
116
|
+
new MuseCapacityHost({
|
|
117
|
+
...opts.hostOptions,
|
|
118
|
+
args: opts.hostOptions?.args ?? [...MUSE_CAPACITY_PROBE_SERVE_ARGV],
|
|
119
|
+
timeoutMs: Math.min(opts.hostOptions?.timeoutMs ?? timeoutMs, timeoutMs),
|
|
120
|
+
});
|
|
121
|
+
let result;
|
|
122
|
+
try {
|
|
123
|
+
const turn = await runMuseServeTurn({
|
|
124
|
+
host,
|
|
125
|
+
prompt: MUSE_CAPACITY_PROBE_PROMPT,
|
|
126
|
+
model: MUSE_CODE_CONTRIBUTOR_MODEL,
|
|
127
|
+
cwd: os.tmpdir(),
|
|
128
|
+
timeoutMs,
|
|
129
|
+
rpcTimeoutMs: Math.min(15_000, timeoutMs),
|
|
130
|
+
});
|
|
131
|
+
if (!turn.admitted) {
|
|
132
|
+
result = {
|
|
133
|
+
ok: false,
|
|
134
|
+
admitted: false,
|
|
135
|
+
terminal: null,
|
|
136
|
+
failureKind: "unadmitted",
|
|
137
|
+
windowsUpdated: [],
|
|
138
|
+
durationMs: Date.now() - startedRealMs,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
await host.readUsage();
|
|
143
|
+
const checkNowMs = Math.max(nowMs, Date.now());
|
|
144
|
+
const ok = hasFreshMuseCapacityPair(checkNowMs);
|
|
145
|
+
result = {
|
|
146
|
+
ok,
|
|
147
|
+
admitted: true,
|
|
148
|
+
terminal: turn.terminal,
|
|
149
|
+
failureKind: ok
|
|
150
|
+
? null
|
|
151
|
+
: turn.timedOut
|
|
152
|
+
? "timeout"
|
|
153
|
+
: turn.failure !== null
|
|
154
|
+
? "turn_failed"
|
|
155
|
+
: "no_windows",
|
|
156
|
+
windowsUpdated: ok ? ["rolling_all_models", "weekly_all_models"] : [],
|
|
157
|
+
durationMs: Date.now() - startedRealMs,
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
result = {
|
|
163
|
+
ok: false,
|
|
164
|
+
admitted: false,
|
|
165
|
+
terminal: null,
|
|
166
|
+
failureKind: "exception",
|
|
167
|
+
windowsUpdated: [],
|
|
168
|
+
durationMs: Date.now() - startedRealMs,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
finally {
|
|
172
|
+
if (ownsHost)
|
|
173
|
+
await host.shutdown().catch(() => undefined);
|
|
174
|
+
}
|
|
175
|
+
appendMuseProbeDiagnostic(nowMs, result);
|
|
176
|
+
const outcome = result.ok ? "ok" : `failed: ${result.failureKind ?? "unknown"}`;
|
|
177
|
+
console.error(`[muse-capacity] fallback ${outcome}`);
|
|
178
|
+
return result;
|
|
179
|
+
}
|
|
180
|
+
let museProbeInFlight = null;
|
|
181
|
+
let lastMuseProbeAttemptMs = 0;
|
|
182
|
+
let consecutiveMuseProbeFailures = 0;
|
|
183
|
+
function museProbeCooldownMs() {
|
|
184
|
+
const shift = Math.min(consecutiveMuseProbeFailures, 3);
|
|
185
|
+
return Math.min(MUSE_CAPACITY_PROBE_ATTEMPT_COOLDOWN_MS * 2 ** shift, MUSE_CAPACITY_PROBE_BACKOFF_CAP_MS);
|
|
186
|
+
}
|
|
187
|
+
export function resetMuseCapacityProbeStateForTests() {
|
|
188
|
+
museProbeInFlight = null;
|
|
189
|
+
lastMuseProbeAttemptMs = 0;
|
|
190
|
+
consecutiveMuseProbeFailures = 0;
|
|
191
|
+
}
|
|
192
|
+
export async function maybeProbeMuseCapacity(nowMs = Date.now(), opts = {}) {
|
|
193
|
+
try {
|
|
194
|
+
if (!isMusePaidFallbackEnabled())
|
|
195
|
+
return { probed: false, reason: "disabled" };
|
|
196
|
+
if (!configuredCapacityRuntimes().includes(MUSE_RUNTIME)) {
|
|
197
|
+
return { probed: false, reason: "unconfigured" };
|
|
198
|
+
}
|
|
199
|
+
if (hasFreshMuseCapacityPair(nowMs))
|
|
200
|
+
return { probed: false, reason: "fresh" };
|
|
201
|
+
if (museProbeInFlight) {
|
|
202
|
+
const shared = await museProbeInFlight;
|
|
203
|
+
return {
|
|
204
|
+
probed: true,
|
|
205
|
+
reason: "shared",
|
|
206
|
+
ok: shared.ok,
|
|
207
|
+
failureKind: shared.failureKind,
|
|
208
|
+
windowsUpdated: shared.windowsUpdated,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
if (nowMs - lastMuseProbeAttemptMs < museProbeCooldownMs()) {
|
|
212
|
+
return { probed: false, reason: "backoff" };
|
|
213
|
+
}
|
|
214
|
+
lastMuseProbeAttemptMs = nowMs;
|
|
215
|
+
const run = runMuseCapacityProbe(nowMs, opts);
|
|
216
|
+
museProbeInFlight = run;
|
|
217
|
+
try {
|
|
218
|
+
const result = await run;
|
|
219
|
+
consecutiveMuseProbeFailures = result.ok ? 0 : consecutiveMuseProbeFailures + 1;
|
|
220
|
+
return {
|
|
221
|
+
probed: true,
|
|
222
|
+
reason: "completed",
|
|
223
|
+
ok: result.ok,
|
|
224
|
+
failureKind: result.failureKind,
|
|
225
|
+
windowsUpdated: result.windowsUpdated,
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
finally {
|
|
229
|
+
if (museProbeInFlight === run)
|
|
230
|
+
museProbeInFlight = null;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
return { probed: false, reason: "error" };
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
/** Free read first; only a still-missing/stale complete pair reaches the
|
|
238
|
+
* bounded paid fallback. Route callers run this in the background. */
|
|
239
|
+
export async function refreshMuseCapacityIfStale(nowMs = Date.now(), opts = {}) {
|
|
240
|
+
if (!configuredCapacityRuntimes().includes(MUSE_RUNTIME)) {
|
|
241
|
+
return { probed: false, reason: "unconfigured" };
|
|
242
|
+
}
|
|
243
|
+
try {
|
|
244
|
+
await maybeRefreshMuseCapacityFromHost({}, nowMs);
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
// The paid gate below still decides from durable last-good rows.
|
|
248
|
+
}
|
|
249
|
+
return maybeProbeMuseCapacity(nowMs, opts);
|
|
250
|
+
}
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
// Default-on one-hour Muse capacity fallback. Fake MSP host only: no live
|
|
2
|
+
// provider request and no paid model call in this suite.
|
|
3
|
+
import { beforeEach, test } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import fs from "node:fs";
|
|
6
|
+
import os from "node:os";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
process.env.AGENT_DEALER_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-muse-probe-"));
|
|
9
|
+
import { MuseCapacityHost } from "./muse-host.js";
|
|
10
|
+
import { hasFreshMuseCapacityPair, isMusePaidFallbackEnabled, maybeProbeMuseCapacity, MUSE_CAPACITY_PROBE_SERVE_ARGV, MUSE_CAPACITY_REFRESH_ENV, resetMuseCapacityProbeStateForTests, runMuseCapacityProbe, } from "./muse-probe.js";
|
|
11
|
+
import { ingestMuseUsagePayload } from "./muse.js";
|
|
12
|
+
const { migrate } = await import("../db/index.js");
|
|
13
|
+
const { createAgent } = await import("../repository/agents.js");
|
|
14
|
+
const { clearAllCapacitySnapshots, listCapacitySnapshots } = await import("../repository/runtime-capacity.js");
|
|
15
|
+
const FAKE = new URL("./fixtures/fake-muse-serve.mjs", import.meta.url).pathname;
|
|
16
|
+
function hostOpts(mode, extraEnv = {}) {
|
|
17
|
+
const now = Date.now();
|
|
18
|
+
return {
|
|
19
|
+
command: process.execPath,
|
|
20
|
+
args: [FAKE],
|
|
21
|
+
env: {
|
|
22
|
+
META_API_KEY: "test-fake-key",
|
|
23
|
+
FAKE_MSP_MODE: mode,
|
|
24
|
+
FAKE_MSP_NOW_MS: String(now),
|
|
25
|
+
...extraEnv,
|
|
26
|
+
},
|
|
27
|
+
timeoutMs: 10_000,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
function payload(nowMs, roles = "both") {
|
|
31
|
+
return {
|
|
32
|
+
usage: {
|
|
33
|
+
observedAtMs: nowMs,
|
|
34
|
+
tier: "synthetic",
|
|
35
|
+
window: {
|
|
36
|
+
usedPercent: 25,
|
|
37
|
+
resetsAtMs: nowMs + 2 * 3600_000,
|
|
38
|
+
windowDurationMins: 300,
|
|
39
|
+
},
|
|
40
|
+
...(roles === "both"
|
|
41
|
+
? { weekly: { usedPercent: 40, resetsAtMs: nowMs + 3 * 24 * 3600_000 } }
|
|
42
|
+
: {}),
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function recordedMethods(file) {
|
|
47
|
+
if (!fs.existsSync(file))
|
|
48
|
+
return [];
|
|
49
|
+
return fs
|
|
50
|
+
.readFileSync(file, "utf8")
|
|
51
|
+
.split("\n")
|
|
52
|
+
.filter(Boolean)
|
|
53
|
+
.map((line) => String(JSON.parse(line).method ?? ""));
|
|
54
|
+
}
|
|
55
|
+
beforeEach(() => {
|
|
56
|
+
migrate();
|
|
57
|
+
clearAllCapacitySnapshots();
|
|
58
|
+
resetMuseCapacityProbeStateForTests();
|
|
59
|
+
delete process.env[MUSE_CAPACITY_REFRESH_ENV];
|
|
60
|
+
delete process.env.AGENT_DEALER_MUSE_CAPACITY_REFRESH_MS;
|
|
61
|
+
const existing = createAgent({
|
|
62
|
+
name: `muse-probe-${Date.now()}-${Math.random()}`,
|
|
63
|
+
runtime: "muse_code",
|
|
64
|
+
deckId: "33333333-3333-4333-8333-333333333333",
|
|
65
|
+
});
|
|
66
|
+
assert.equal(existing.runtime, "muse_code");
|
|
67
|
+
});
|
|
68
|
+
test("fallback defaults on; explicit off and unrecognized values disable spending", async () => {
|
|
69
|
+
for (const value of [undefined, "", "paid-after-1h"]) {
|
|
70
|
+
if (value === undefined)
|
|
71
|
+
delete process.env[MUSE_CAPACITY_REFRESH_ENV];
|
|
72
|
+
else
|
|
73
|
+
process.env[MUSE_CAPACITY_REFRESH_ENV] = value;
|
|
74
|
+
assert.equal(isMusePaidFallbackEnabled(), true);
|
|
75
|
+
}
|
|
76
|
+
for (const value of ["off", "auto", "paid-after-1h "]) {
|
|
77
|
+
process.env[MUSE_CAPACITY_REFRESH_ENV] = value;
|
|
78
|
+
assert.equal(isMusePaidFallbackEnabled(), false);
|
|
79
|
+
assert.deepEqual(await maybeProbeMuseCapacity(Date.now()), {
|
|
80
|
+
probed: false,
|
|
81
|
+
reason: "disabled",
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
test("legacy refresh off remains no-spend unless paid fallback is explicitly enabled", () => {
|
|
86
|
+
process.env.AGENT_DEALER_MUSE_CAPACITY_REFRESH_MS = "off";
|
|
87
|
+
delete process.env[MUSE_CAPACITY_REFRESH_ENV];
|
|
88
|
+
assert.equal(isMusePaidFallbackEnabled(), false);
|
|
89
|
+
process.env[MUSE_CAPACITY_REFRESH_ENV] = "paid-after-1h";
|
|
90
|
+
assert.equal(isMusePaidFallbackEnabled(), true);
|
|
91
|
+
});
|
|
92
|
+
test("a complete pair under one hour suppresses the fallback", async () => {
|
|
93
|
+
const now = Date.now();
|
|
94
|
+
await ingestMuseUsagePayload(payload(now - 59 * 60_000));
|
|
95
|
+
assert.equal(hasFreshMuseCapacityPair(now), true);
|
|
96
|
+
assert.deepEqual(await maybeProbeMuseCapacity(now), { probed: false, reason: "fresh" });
|
|
97
|
+
});
|
|
98
|
+
test("one missing half triggers one minimal turn and stores exactly 5H plus 1W", async () => {
|
|
99
|
+
const now = Date.now();
|
|
100
|
+
await ingestMuseUsagePayload(payload(now - 5 * 60_000, "five_hour"));
|
|
101
|
+
assert.equal(hasFreshMuseCapacityPair(now), false, "a fresh half is not a complete pair");
|
|
102
|
+
const host = new MuseCapacityHost(hostOpts("serve-turn-full"));
|
|
103
|
+
try {
|
|
104
|
+
const outcome = await maybeProbeMuseCapacity(now, { host, timeoutMs: 10_000 });
|
|
105
|
+
assert.equal(outcome.probed, true);
|
|
106
|
+
assert.equal(outcome.reason, "completed");
|
|
107
|
+
assert.equal(outcome.ok, true);
|
|
108
|
+
assert.deepEqual(outcome.windowsUpdated?.sort(), ["rolling_all_models", "weekly_all_models"]);
|
|
109
|
+
const roles = listCapacitySnapshots("muse_code")
|
|
110
|
+
.filter((w) => w.criticalRole !== null)
|
|
111
|
+
.map((w) => w.criticalRole)
|
|
112
|
+
.sort();
|
|
113
|
+
assert.deepEqual(roles, ["five_hour", "weekly"]);
|
|
114
|
+
}
|
|
115
|
+
finally {
|
|
116
|
+
await host.shutdown();
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
test("concurrent stale readers single-flight one turn", async () => {
|
|
120
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-muse-probe-record-"));
|
|
121
|
+
const record = path.join(dir, "methods.jsonl");
|
|
122
|
+
const host = new MuseCapacityHost(hostOpts("serve-turn-full", { FAKE_MSP_RECORD: record }));
|
|
123
|
+
try {
|
|
124
|
+
const now = Date.now();
|
|
125
|
+
const outcomes = await Promise.all(Array.from({ length: 5 }, () => maybeProbeMuseCapacity(now, { host, timeoutMs: 10_000 })));
|
|
126
|
+
assert.equal(outcomes.filter((o) => o.reason === "completed").length, 1);
|
|
127
|
+
assert.equal(outcomes.filter((o) => o.reason === "shared").length, 4);
|
|
128
|
+
assert.ok(outcomes.every((o) => o.probed && o.ok));
|
|
129
|
+
assert.equal(recordedMethods(record).filter((m) => m === "turn/start").length, 1);
|
|
130
|
+
}
|
|
131
|
+
finally {
|
|
132
|
+
await host.shutdown();
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
test("failed admission preserves rows and enters hourly backoff", async () => {
|
|
136
|
+
const now = Date.now();
|
|
137
|
+
await ingestMuseUsagePayload(payload(now - 61 * 60_000));
|
|
138
|
+
const before = listCapacitySnapshots("muse_code").map((w) => ({
|
|
139
|
+
key: w.windowKey,
|
|
140
|
+
remaining: w.remainingPercent,
|
|
141
|
+
observedAt: w.observedAt,
|
|
142
|
+
}));
|
|
143
|
+
const host = new MuseCapacityHost(hostOpts("serve-turn-rejected"));
|
|
144
|
+
try {
|
|
145
|
+
const first = await maybeProbeMuseCapacity(now, { host, timeoutMs: 10_000 });
|
|
146
|
+
assert.equal(first.probed, true);
|
|
147
|
+
assert.equal(first.ok, false);
|
|
148
|
+
assert.equal(first.failureKind, "unadmitted");
|
|
149
|
+
assert.deepEqual(await maybeProbeMuseCapacity(now + 60_000, { host, timeoutMs: 10_000 }), {
|
|
150
|
+
probed: false,
|
|
151
|
+
reason: "backoff",
|
|
152
|
+
});
|
|
153
|
+
assert.deepEqual(listCapacitySnapshots("muse_code").map((w) => ({
|
|
154
|
+
key: w.windowKey,
|
|
155
|
+
remaining: w.remainingPercent,
|
|
156
|
+
observedAt: w.observedAt,
|
|
157
|
+
})), before);
|
|
158
|
+
}
|
|
159
|
+
finally {
|
|
160
|
+
await host.shutdown();
|
|
161
|
+
}
|
|
162
|
+
});
|
|
163
|
+
test("production probe host disables shell/write and keeps restricted network", () => {
|
|
164
|
+
assert.deepEqual([...MUSE_CAPACITY_PROBE_SERVE_ARGV], [
|
|
165
|
+
"serve",
|
|
166
|
+
"--sandbox-network",
|
|
167
|
+
"restricted",
|
|
168
|
+
"--disable-write",
|
|
169
|
+
"--disable-shell",
|
|
170
|
+
]);
|
|
171
|
+
});
|
|
172
|
+
test("direct probe success requires the complete pair", async () => {
|
|
173
|
+
const host = new MuseCapacityHost(hostOpts("serve-turn-full"));
|
|
174
|
+
try {
|
|
175
|
+
const result = await runMuseCapacityProbe(Date.now(), { host, timeoutMs: 10_000 });
|
|
176
|
+
assert.equal(result.ok, true);
|
|
177
|
+
assert.equal(result.admitted, true);
|
|
178
|
+
assert.equal(result.failureKind, null);
|
|
179
|
+
}
|
|
180
|
+
finally {
|
|
181
|
+
await host.shutdown();
|
|
182
|
+
}
|
|
183
|
+
});
|
|
@@ -9,14 +9,15 @@
|
|
|
9
9
|
// retired: only a host that observed the account's provider traffic can
|
|
10
10
|
// answer `usage/read` / emit `usage/changed`.
|
|
11
11
|
//
|
|
12
|
-
//
|
|
13
|
-
// sending a prompt
|
|
12
|
+
// The free read phase reads observed capacity through the stable Muse Session
|
|
13
|
+
// Protocol without sending a prompt or consuming model tokens: `initialize` →
|
|
14
14
|
// `initialized` → `usage/read` on the owned host, with `usage/changed`
|
|
15
15
|
// notifications ingested as received. The client enforces a read-only
|
|
16
16
|
// allowlist (`initialize`, `initialized`, `usage/read`) — any other method
|
|
17
17
|
// throws before it is written, so a capacity read can never start a
|
|
18
|
-
// session, send a prompt, or run model work.
|
|
19
|
-
//
|
|
18
|
+
// session, send a prompt, or run model work. The separate default-on fallback
|
|
19
|
+
// in `muse-probe.ts` runs only after this phase still lacks a complete current
|
|
20
|
+
// pair for an hour. Neither path touches the Keychain or undocumented endpoints.
|
|
20
21
|
//
|
|
21
22
|
// Stable `usage/read` shape:
|
|
22
23
|
// result: {
|
|
@@ -48,6 +49,11 @@ import fs from "node:fs";
|
|
|
48
49
|
import { MUSE_CLI_ENV, resolveMuseAuthFile, resolveMuseBin } from "../cli-env.js";
|
|
49
50
|
import { normalizeAdapterWindow, normalizeUnavailableWindow, } from "./adapter.js";
|
|
50
51
|
export const MUSE_RUNTIME = "muse_code";
|
|
52
|
+
/** Muse observations remain the last-known current value until the one-hour
|
|
53
|
+
* fallback boundary. This avoids the shared adapter's generic 15-minute N/A
|
|
54
|
+
* gap before a bounded refresh is eligible to run. */
|
|
55
|
+
export const MUSE_CAPACITY_STALE_AFTER_MS = 60 * 60 * 1000;
|
|
56
|
+
export const MUSE_CAPACITY_EXPIRES_AFTER_MS = 60 * 60 * 1000;
|
|
51
57
|
/** Methods this client may ever send. Anything else throws before write. */
|
|
52
58
|
export const MSP_READ_ONLY_METHODS = ["initialize", "initialized", "usage/read"];
|
|
53
59
|
/**
|
|
@@ -189,6 +195,8 @@ function windowReadingFromEntry(kind, entry, observedAt) {
|
|
|
189
195
|
usedPercent,
|
|
190
196
|
resetAt,
|
|
191
197
|
observedAt,
|
|
198
|
+
staleAfterMs: MUSE_CAPACITY_STALE_AFTER_MS,
|
|
199
|
+
expiresAfterMs: MUSE_CAPACITY_EXPIRES_AFTER_MS,
|
|
192
200
|
source: "supported_protocol",
|
|
193
201
|
// Muse reports exactly this one rolling/weekly pair — it is always the
|
|
194
202
|
// account-wide critical window, never a model-specific extra.
|
|
@@ -868,8 +876,10 @@ export function createMuseCapacityAdapter(opts = {}) {
|
|
|
868
876
|
export const MUSE_REFRESH_THROTTLE_MS_DEFAULT = 5 * 60 * 1000;
|
|
869
877
|
/**
|
|
870
878
|
* Throttle bound for production refreshes. Override with
|
|
871
|
-
* `AGENT_DEALER_MUSE_CAPACITY_REFRESH_MS` (milliseconds); `off` disables
|
|
872
|
-
*
|
|
879
|
+
* `AGENT_DEALER_MUSE_CAPACITY_REFRESH_MS` (milliseconds); `off` disables the
|
|
880
|
+
* free host read and, for backward-compatible no-spend behavior, also disables
|
|
881
|
+
* the paid fallback unless `AGENT_DEALER_MUSE_CAPACITY_REFRESH=paid-after-1h`
|
|
882
|
+
* is explicit. Non-positive or unparsable values fall back to the default.
|
|
873
883
|
* Shared with the owned host (`./muse-host.js`).
|
|
874
884
|
*/
|
|
875
885
|
export function museRefreshThrottleMs() {
|