agent-dealer 1.2.2 → 1.2.4

package/bundle/server/dist/capacity/muse-probe.js

@@ -0,0 +1,250 @@
+// packages/server/src/capacity/muse-probe.ts
+//
+// Reliable Muse 5H/1W fallback. Muse's stable protocol only exposes
+// subscription capacity after the same `muse serve` process has observed
+// real provider traffic. A read-only poll, session resume, or exec log cannot
+// populate it (NOT-269). When the free sources have no complete current pair
+// for one hour, this module runs one tiny bounded turn on a dedicated owned
+// host, then performs `usage/read` on that exact host.
+//
+// The fallback is default-on and can be disabled with
+// `AGENT_DEALER_MUSE_CAPACITY_REFRESH=off`. It is single-flight, attempted at
+// most once per hour, and backs off 1h -> 2h -> 4h -> 8h after failures. The
+// host disables shell and workspace writes and keeps restricted network;
+// Muse 1.4's serve protocol has no per-turn max-step or disable-web-tools
+// switch, so the fixed one-word prompt plus wall-clock cancellation is the
+// tightest supported bound. No transcript, prompt, credential, or account
+// payload is logged.
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { MUSE_CODE_CONTRIBUTOR_MODEL } from "@agent-dealer/shared";
+import { getDataDir } from "../db/index.js";
+import { listCapacitySnapshots } from "../repository/runtime-capacity.js";
+import { runMuseServeTurn } from "../runners/muse-serve-session.js";
+import { configuredCapacityRuntimes } from "./service.js";
+import { maybeRefreshMuseCapacityFromHost, MuseCapacityHost, } from "./muse-host.js";
+import { MUSE_RUNTIME } from "./muse.js";
+export const MUSE_CAPACITY_REFRESH_ENV = "AGENT_DEALER_MUSE_CAPACITY_REFRESH";
+export const MUSE_CAPACITY_REFRESH_PAID_VALUE = "paid-after-1h";
+export const MUSE_CAPACITY_REFRESH_OFF_VALUE = "off";
+export const MUSE_CAPACITY_PROBE_TIMEOUT_ENV = "AGENT_DEALER_MUSE_CAPACITY_PROBE_TIMEOUT_MS";
+export const MUSE_CAPACITY_PROBE_TIMEOUT_MS_DEFAULT = 60_000;
+export const MUSE_CAPACITY_PROBE_STALE_AFTER_MS = 60 * 60 * 1000;
+export const MUSE_CAPACITY_PROBE_ATTEMPT_COOLDOWN_MS = 60 * 60 * 1000;
+export const MUSE_CAPACITY_PROBE_BACKOFF_CAP_MS = 8 * 60 * 60 * 1000;
+export const MUSE_CAPACITY_PROBE_PROMPT = "Reply with exactly: OK";
+export const MUSE_CAPACITY_PROBE_SERVE_ARGV = [
+    "serve",
+    "--sandbox-network",
+    "restricted",
+    "--disable-write",
+    "--disable-shell",
+];
+export function isMusePaidFallbackEnabled() {
+    const setting = process.env[MUSE_CAPACITY_REFRESH_ENV];
+    if (setting === MUSE_CAPACITY_REFRESH_PAID_VALUE)
+        return true;
+    if (setting !== undefined && setting !== "")
+        return false;
+    // Backward compatibility: this older switch used to disable every Muse
+    // capacity refresh. Do not turn an existing no-refresh deployment into a
+    // spender on upgrade. An explicit paid-after-1h above may override it.
+    if (process.env.AGENT_DEALER_MUSE_CAPACITY_REFRESH_MS === "off")
+        return false;
+    return true;
+}
+export function museCapacityProbeTimeoutMs() {
+    const raw = process.env[MUSE_CAPACITY_PROBE_TIMEOUT_ENV];
+    if (raw !== undefined && raw !== "") {
+        const n = Number(raw);
+        if (Number.isFinite(n) && n > 0)
+            return n;
+    }
+    return MUSE_CAPACITY_PROBE_TIMEOUT_MS_DEFAULT;
+}
+function validRoleObservedAtMs(role, nowMs) {
+    const row = listCapacitySnapshots(MUSE_RUNTIME).find((w) => w.criticalRole === role);
+    if (!row || row.remainingPercent === null || row.unavailableReason !== null || row.source === "unavailable") {
+        return null;
+    }
+    const observedMs = Date.parse(row.observedAt);
+    if (!Number.isFinite(observedMs) || observedMs > nowMs)
+        return null;
+    if (row.resetAt !== null) {
+        const resetMs = Date.parse(row.resetAt);
+        if (!Number.isFinite(resetMs) || resetMs <= nowMs)
+            return null;
+    }
+    return observedMs;
+}
+/** Both halves must be valid and under one hour old. A fresh 5H window may
+ * never hide a missing/stale 1W sibling (or vice versa). */
+export function hasFreshMuseCapacityPair(nowMs = Date.now()) {
+    const fiveHour = validRoleObservedAtMs("five_hour", nowMs);
+    const weekly = validRoleObservedAtMs("weekly", nowMs);
+    return (fiveHour !== null &&
+        weekly !== null &&
+        nowMs - fiveHour < MUSE_CAPACITY_PROBE_STALE_AFTER_MS &&
+        nowMs - weekly < MUSE_CAPACITY_PROBE_STALE_AFTER_MS);
+}
+function museProbeDiagnosticLogPath() {
+    return path.join(getDataDir(), "capacity", "muse-probe.log");
+}
+function appendMuseProbeDiagnostic(nowMs, result) {
+    try {
+        const file = museProbeDiagnosticLogPath();
+        fs.mkdirSync(path.dirname(file), { recursive: true });
+        fs.appendFileSync(file, `${JSON.stringify({
+            ts: new Date(nowMs).toISOString(),
+            event: "muse_capacity_probe",
+            trigger: "stale_60m",
+            model: MUSE_CODE_CONTRIBUTOR_MODEL,
+            ...result,
+        })}\n`);
+    }
+    catch {
+        // Diagnostics are advisory; capacity refresh must never fail on logging.
+    }
+}
+/** One bounded real turn followed by a read on the same restricted host. */
+export async function runMuseCapacityProbe(nowMs = Date.now(), opts = {}) {
+    const startedRealMs = Date.now();
+    const timeoutMs = opts.timeoutMs ?? museCapacityProbeTimeoutMs();
+    const ownsHost = opts.host === undefined;
+    const host = opts.host ??
+        new MuseCapacityHost({
+            ...opts.hostOptions,
+            args: opts.hostOptions?.args ?? [...MUSE_CAPACITY_PROBE_SERVE_ARGV],
+            timeoutMs: Math.min(opts.hostOptions?.timeoutMs ?? timeoutMs, timeoutMs),
+        });
+    let result;
+    try {
+        const turn = await runMuseServeTurn({
+            host,
+            prompt: MUSE_CAPACITY_PROBE_PROMPT,
+            model: MUSE_CODE_CONTRIBUTOR_MODEL,
+            cwd: os.tmpdir(),
+            timeoutMs,
+            rpcTimeoutMs: Math.min(15_000, timeoutMs),
+        });
+        if (!turn.admitted) {
+            result = {
+                ok: false,
+                admitted: false,
+                terminal: null,
+                failureKind: "unadmitted",
+                windowsUpdated: [],
+                durationMs: Date.now() - startedRealMs,
+            };
+        }
+        else {
+            await host.readUsage();
+            const checkNowMs = Math.max(nowMs, Date.now());
+            const ok = hasFreshMuseCapacityPair(checkNowMs);
+            result = {
+                ok,
+                admitted: true,
+                terminal: turn.terminal,
+                failureKind: ok
+                    ? null
+                    : turn.timedOut
+                        ? "timeout"
+                        : turn.failure !== null
+                            ? "turn_failed"
+                            : "no_windows",
+                windowsUpdated: ok ? ["rolling_all_models", "weekly_all_models"] : [],
+                durationMs: Date.now() - startedRealMs,
+            };
+        }
+    }
+    catch {
+        result = {
+            ok: false,
+            admitted: false,
+            terminal: null,
+            failureKind: "exception",
+            windowsUpdated: [],
+            durationMs: Date.now() - startedRealMs,
+        };
+    }
+    finally {
+        if (ownsHost)
+            await host.shutdown().catch(() => undefined);
+    }
+    appendMuseProbeDiagnostic(nowMs, result);
+    const outcome = result.ok ? "ok" : `failed: ${result.failureKind ?? "unknown"}`;
+    console.error(`[muse-capacity] fallback ${outcome}`);
+    return result;
+}
+let museProbeInFlight = null;
+let lastMuseProbeAttemptMs = 0;
+let consecutiveMuseProbeFailures = 0;
+function museProbeCooldownMs() {
+    const shift = Math.min(consecutiveMuseProbeFailures, 3);
+    return Math.min(MUSE_CAPACITY_PROBE_ATTEMPT_COOLDOWN_MS * 2 ** shift, MUSE_CAPACITY_PROBE_BACKOFF_CAP_MS);
+}
+export function resetMuseCapacityProbeStateForTests() {
+    museProbeInFlight = null;
+    lastMuseProbeAttemptMs = 0;
+    consecutiveMuseProbeFailures = 0;
+}
+export async function maybeProbeMuseCapacity(nowMs = Date.now(), opts = {}) {
+    try {
+        if (!isMusePaidFallbackEnabled())
+            return { probed: false, reason: "disabled" };
+        if (!configuredCapacityRuntimes().includes(MUSE_RUNTIME)) {
+            return { probed: false, reason: "unconfigured" };
+        }
+        if (hasFreshMuseCapacityPair(nowMs))
+            return { probed: false, reason: "fresh" };
+        if (museProbeInFlight) {
+            const shared = await museProbeInFlight;
+            return {
+                probed: true,
+                reason: "shared",
+                ok: shared.ok,
+                failureKind: shared.failureKind,
+                windowsUpdated: shared.windowsUpdated,
+            };
+        }
+        if (nowMs - lastMuseProbeAttemptMs < museProbeCooldownMs()) {
+            return { probed: false, reason: "backoff" };
+        }
+        lastMuseProbeAttemptMs = nowMs;
+        const run = runMuseCapacityProbe(nowMs, opts);
+        museProbeInFlight = run;
+        try {
+            const result = await run;
+            consecutiveMuseProbeFailures = result.ok ? 0 : consecutiveMuseProbeFailures + 1;
+            return {
+                probed: true,
+                reason: "completed",
+                ok: result.ok,
+                failureKind: result.failureKind,
+                windowsUpdated: result.windowsUpdated,
+            };
+        }
+        finally {
+            if (museProbeInFlight === run)
+                museProbeInFlight = null;
+        }
+    }
+    catch {
+        return { probed: false, reason: "error" };
+    }
+}
+/** Free read first; only a still-missing/stale complete pair reaches the
+ * bounded paid fallback. Route callers run this in the background. */
+export async function refreshMuseCapacityIfStale(nowMs = Date.now(), opts = {}) {
+    if (!configuredCapacityRuntimes().includes(MUSE_RUNTIME)) {
+        return { probed: false, reason: "unconfigured" };
+    }
+    try {
+        await maybeRefreshMuseCapacityFromHost({}, nowMs);
+    }
+    catch {
+        // The paid gate below still decides from durable last-good rows.
+    }
+    return maybeProbeMuseCapacity(nowMs, opts);
+}

package/bundle/server/dist/capacity/muse-probe.test.js

@@ -0,0 +1,183 @@
+// Default-on one-hour Muse capacity fallback. Fake MSP host only: no live
+// provider request and no paid model call in this suite.
+import { beforeEach, test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+process.env.AGENT_DEALER_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-muse-probe-"));
+import { MuseCapacityHost } from "./muse-host.js";
+import { hasFreshMuseCapacityPair, isMusePaidFallbackEnabled, maybeProbeMuseCapacity, MUSE_CAPACITY_PROBE_SERVE_ARGV, MUSE_CAPACITY_REFRESH_ENV, resetMuseCapacityProbeStateForTests, runMuseCapacityProbe, } from "./muse-probe.js";
+import { ingestMuseUsagePayload } from "./muse.js";
+const { migrate } = await import("../db/index.js");
+const { createAgent } = await import("../repository/agents.js");
+const { clearAllCapacitySnapshots, listCapacitySnapshots } = await import("../repository/runtime-capacity.js");
+const FAKE = new URL("./fixtures/fake-muse-serve.mjs", import.meta.url).pathname;
+function hostOpts(mode, extraEnv = {}) {
+    const now = Date.now();
+    return {
+        command: process.execPath,
+        args: [FAKE],
+        env: {
+            META_API_KEY: "test-fake-key",
+            FAKE_MSP_MODE: mode,
+            FAKE_MSP_NOW_MS: String(now),
+            ...extraEnv,
+        },
+        timeoutMs: 10_000,
+    };
+}
+function payload(nowMs, roles = "both") {
+    return {
+        usage: {
+            observedAtMs: nowMs,
+            tier: "synthetic",
+            window: {
+                usedPercent: 25,
+                resetsAtMs: nowMs + 2 * 3600_000,
+                windowDurationMins: 300,
+            },
+            ...(roles === "both"
+                ? { weekly: { usedPercent: 40, resetsAtMs: nowMs + 3 * 24 * 3600_000 } }
+                : {}),
+        },
+    };
+}
+function recordedMethods(file) {
+    if (!fs.existsSync(file))
+        return [];
+    return fs
+        .readFileSync(file, "utf8")
+        .split("\n")
+        .filter(Boolean)
+        .map((line) => String(JSON.parse(line).method ?? ""));
+}
+beforeEach(() => {
+    migrate();
+    clearAllCapacitySnapshots();
+    resetMuseCapacityProbeStateForTests();
+    delete process.env[MUSE_CAPACITY_REFRESH_ENV];
+    delete process.env.AGENT_DEALER_MUSE_CAPACITY_REFRESH_MS;
+    const existing = createAgent({
+        name: `muse-probe-${Date.now()}-${Math.random()}`,
+        runtime: "muse_code",
+        deckId: "33333333-3333-4333-8333-333333333333",
+    });
+    assert.equal(existing.runtime, "muse_code");
+});
+test("fallback defaults on; explicit off and unrecognized values disable spending", async () => {
+    for (const value of [undefined, "", "paid-after-1h"]) {
+        if (value === undefined)
+            delete process.env[MUSE_CAPACITY_REFRESH_ENV];
+        else
+            process.env[MUSE_CAPACITY_REFRESH_ENV] = value;
+        assert.equal(isMusePaidFallbackEnabled(), true);
+    }
+    for (const value of ["off", "auto", "paid-after-1h "]) {
+        process.env[MUSE_CAPACITY_REFRESH_ENV] = value;
+        assert.equal(isMusePaidFallbackEnabled(), false);
+        assert.deepEqual(await maybeProbeMuseCapacity(Date.now()), {
+            probed: false,
+            reason: "disabled",
+        });
+    }
+});
+test("legacy refresh off remains no-spend unless paid fallback is explicitly enabled", () => {
+    process.env.AGENT_DEALER_MUSE_CAPACITY_REFRESH_MS = "off";
+    delete process.env[MUSE_CAPACITY_REFRESH_ENV];
+    assert.equal(isMusePaidFallbackEnabled(), false);
+    process.env[MUSE_CAPACITY_REFRESH_ENV] = "paid-after-1h";
+    assert.equal(isMusePaidFallbackEnabled(), true);
+});
+test("a complete pair under one hour suppresses the fallback", async () => {
+    const now = Date.now();
+    await ingestMuseUsagePayload(payload(now - 59 * 60_000));
+    assert.equal(hasFreshMuseCapacityPair(now), true);
+    assert.deepEqual(await maybeProbeMuseCapacity(now), { probed: false, reason: "fresh" });
+});
+test("one missing half triggers one minimal turn and stores exactly 5H plus 1W", async () => {
+    const now = Date.now();
+    await ingestMuseUsagePayload(payload(now - 5 * 60_000, "five_hour"));
+    assert.equal(hasFreshMuseCapacityPair(now), false, "a fresh half is not a complete pair");
+    const host = new MuseCapacityHost(hostOpts("serve-turn-full"));
+    try {
+        const outcome = await maybeProbeMuseCapacity(now, { host, timeoutMs: 10_000 });
+        assert.equal(outcome.probed, true);
+        assert.equal(outcome.reason, "completed");
+        assert.equal(outcome.ok, true);
+        assert.deepEqual(outcome.windowsUpdated?.sort(), ["rolling_all_models", "weekly_all_models"]);
+        const roles = listCapacitySnapshots("muse_code")
+            .filter((w) => w.criticalRole !== null)
+            .map((w) => w.criticalRole)
+            .sort();
+        assert.deepEqual(roles, ["five_hour", "weekly"]);
+    }
+    finally {
+        await host.shutdown();
+    }
+});
+test("concurrent stale readers single-flight one turn", async () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-muse-probe-record-"));
+    const record = path.join(dir, "methods.jsonl");
+    const host = new MuseCapacityHost(hostOpts("serve-turn-full", { FAKE_MSP_RECORD: record }));
+    try {
+        const now = Date.now();
+        const outcomes = await Promise.all(Array.from({ length: 5 }, () => maybeProbeMuseCapacity(now, { host, timeoutMs: 10_000 })));
+        assert.equal(outcomes.filter((o) => o.reason === "completed").length, 1);
+        assert.equal(outcomes.filter((o) => o.reason === "shared").length, 4);
+        assert.ok(outcomes.every((o) => o.probed && o.ok));
+        assert.equal(recordedMethods(record).filter((m) => m === "turn/start").length, 1);
+    }
+    finally {
+        await host.shutdown();
+    }
+});
+test("failed admission preserves rows and enters hourly backoff", async () => {
+    const now = Date.now();
+    await ingestMuseUsagePayload(payload(now - 61 * 60_000));
+    const before = listCapacitySnapshots("muse_code").map((w) => ({
+        key: w.windowKey,
+        remaining: w.remainingPercent,
+        observedAt: w.observedAt,
+    }));
+    const host = new MuseCapacityHost(hostOpts("serve-turn-rejected"));
+    try {
+        const first = await maybeProbeMuseCapacity(now, { host, timeoutMs: 10_000 });
+        assert.equal(first.probed, true);
+        assert.equal(first.ok, false);
+        assert.equal(first.failureKind, "unadmitted");
+        assert.deepEqual(await maybeProbeMuseCapacity(now + 60_000, { host, timeoutMs: 10_000 }), {
+            probed: false,
+            reason: "backoff",
+        });
+        assert.deepEqual(listCapacitySnapshots("muse_code").map((w) => ({
+            key: w.windowKey,
+            remaining: w.remainingPercent,
+            observedAt: w.observedAt,
+        })), before);
+    }
+    finally {
+        await host.shutdown();
+    }
+});
+test("production probe host disables shell/write and keeps restricted network", () => {
+    assert.deepEqual([...MUSE_CAPACITY_PROBE_SERVE_ARGV], [
+        "serve",
+        "--sandbox-network",
+        "restricted",
+        "--disable-write",
+        "--disable-shell",
+    ]);
+});
+test("direct probe success requires the complete pair", async () => {
+    const host = new MuseCapacityHost(hostOpts("serve-turn-full"));
+    try {
+        const result = await runMuseCapacityProbe(Date.now(), { host, timeoutMs: 10_000 });
+        assert.equal(result.ok, true);
+        assert.equal(result.admitted, true);
+        assert.equal(result.failureKind, null);
+    }
+    finally {
+        await host.shutdown();
+    }
+});