agent-dealer 1.2.7 → 1.2.9

package/bundle/server/dist/capacity/muse-probe.test.js

@@ -6,10 +6,11 @@ import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 process.env.AGENT_DEALER_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-muse-probe-"));
+import { MUSE_CODE_CONTRIBUTOR_MODEL } from "@agent-dealer/shared";
 import { MuseCapacityHost } from "./muse-host.js";
 import { hasFreshMuseCapacityPair, isMusePaidFallbackEnabled, maybeProbeMuseCapacity, MUSE_CAPACITY_PROBE_SERVE_ARGV, MUSE_CAPACITY_REFRESH_ENV, resetMuseCapacityProbeStateForTests, runMuseCapacityProbe, } from "./muse-probe.js";
 import { ingestMuseUsagePayload } from "./muse.js";
-const { migrate } = await import("../db/index.js");
+const { migrate, getDataDir } = await import("../db/index.js");
 const { createAgent } = await import("../repository/agents.js");
 const { clearAllCapacitySnapshots, listCapacitySnapshots } = await import("../repository/runtime-capacity.js");
 const FAKE = new URL("./fixtures/fake-muse-serve.mjs", import.meta.url).pathname;
@@ -181,3 +182,189 @@ test("direct probe success requires the complete pair", async () => {
         await host.shutdown();
     }
 });
+// NOT-300: the probe log must explain the next failure. A stub host drives
+// runMuseServeTurn to the exact turn outcome — no live model call.
+function stubProbeHost(opts = {}) {
+    const sessionId = "sess-stub-1";
+    const session = { sessionId, modelId: MUSE_CODE_CONTRIBUTOR_MODEL };
+    let notifications = 0;
+    return {
+        ensureStarted: async () => {
+            if (opts.failStart)
+                throw opts.failStart;
+            return true;
+        },
+        execRequest: async (_id, method) => {
+            if (method === "session/start")
+                return { result: { session } };
+            if (method === "turn/start")
+                return { result: { turnId: "turn-stub-1" } };
+            if (method === "session/read") {
+                return { result: { session, history: { items: [] } } };
+            }
+            return { result: {} };
+        },
+        waitForHostNotification: async () => {
+            notifications += 1;
+            if (opts.hangTurnMs !== undefined && notifications === 1) {
+                await new Promise((r) => setTimeout(r, opts.hangTurnMs));
+                return null;
+            }
+            if (opts.turnParams !== undefined || opts.hangTurnMs === undefined) {
+                return {
+                    method: "turn/completed",
+                    params: opts.turnParams ??
+                        {
+                            sessionId,
+                            terminal: "failed",
+                            error: { kind: "other", message: "boom" },
+                            durationMs: 982,
+                        },
+                };
+            }
+            return {
+                method: "turn/completed",
+                params: { sessionId, terminal: "cancelled", reason: "cancel requested" },
+            };
+        },
+        isConnected: () => true,
+        cancelExecTurn: async () => { },
+        readUsage: async () => ({ ok: false, reason: "missing" }),
+        shutdown: async () => { },
+    };
+}
+function probeLogPath() {
+    return path.join(getDataDir(), "capacity", "muse-probe.log");
+}
+function probeLogSize() {
+    return fs.existsSync(probeLogPath()) ? fs.statSync(probeLogPath()).size : 0;
+}
+function probeLogLinesSince(bytes) {
+    const text = fs.existsSync(probeLogPath())
+        ? fs.readFileSync(probeLogPath(), "utf8").slice(bytes)
+        : "";
+    return text
+        .split("\n")
+        .filter((l) => l.trim())
+        .map((l) => JSON.parse(l));
+}
+const PROBE_LOG_ALLOWED_KEYS = [
+    "admitted",
+    "durationMs",
+    "event",
+    "failureDetail",
+    "failureKind",
+    "model",
+    "ok",
+    "terminal",
+    "trigger",
+    "ts",
+    "windowsUpdated",
+];
+function assertProbeLogShape(line) {
+    assert.deepEqual(Object.keys(line).sort(), [...PROBE_LOG_ALLOWED_KEYS].sort());
+    const detail = line.failureDetail;
+    if (detail !== null) {
+        assert.deepEqual(Object.keys(detail).sort(), ["kind", "message"]);
+    }
+    const raw = JSON.stringify(line);
+    for (const secret of ["Reply with exactly", "transcript", "META_API_KEY", "password"]) {
+        assert.ok(!raw.includes(secret), `log line leaks ${secret}`);
+    }
+}
+test("failed turn records failureDetail kind+message in the probe log", async () => {
+    const before = probeLogSize();
+    const stderr = [];
+    const origError = console.error;
+    console.error = (...args) => {
+        stderr.push(args.map(String).join(" "));
+    };
+    try {
+        const host = stubProbeHost();
+        const result = await runMuseCapacityProbe(Date.now(), { host, timeoutMs: 10_000 });
+        assert.equal(result.ok, false);
+        assert.equal(result.failureKind, "turn_failed");
+        assert.deepEqual(result.failureDetail, { kind: "other", message: "boom" });
+    }
+    finally {
+        console.error = origError;
+    }
+    assert.match(stderr.join("\n"), /fallback failed: turn_failed \(other: boom\)/);
+    const lines = probeLogLinesSince(before);
+    assert.equal(lines.length, 1);
+    assertProbeLogShape(lines[0]);
+    assert.equal(lines[0].failureKind, "turn_failed");
+    assert.deepEqual(lines[0].failureDetail, { kind: "other", message: "boom" });
+});
+test("thrown probe error records failureDetail with the error message", async () => {
+    const before = probeLogSize();
+    const host = stubProbeHost({ failStart: new Error("x") });
+    const result = await runMuseCapacityProbe(Date.now(), { host, timeoutMs: 10_000 });
+    assert.equal(result.ok, false);
+    assert.equal(result.failureKind, "exception");
+    assert.equal(result.failureDetail?.kind, "exception");
+    assert.ok(result.failureDetail?.message.includes("x"));
+    const lines = probeLogLinesSince(before);
+    assert.equal(lines.length, 1);
+    assertProbeLogShape(lines[0]);
+    assert.equal(lines[0].failureKind, "exception");
+    assert.ok(String(lines[0].failureDetail.message).includes("x"));
+});
+test("timed-out turn records the timeout failureDetail", async () => {
+    const before = probeLogSize();
+    const host = stubProbeHost({ hangTurnMs: 400 });
+    const result = await runMuseCapacityProbe(Date.now(), { host, timeoutMs: 200 });
+    assert.equal(result.ok, false);
+    assert.equal(result.failureKind, "timeout");
+    assert.deepEqual(result.failureDetail, {
+        kind: "other",
+        message: "turn timed out and was cancelled",
+    });
+    const lines = probeLogLinesSince(before);
+    assert.equal(lines.length, 1);
+    assertProbeLogShape(lines[0]);
+    assert.equal(lines[0].failureKind, "timeout");
+    assert.deepEqual(lines[0].failureDetail, result.failureDetail);
+});
+test("failureDetail message is truncated to 300 chars with controls stripped", async () => {
+    const before = probeLogSize();
+    const long = `AB\nCD\t${"x".repeat(1000)}\u0000\u001f END`;
+    const host = stubProbeHost({
+        turnParams: {
+            sessionId: "sess-stub-1",
+            terminal: "failed",
+            error: { kind: "other", message: long },
+            durationMs: 5,
+        },
+    });
+    const result = await runMuseCapacityProbe(Date.now(), { host, timeoutMs: 10_000 });
+    assert.equal(result.failureKind, "turn_failed");
+    const message = result.failureDetail?.message ?? "";
+    assert.equal(message.length, 300);
+    assert.ok(!/[\x00-\x1f\x7f]/.test(message));
+    assert.ok(message.startsWith("ABCD"));
+    const lines = probeLogLinesSince(before);
+    assert.equal(lines.length, 1);
+    assertProbeLogShape(lines[0]);
+    assert.equal(lines[0].failureDetail.message, message);
+});
+test("successful probe logs failureDetail null with existing fields unchanged", async () => {
+    const before = probeLogSize();
+    const host = new MuseCapacityHost(hostOpts("serve-turn-full"));
+    try {
+        const result = await runMuseCapacityProbe(Date.now(), { host, timeoutMs: 10_000 });
+        assert.equal(result.ok, true);
+        assert.equal(result.failureDetail, null);
+    }
+    finally {
+        await host.shutdown();
+    }
+    const lines = probeLogLinesSince(before);
+    assert.equal(lines.length, 1);
+    assertProbeLogShape(lines[0]);
+    assert.equal(lines[0].failureDetail, null);
+    assert.equal(lines[0].event, "muse_capacity_probe");
+    assert.equal(lines[0].trigger, "stale_60m");
+    assert.equal(lines[0].failureKind, null);
+    assert.equal(lines[0].ok, true);
+});

package/bundle/server/dist/coordinator/admission.js

@@ -10,6 +10,7 @@
 import { admissionLimitOptions, getEffectiveMaxActiveIssues, getMaxActiveIssues, workerSpawnCeiling, } from "../repository/admission-settings.js";
 import { checkAgentDeckHealth } from "../adapters/agent-deck.js";
 import { healthForAgent } from "../adapters/agent-health.js";
+import { ensureMuseCapabilityEscalation } from "../adapters/muse-capability.js";
 import { getDb } from "../db/index.js";
 import { getAgent } from "../repository/agents.js";
 import { getIssue, listIssues } from "../repository/issues.js";
@@ -150,7 +151,19 @@ export async function checkRoleAgentHealthy(issue, role, ctx) {
     if (!agent)
         return { ok: false, reason: `${role} agent not found` };
     const check = healthChecker ?? ((a, r) => defaultAgentHealth(a, r, ctx.deckOnline));
-    return check(agent, role);
+    const result = await check(agent, role);
+    // NOT-308: a Muse verdict the gate cannot clear by itself (`missing`, or an exhausted
+    // `error`) must reach the operator as a human action on this issue — deduped per
+    // version and verdict kind, so repeated polls never pile up. Best-effort: escalation never fails health.
+    if (role === "developer" && agent.runtime === "muse_code") {
+        try {
+            ensureMuseCapabilityEscalation(issue.id);
+        }
+        catch {
+            // The gate already reported; a failed escalation must not flip it.
+        }
+    }
+    return result;
 }
 async function checkAgentsHealthy(issue, ctx) {
     // Admit always starts a developer round — do not require reviewer health yet (NOT-156).

package/bundle/server/dist/coordinator/admission.test.js

@@ -403,7 +403,10 @@ test("NOT-178: a Muse Code developer without credentials is refused at admission
 });
 // NOT-277: a Muse Code auto-update is re-validated automatically. A version whose developer
 // shell/write check passes is admitted with no manual step; one that lost it is refused with the
-// versions and capability named; one whose check could not complete is refused, never assumed ok.
+// versions and capability named.
+// NOT-308: an inconclusive check never blocks on a confirmed baseline (admitted, retried in
+// the background); a confirmed loss escalates exactly one human action, and acknowledging
+// it lifts the block.
 test("NOT-277: a Muse Code update is admitted when capable, refused by name when not", async () => {
     const { clearAgentHealthCaches, runtimeIssuesUncached } = await import("../adapters/agent-health.js");
     const { setMuseCapabilityProbeForTests, settleMuseCapabilityCheckForTests } = await import("../adapters/muse-capability.js");
@@ -463,11 +466,22 @@ test("NOT-277: a Muse Code update is admitted when capable, refused by name when
         const reason = getQueuedEntryForIssue(issue.id).waitReason;
         assert.match(reason, /developer unhealthy/);
         assert.match(reason, new RegExp(`Muse Code updated ${NEW} → ${BROKEN}: developer sessions no longer get shell/write access`));
+        // NOT-308: the confirmed loss escalates exactly one human action naming the update
+        // and the capability — and acknowledging it lifts the admission block.
+        const { listHumanActionsByRequestId } = await import("../repository/human-actions.js");
+        const { museCapabilityRequestId } = await import("../adapters/muse-capability.js");
+        const { resolveHumanActionAndAdvance } = await import("./commands.js");
+        const escalated = listHumanActionsByRequestId("muse_capability", museCapabilityRequestId(BROKEN, "missing"));
+        assert.equal(escalated.filter((a) => a.status === "open").length, 1);
+        assert.match(escalated[0].reason, new RegExp(`Muse Code updated ${NEW} → ${BROKEN}: developer sessions no longer get shell/write access`));
+        const acknowledged = resolveHumanActionAndAdvance(escalated[0].id, "test", "acknowledge");
+        assert.equal(acknowledged.ok, true);
+        assert.deepEqual(await developerHealth(), { ok: true }, "acknowledge lifts the block");
+        // NOT-308: an inconclusive check (the probe throws) does NOT block on the baseline.
         await museReports("9.9.9-R1");
-        const unverified = await developerHealth();
-        assert.equal(unverified.ok, false);
-        assert.match(unverified.reason, new RegExp(`Could not verify Muse Code developer shell/write access after version change \\(${BROKEN} → 9\\.9\\.9-R1\\)`));
+        assert.deepEqual(await developerHealth(), { ok: true });
         assert.deepEqual(probed, [OLD, NEW, BROKEN, "9.9.9-R1"]);
+        assert.equal(listHumanActionsByRequestId("muse_capability", museCapabilityRequestId("9.9.9-R1", "unverified")).length, 0, "a first inconclusive result escalates nothing");
     }
     finally {
         for (const [key, value] of Object.entries({
@@ -486,6 +500,185 @@ test("NOT-277: a Muse Code update is admitted when capable, refused by name when
         setAdmissionHealthCheckerForTests(async () => ({ ok: true }));
     }
 });
+// NOT-308: real-checker Muse harness — stub `muse --version`, stub gh login, stub the
+// capability probe. Restores every override so later tests keep the unit-test skip.
+async function withMuseHarness(probe, fn) {
+    const { clearAgentHealthCaches, invalidateMuseHealthCache, runtimeIssuesUncached } = await import("../adapters/agent-health.js");
+    const cap = await import("../adapters/muse-capability.js");
+    cap.resetMuseCapabilityStateForTests();
+    const stubDir = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-muse-stub-"));
+    const versionFile = path.join(stubDir, "version");
+    const stub = path.join(stubDir, "muse");
+    fs.writeFileSync(stub, `#!/bin/sh\ncat ${JSON.stringify(versionFile)}\n`);
+    fs.chmodSync(stub, 0o755);
+    const ghStub = path.join(stubDir, "gh");
+    fs.writeFileSync(ghStub, "#!/bin/sh\necho 'Logged in to github.com account dealer-test'\nexit 0\n");
+    fs.chmodSync(ghStub, 0o755);
+    cap.setMuseCapabilityProbeForTests(probe);
+    const saved = {
+        MUSE_CLI: process.env.MUSE_CLI,
+        META_API_KEY: process.env.META_API_KEY,
+        SKIP: process.env.AGENT_DEALER_SKIP_AGENT_HEALTH,
+        PATH: process.env.PATH,
+    };
+    process.env.MUSE_CLI = stub;
+    process.env.PATH = `${stubDir}${path.delimiter}${process.env.PATH ?? ""}`;
+    process.env.META_API_KEY = "k";
+    delete process.env.AGENT_DEALER_SKIP_AGENT_HEALTH;
+    setAdmissionHealthCheckerForTests(null);
+    clearAgentHealthCaches();
+    const writeVersion = (version) => {
+        fs.writeFileSync(versionFile, `Muse Code ${version.split("-")[0]} (${version})\n`);
+    };
+    try {
+        await fn({
+            versionFile,
+            show: writeVersion,
+            report: async (version) => {
+                writeVersion(version);
+                await runtimeIssuesUncached("muse_code");
+                await cap.settleMuseCapabilityCheckForTests();
+            },
+            reportUnsettled: async (version) => {
+                writeVersion(version);
+                // Targeted invalidation only: clearAgentHealthCaches would also reset the
+                // capability state under test.
+                invalidateMuseHealthCache();
+                return runtimeIssuesUncached("muse_code");
+            },
+        });
+    }
+    finally {
+        for (const [key, value] of Object.entries({
+            MUSE_CLI: saved.MUSE_CLI,
+            META_API_KEY: saved.META_API_KEY,
+            AGENT_DEALER_SKIP_AGENT_HEALTH: saved.SKIP,
+            PATH: saved.PATH,
+        })) {
+            if (value === undefined)
+                delete process.env[key];
+            else
+                process.env[key] = value;
+        }
+        cap.setMuseCapabilityProbeForTests(null);
+        clearAgentHealthCaches();
+        setAdmissionHealthCheckerForTests(async () => ({ ok: true }));
+    }
+}
+// NOT-308: while the new version is still being verified, developer admission is allowed
+// on the confirmed baseline — no `runtime_capability` block while the check is in flight.
+test("NOT-308: developer admission is allowed while the new version check is in flight", async () => {
+    const cap = await import("../adapters/muse-capability.js");
+    const OLD = "1.3.0-R3401.1";
+    const NEW = "1.4.0-R4161.1";
+    let release;
+    await withMuseHarness(async (version) => {
+        if (version === NEW) {
+            return new Promise((resolve) => {
+                release = () => resolve({ status: "capable" });
+            });
+        }
+        return { status: "capable" };
+    }, async ({ report, reportUnsettled }) => {
+        await report(OLD);
+        const issues = await reportUnsettled(NEW);
+        assert.deepEqual(issues.filter((i) => i.code === "runtime_capability"), [], "no runtime_capability block while verifying on a baseline");
+        assert.equal(cap.museCapabilityCheckInFlight(), true);
+        // The admission gate agrees: healthy, with the check still running.
+        const issue = readyIssue("muse-inflight", { runtimes: { dev: "muse_code", rev: "codex_local" } });
+        const { invalidateMuseHealthCache } = await import("../adapters/agent-health.js");
+        invalidateMuseHealthCache();
+        assert.deepEqual(await checkRoleAgentHealthy(issue, "developer", { deckOnline: true }), {
+            ok: true,
+        });
+        assert.equal(cap.museCapabilityCheckInFlight(), true);
+        release();
+        await cap.settleMuseCapabilityCheckForTests();
+        invalidateMuseHealthCache();
+        assert.deepEqual(await checkRoleAgentHealthy(issue, "developer", { deckOnline: true }), {
+            ok: true,
+        });
+    });
+});
+// NOT-308: three inconclusive polls escalate exactly one "could not verify" action and stop
+// probing — admission never blocks.
+test("NOT-308: three inconclusive polls escalate once, then stop probing, never blocking", async () => {
+    const cap = await import("../adapters/muse-capability.js");
+    const { invalidateMuseHealthCache } = await import("../adapters/agent-health.js");
+    const { listHumanActionsByRequestId } = await import("../repository/human-actions.js");
+    const OLD = "1.3.0-R3401.1";
+    const NEW = "1.4.0-R4161.1";
+    const probed = [];
+    await withMuseHarness(async (version) => {
+        probed.push(version);
+        if (version === NEW)
+            return { status: "error", detail: "probe session timed out" };
+        return { status: "capable" };
+    }, async ({ report, show }) => {
+        await report(OLD);
+        show(NEW);
+        const issue = readyIssue("muse-unverified", {
+            runtimes: { dev: "muse_code", rev: "codex_local" },
+        });
+        const poll = async () => {
+            // Targeted invalidation only: clearAgentHealthCaches would also reset the
+            // capability attempts under test.
+            invalidateMuseHealthCache();
+            const health = await checkRoleAgentHealthy(issue, "developer", { deckOnline: true });
+            await cap.settleMuseCapabilityCheckForTests();
+            return health;
+        };
+        const { ageMuseCapabilityCheckForTests } = await import("../adapters/muse-capability.js");
+        assert.deepEqual(await poll(), { ok: true }, "attempt 1: allowed");
+        ageMuseCapabilityCheckForTests(61_000);
+        assert.deepEqual(await poll(), { ok: true }, "attempt 2: allowed");
+        ageMuseCapabilityCheckForTests(122_000);
+        assert.deepEqual(await poll(), { ok: true }, "attempt 3: allowed");
+        assert.deepEqual(probed, [OLD, NEW, NEW, NEW]);
+        // The third poll settles exhausted, so the next polls escalate — exactly once.
+        assert.deepEqual(await poll(), { ok: true }, "exhausted: still allowed");
+        assert.deepEqual(await poll(), { ok: true }, "second post-exhaustion poll: still allowed");
+        const actions = listHumanActionsByRequestId("muse_capability", cap.museCapabilityRequestId(NEW, "unverified"));
+        assert.equal(actions.filter((a) => a.status === "open").length, 1);
+        assert.match(actions[0].reason, /Could not verify Muse Code developer shell\/write access/);
+        assert.match(actions[0].reason, new RegExp(`for ${NEW} after 3 inconclusive checks`));
+        // No fourth probe ever, however long the wait.
+        ageMuseCapabilityCheckForTests(10 * 60_000);
+        invalidateMuseHealthCache();
+        assert.deepEqual(await checkRoleAgentHealthy(issue, "developer", { deckOnline: true }), {
+            ok: true,
+        });
+        assert.deepEqual(probed, [OLD, NEW, NEW, NEW], "no further probes after exhaustion");
+    });
+});
+// NOT-308: with no confirmed baseline ever (fresh install), the unknown still fails
+// closed — in flight and errored alike.
+test("NOT-308: with no confirmed baseline, admission stays blocked while unconfirmed", async () => {
+    const cap = await import("../adapters/muse-capability.js");
+    const NEW = "1.4.0-R4161.1";
+    let release;
+    let calls = 0;
+    await withMuseHarness(async () => {
+        calls += 1;
+        return new Promise((resolve) => {
+            release = () => resolve({ status: "error", detail: "timed out" });
+        });
+    }, async ({ reportUnsettled }) => {
+        const issue = readyIssue("muse-fresh", { runtimes: { dev: "muse_code", rev: "codex_local" } });
+        const health = (i) => checkRoleAgentHealthy(i, "developer", { deckOnline: true });
+        await reportUnsettled(NEW);
+        assert.equal(cap.museCapabilityCheckInFlight(), true);
+        const blocked = await health(issue);
+        assert.equal(blocked.ok, false);
+        assert.match(blocked.reason, /verifying developer shell\/write access/);
+        release();
+        await cap.settleMuseCapabilityCheckForTests();
+        const errored = await health(issue);
+        assert.equal(errored.ok, false);
+        assert.match(errored.reason, /Could not verify Muse Code/);
+        assert.equal(calls, 1);
+    });
+});
 // NOT-178: Muse Code is developer-only, so it is never a healthy reviewer — even with the
 // unit-test health skip on, and without probing the CLI.
 test("NOT-178: a Muse Code reviewer is refused regardless of CLI health", async () => {