muonroi-cli 1.3.4 → 1.4.1

package/dist/src/self-qa/index.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Self-QA — public entry point.
+ *
+ * Usage:
+ *   import { runSelfVerify } from "./self-qa/index.js";
+ *   const report = await runSelfVerify({ baseRef: "HEAD~1" });
+ */
+import { judge, summariseResults } from "./judge.js";
+import { runScenarios } from "./orchestrator.js";
+import { planScenarios } from "./scenario-planner.js";
+import { emitSpec } from "./spec-emitter.js";
+import type { JudgeResult, Scenario, ScenarioRun } from "./types.js";
+export type SelfVerifyOptions = {
+    baseRef?: string;
+    cwd?: string;
+    maxScenarios?: number;
+    mockLlmDir?: string;
+    /** Emit .spec.ts for every passing scenario. Default: true. */
+    emitSpecs?: boolean;
+    /** Override spec output dir. */
+    specOutDir?: string;
+    /** Override file list (skip git diff). */
+    diffFilesOverride?: string[];
+    /** Optional logger. */
+    log?: (msg: string) => void;
+};
+export type SelfVerifyReport = {
+    scenarios: Scenario[];
+    runs: ScenarioRun[];
+    results: JudgeResult[];
+    summary: ReturnType<typeof summariseResults>;
+    emittedSpecs: string[];
+    /** Wall-clock duration of the entire batch. */
+    durationMs: number;
+};
+export declare function runSelfVerify(opts?: SelfVerifyOptions): Promise<SelfVerifyReport>;
+export type { AgenticContextBlock, AgenticContextOptions } from "./agentic-context.js";
+export { buildAgenticContext } from "./agentic-context.js";
+export type { AgenticBrain, AgenticDecision, AgenticLoopOptions, AgenticReport, AgenticTurn, LLMBrainOptions, } from "./agentic-loop.js";
+export { createLLMBrain, createMockBrain, parseDecision, runAgenticLoop } from "./agentic-loop.js";
+export { applyDelta, compressionRatio, encodeDelta } from "./delta-encoder.js";
+export type { CheckResult, Expectation, FrameDelta, JudgeResult, JudgeVerdict, Scenario, ScenarioRun, ScenarioStep, SemanticHit, } from "./types.js";
+export { emitSpec, judge, planScenarios, runScenarios, summariseResults };

package/dist/src/self-qa/index.js

@@ -0,0 +1,65 @@
+/**
+ * Self-QA — public entry point.
+ *
+ * Usage:
+ *   import { runSelfVerify } from "./self-qa/index.js";
+ *   const report = await runSelfVerify({ baseRef: "HEAD~1" });
+ */
+import { judge, summariseResults } from "./judge.js";
+import { runScenarios } from "./orchestrator.js";
+import { planScenarios } from "./scenario-planner.js";
+import { emitSpec } from "./spec-emitter.js";
+export async function runSelfVerify(opts = {}) {
+    const log = opts.log ?? (() => { });
+    const t0 = Date.now();
+    const plannerOpts = {
+        baseRef: opts.baseRef,
+        cwd: opts.cwd,
+        maxScenarios: opts.maxScenarios,
+        diffFilesOverride: opts.diffFilesOverride,
+    };
+    const scenarios = planScenarios(plannerOpts);
+    log(`[self-verify] Planned ${scenarios.length} scenario(s)`);
+    const orchOpts = {
+        mockLlmDir: opts.mockLlmDir,
+        log,
+    };
+    const runs = await runScenarios(scenarios, orchOpts);
+    const results = runs.map(judge);
+    const summary = summariseResults(results);
+    log(`[self-verify] Summary: ${summary.passed}/${summary.total} passed, ` +
+        `${summary.failed} failed, ${summary.inconclusive} inconclusive`);
+    const emittedSpecs = [];
+    if (opts.emitSpecs !== false) {
+        for (let i = 0; i < results.length; i++) {
+            const result = results[i];
+            const scenario = scenarios[i];
+            if (!result || !scenario)
+                continue;
+            if (result.verdict !== "pass")
+                continue;
+            try {
+                const path = emitSpec(scenario, result, { outDir: opts.specOutDir });
+                emittedSpecs.push(path);
+                log(`[self-verify] Emitted spec: ${path}`);
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                log(`[self-verify] Spec emit failed for ${scenario.id}: ${msg}`);
+            }
+        }
+    }
+    return {
+        scenarios,
+        runs,
+        results,
+        summary,
+        emittedSpecs,
+        durationMs: Date.now() - t0,
+    };
+}
+export { buildAgenticContext } from "./agentic-context.js";
+export { createLLMBrain, createMockBrain, parseDecision, runAgenticLoop } from "./agentic-loop.js";
+export { applyDelta, compressionRatio, encodeDelta } from "./delta-encoder.js";
+export { emitSpec, judge, planScenarios, runScenarios, summariseResults };
+//# sourceMappingURL=index.js.map

package/dist/src/self-qa/judge.d.ts

@@ -0,0 +1,25 @@
+/**
+ * judge.ts — M4 of Self-QA.
+ *
+ * Pure function: takes a ScenarioRun (events + final frame) and a list of
+ * Expectations, returns a JudgeVerdict + per-expectation reasoning.
+ *
+ * Rule-based — NO LLM call. Reproducible and cheap.
+ *
+ * Verdict policy:
+ *   - All checks passed              → "pass"
+ *   - One or more checks failed      → "fail"
+ *   - Run crashed / timed out        → "inconclusive"  (don't claim pass/fail)
+ */
+import type { JudgeResult, ScenarioRun } from "./types.js";
+export declare function judge(run: ScenarioRun): JudgeResult;
+/**
+ * Convenience for the orchestrator/CLI to summarise a batch of judgments.
+ */
+export declare function summariseResults(results: JudgeResult[]): {
+    total: number;
+    passed: number;
+    failed: number;
+    inconclusive: number;
+    passRate: number;
+};

package/dist/src/self-qa/judge.js

@@ -0,0 +1,233 @@
+/**
+ * judge.ts — M4 of Self-QA.
+ *
+ * Pure function: takes a ScenarioRun (events + final frame) and a list of
+ * Expectations, returns a JudgeVerdict + per-expectation reasoning.
+ *
+ * Rule-based — NO LLM call. Reproducible and cheap.
+ *
+ * Verdict policy:
+ *   - All checks passed              → "pass"
+ *   - One or more checks failed      → "fail"
+ *   - Run crashed / timed out        → "inconclusive"  (don't claim pass/fail)
+ */
+import { matchSelector } from "@muonroi/agent-harness-core/selector";
+export function judge(run) {
+    const durationMs = run.endedAt - run.startedAt;
+    if (run.crashed) {
+        return {
+            verdict: "inconclusive",
+            scenarioId: run.scenario.id,
+            durationMs,
+            checks: [
+                {
+                    expectation: { kind: "idleReached" },
+                    passed: false,
+                    reason: `Child process crashed before scenario completed: ${run.errorTrace ?? "unknown"}`,
+                },
+            ],
+        };
+    }
+    if (run.timedOut) {
+        return {
+            verdict: "inconclusive",
+            scenarioId: run.scenario.id,
+            durationMs,
+            checks: [
+                {
+                    expectation: { kind: "idleReached" },
+                    passed: false,
+                    reason: `Scenario exceeded budget of ${run.scenario.budgetMs}ms`,
+                },
+            ],
+        };
+    }
+    const checks = [];
+    for (const exp of run.scenario.expectations) {
+        checks.push(evaluate(exp, run));
+    }
+    const allPassed = checks.every((c) => c.passed);
+    return {
+        verdict: allPassed ? "pass" : "fail",
+        scenarioId: run.scenario.id,
+        durationMs,
+        checks,
+    };
+}
+function evaluate(exp, run) {
+    switch (exp.kind) {
+        case "noErrorToast":
+            return checkNoErrorToast(exp, run.events);
+        case "eventFired":
+            return checkEventFired(exp, run.events);
+        case "eventAbsent":
+            return checkEventAbsent(exp, run.events);
+        case "selectorPresent":
+            return checkSelectorPresent(exp, run.finalFrame);
+        case "selectorAbsent":
+            return checkSelectorAbsent(exp, run.finalFrame);
+        case "idleReached":
+            return checkIdleReached(exp, run);
+    }
+}
+function checkNoErrorToast(exp, events) {
+    const errorToasts = events.filter((e) => e.t === "event" && e.kind === "toast" && e.level === "error");
+    if (errorToasts.length === 0) {
+        return { expectation: exp, passed: true, reason: "No error-level toasts observed" };
+    }
+    const sample = errorToasts[0];
+    const text = sample && "text" in sample ? sample.text : "<no text>";
+    return {
+        expectation: exp,
+        passed: false,
+        reason: `Found ${errorToasts.length} error toast(s); first: ${text}`,
+    };
+}
+function checkEventFired(exp, events) {
+    if (exp.kind !== "eventFired")
+        throw new Error("invariant");
+    const matches = events.filter((e) => e.t === "event" && e.kind === exp.event);
+    if (matches.length === 0) {
+        return {
+            expectation: exp,
+            passed: false,
+            reason: `Event '${exp.event}' was never emitted`,
+        };
+    }
+    if (!exp.payloadMatch) {
+        return { expectation: exp, passed: true, reason: `Event '${exp.event}' fired ${matches.length}×` };
+    }
+    const hit = matches.find((e) => payloadMatches(e, exp.payloadMatch));
+    if (hit) {
+        return {
+            expectation: exp,
+            passed: true,
+            reason: `Event '${exp.event}' fired with matching payload`,
+        };
+    }
+    return {
+        expectation: exp,
+        passed: false,
+        reason: `Event '${exp.event}' fired but payload did not match expected fields`,
+    };
+}
+function checkEventAbsent(exp, events) {
+    if (exp.kind !== "eventAbsent")
+        throw new Error("invariant");
+    const hit = events.find((e) => e.t === "event" && e.kind === exp.event);
+    if (hit) {
+        return {
+            expectation: exp,
+            passed: false,
+            reason: `Event '${exp.event}' was emitted but expected absent`,
+        };
+    }
+    return { expectation: exp, passed: true, reason: `Event '${exp.event}' correctly absent` };
+}
+function checkSelectorPresent(exp, frame) {
+    if (exp.kind !== "selectorPresent")
+        throw new Error("invariant");
+    if (!frame) {
+        return {
+            expectation: exp,
+            passed: false,
+            reason: "No final frame captured — cannot verify selector presence",
+        };
+    }
+    const hits = findBySelector(frame, exp.selector);
+    if (hits.length > 0) {
+        return {
+            expectation: exp,
+            passed: true,
+            reason: `Selector '${exp.selector}' matched ${hits.length} node(s)`,
+        };
+    }
+    return {
+        expectation: exp,
+        passed: false,
+        reason: `Selector '${exp.selector}' matched 0 nodes`,
+    };
+}
+function checkSelectorAbsent(exp, frame) {
+    if (exp.kind !== "selectorAbsent")
+        throw new Error("invariant");
+    if (!frame) {
+        return { expectation: exp, passed: true, reason: "No frame; treating as absent" };
+    }
+    const hits = findBySelector(frame, exp.selector);
+    if (hits.length === 0) {
+        return { expectation: exp, passed: true, reason: `Selector '${exp.selector}' correctly absent` };
+    }
+    return {
+        expectation: exp,
+        passed: false,
+        reason: `Selector '${exp.selector}' matched ${hits.length} node(s) but expected none`,
+    };
+}
+function checkIdleReached(exp, run) {
+    if (exp.kind !== "idleReached")
+        throw new Error("invariant");
+    const duration = run.endedAt - run.startedAt;
+    const budget = exp.withinMs ?? run.scenario.budgetMs;
+    // Idle is signalled by the harness via `{ t: "idle" }` — driver._ingest
+    // converts it to a non-event sentinel. We can also infer idle when the
+    // scenario completed without timeout/crash.
+    if (run.timedOut || run.crashed) {
+        return {
+            expectation: exp,
+            passed: false,
+            reason: "Run did not finish cleanly — idle not reached",
+        };
+    }
+    if (duration <= budget) {
+        return {
+            expectation: exp,
+            passed: true,
+            reason: `Idle reached in ${duration}ms (budget ${budget}ms)`,
+        };
+    }
+    return {
+        expectation: exp,
+        passed: false,
+        reason: `Run finished in ${duration}ms which exceeds idle budget ${budget}ms`,
+    };
+}
+function payloadMatches(event, expected) {
+    // biome-ignore lint/suspicious/noExplicitAny: event is heterogeneous union
+    const e = event;
+    for (const [k, v] of Object.entries(expected)) {
+        if (e[k] !== v)
+            return false;
+    }
+    return true;
+}
+function findBySelector(frame, selector) {
+    const out = [];
+    const visit = (nodes) => {
+        for (const n of nodes) {
+            if (matchSelector(n, selector))
+                out.push(n);
+            if (n.children)
+                visit(n.children);
+        }
+    };
+    visit(frame.nodes);
+    return out;
+}
+/**
+ * Convenience for the orchestrator/CLI to summarise a batch of judgments.
+ */
+export function summariseResults(results) {
+    const total = results.length;
+    const passed = results.filter((r) => r.verdict === "pass").length;
+    const failed = results.filter((r) => r.verdict === "fail").length;
+    const inconclusive = results.filter((r) => r.verdict === "inconclusive").length;
+    return {
+        total,
+        passed,
+        failed,
+        inconclusive,
+        passRate: total === 0 ? 0 : passed / total,
+    };
+}
+//# sourceMappingURL=judge.js.map

package/dist/src/self-qa/orchestrator.d.ts

@@ -0,0 +1,31 @@
+/**
+ * orchestrator.ts — M2 of Self-QA.
+ *
+ * Spawns an inner muonroi-cli via the agent-harness, drives each Scenario
+ * deterministically with the Driver API, and returns ScenarioRun[] that
+ * the judge can score.
+ *
+ * Design notes:
+ *   - The inner instance MUST run with --mock-llm so scenarios are reproducible
+ *     and free. Real LLM verification is a future opt-in (set realLlm: true).
+ *   - We close the child after the scenario batch finishes (or on first crash).
+ *     Each scenario does NOT spawn its own child — the cost would dominate.
+ *   - All event capture goes through driver.events() so the ring buffer's
+ *     late-subscribe replay covers events emitted between spawn and subscribe.
+ */
+import type { Scenario, ScenarioRun } from "./types.js";
+export type OrchestratorOptions = {
+    /** Path to muonroi-cli entry file. Default: resolved src/index.ts of this repo. */
+    entry?: string;
+    /** Path to mock-llm fixture dir. Default: tests/harness/fixtures/llm. */
+    mockLlmDir?: string;
+    /** Extra CLI args appended after --agent-mode --mock-llm <dir>. */
+    extraArgs?: string[];
+    /** Env vars merged with process.env. */
+    env?: Record<string, string>;
+    /** Hard cap on total batch runtime. Default: 5 minutes. */
+    batchBudgetMs?: number;
+    /** Optional logger — receives short status strings. */
+    log?: (msg: string) => void;
+};
+export declare function runScenarios(scenarios: Scenario[], opts?: OrchestratorOptions): Promise<ScenarioRun[]>;

package/dist/src/self-qa/orchestrator.js

@@ -0,0 +1,236 @@
+/**
+ * orchestrator.ts — M2 of Self-QA.
+ *
+ * Spawns an inner muonroi-cli via the agent-harness, drives each Scenario
+ * deterministically with the Driver API, and returns ScenarioRun[] that
+ * the judge can score.
+ *
+ * Design notes:
+ *   - The inner instance MUST run with --mock-llm so scenarios are reproducible
+ *     and free. Real LLM verification is a future opt-in (set realLlm: true).
+ *   - We close the child after the scenario batch finishes (or on first crash).
+ *     Each scenario does NOT spawn its own child — the cost would dominate.
+ *   - All event capture goes through driver.events() so the ring buffer's
+ *     late-subscribe replay covers events emitted between spawn and subscribe.
+ */
+import { resolve } from "node:path";
+import { createDriver } from "@muonroi/agent-harness-core/driver";
+import { createLineSplitter } from "@muonroi/agent-harness-core/transports/sidechannel";
+import { spawnAgentTui } from "../agent-harness/test-spawn.js";
+export async function runScenarios(scenarios, opts = {}) {
+    const log = opts.log ?? (() => { });
+    const entry = opts.entry ?? resolve("src/index.ts");
+    const mockDir = opts.mockLlmDir ?? resolve("tests/harness/fixtures/llm");
+    const batchBudget = opts.batchBudgetMs ?? 5 * 60_000;
+    const batchStart = Date.now();
+    if (scenarios.length === 0) {
+        log("[self-qa] No scenarios to run");
+        return [];
+    }
+    log(`[self-qa] Spawning child: ${entry} (mock-llm: ${mockDir})`);
+    const args = [entry, "--agent-mode", "--mock-llm", mockDir, ...(opts.extraArgs ?? [])];
+    const env = {
+        ...process.env,
+        MUONROI_TEST_NO_PERSIST: "1",
+        MUONROI_INTERNAL_SHIM_OK: "1",
+        ...(opts.env ?? {}),
+    };
+    let spawnResult;
+    try {
+        spawnResult = await spawnAgentTui(args, { spawnOpts: { env } });
+    }
+    catch (err) {
+        const trace = err instanceof Error ? (err.stack ?? err.message) : String(err);
+        log(`[self-qa] Spawn failed: ${trace}`);
+        return scenarios.map((s) => crashedRun(s, trace));
+    }
+    const { proc, inWrite, outRead, cleanup } = spawnResult;
+    const driver = wireDriver(inWrite, outRead);
+    const eventBus = [];
+    attachEventCollector(driver, eventBus);
+    let childCrashed = false;
+    let crashTrace;
+    proc.on("exit", (code, signal) => {
+        if (code !== null && code !== 0) {
+            childCrashed = true;
+            crashTrace = `child exited code=${code} signal=${signal ?? "none"}`;
+        }
+        driver._closeAllSubscribers();
+    });
+    const runs = [];
+    try {
+        // Wait for the TUI to become idle before driving the first scenario.
+        await safeWait(() => driver.wait_for({ idle: true, timeoutMs: 15_000 }));
+        for (const scenario of scenarios) {
+            if (Date.now() - batchStart > batchBudget) {
+                log(`[self-qa] Batch budget exhausted — marking remaining as timed-out`);
+                for (const remaining of scenarios.slice(runs.length)) {
+                    runs.push(timedOutRun(remaining));
+                }
+                break;
+            }
+            log(`[self-qa] → ${scenario.id}: ${scenario.description}`);
+            const before = eventBus.length;
+            const startedAt = Date.now();
+            let timedOut = false;
+            let errorTrace;
+            try {
+                await driveScenario(driver, scenario);
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                if (msg.includes("timeout") || msg.includes("wait_for")) {
+                    timedOut = true;
+                }
+                errorTrace = msg;
+            }
+            const endedAt = Date.now();
+            const sliced = eventBus.slice(before);
+            const finalFrame = driver.snapshot();
+            runs.push({
+                scenario,
+                events: sliced,
+                finalFrame,
+                startedAt,
+                endedAt,
+                timedOut,
+                crashed: childCrashed,
+                errorTrace: childCrashed ? crashTrace : errorTrace,
+            });
+            if (childCrashed) {
+                log(`[self-qa] Child crashed — aborting remaining scenarios`);
+                for (const remaining of scenarios.slice(runs.length)) {
+                    runs.push(crashedRun(remaining, crashTrace ?? "child crashed"));
+                }
+                break;
+            }
+        }
+    }
+    finally {
+        try {
+            proc.kill();
+        }
+        catch {
+            // ignore
+        }
+        cleanup();
+    }
+    return runs;
+}
+function wireDriver(inWrite, outRead) {
+    const driver = createDriver({
+        sendKey: (k) => inWrite.write(JSON.stringify({ op: "press", key: k }) + "\n"),
+        sendType: (t) => inWrite.write(JSON.stringify({ op: "type", text: t }) + "\n"),
+    });
+    const splitter = createLineSplitter((line) => {
+        try {
+            const msg = JSON.parse(line);
+            if (msg["mode"] === "live") {
+                driver._ingest({ kind: "frame", frame: msg });
+            }
+            else if (msg["t"] === "idle") {
+                driver._ingest({ kind: "idle" });
+            }
+            else if (msg["t"] === "event") {
+                driver._ingest({ kind: "event", event: msg });
+            }
+        }
+        catch {
+            // Ignore malformed lines — sidechannel may emit partial chunks.
+        }
+    });
+    outRead.on("data", (chunk) => {
+        splitter(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
+    });
+    return driver;
+}
+function attachEventCollector(driver, bus) {
+    // Buffer every typed event so per-scenario slices can be cut by index.
+    // We listen via the async iterable so late-subscribe replay applies.
+    void (async () => {
+        try {
+            for await (const e of driver.events()) {
+                bus.push(e);
+            }
+        }
+        catch {
+            // iterator terminated cleanly when driver closed.
+        }
+    })();
+}
+async function driveScenario(driver, scenario) {
+    for (const step of scenario.steps) {
+        await runStep(driver, step, scenario.budgetMs);
+    }
+}
+async function runStep(driver, step, budgetMs) {
+    switch (step.op) {
+        case "type":
+            driver.type(step.text);
+            return;
+        case "press":
+            driver.press(step.key);
+            return;
+        case "press_sequence":
+            driver.press_sequence(step.keys);
+            return;
+        case "focus":
+            try {
+                driver.focus(step.selector);
+            }
+            catch {
+                // Focus may throw if selector is ambiguous or missing — non-fatal in
+                // probe scenarios; judge will detect via selectorPresent if needed.
+            }
+            return;
+        case "wait_for": {
+            const timeout = step.timeoutMs ?? Math.min(budgetMs, 5_000);
+            if (step.idle) {
+                await driver.wait_for({ idle: true, timeoutMs: timeout });
+                return;
+            }
+            if (step.selector) {
+                await driver.wait_for({ selector: step.selector, timeoutMs: timeout });
+                return;
+            }
+            if (step.event) {
+                await driver.wait_for({ event: step.event, timeoutMs: timeout });
+                return;
+            }
+            return;
+        }
+    }
+}
+async function safeWait(fn) {
+    try {
+        await fn();
+    }
+    catch {
+        // Initial idle may fail on slow boot — downstream scenarios handle their
+        // own wait_for so this is best-effort.
+    }
+}
+function crashedRun(scenario, trace) {
+    return {
+        scenario,
+        events: [],
+        finalFrame: null,
+        startedAt: Date.now(),
+        endedAt: Date.now(),
+        timedOut: false,
+        crashed: true,
+        errorTrace: trace,
+    };
+}
+function timedOutRun(scenario) {
+    return {
+        scenario,
+        events: [],
+        finalFrame: null,
+        startedAt: Date.now(),
+        endedAt: Date.now(),
+        timedOut: true,
+        crashed: false,
+    };
+}
+//# sourceMappingURL=orchestrator.js.map

package/dist/src/self-qa/scenario-planner.d.ts

@@ -0,0 +1,39 @@
+/**
+ * scenario-planner.ts — M1 of Self-QA.
+ *
+ * Reads `git diff` against a base ref + scans the source tree for
+ * `<Semantic id="..." role="..." ...>` wrappers, then emits Scenario[]
+ * describing how to drive the TUI to exercise the touched UI surface.
+ *
+ * Heuristic-based (no LLM call). Each scenario is tied to a small number
+ * of semantic IDs so judge.ts can assert on exactly those.
+ */
+import type { Scenario, SemanticHit } from "./types.js";
+export type PlannerOptions = {
+    /** Git ref to diff against. Default: HEAD~1. */
+    baseRef?: string;
+    /** Repo root. Default: process.cwd(). */
+    cwd?: string;
+    /** Cap scenarios produced. Default: 8. */
+    maxScenarios?: number;
+    /** Force include these files even if not in diff. */
+    extraFiles?: string[];
+    /** Override the diff list (for tests / dry-runs). */
+    diffFilesOverride?: string[];
+};
+/**
+ * Plan scenarios from a git diff window.
+ */
+export declare function planScenarios(opts?: PlannerOptions): Scenario[];
+/**
+ * Get the list of files touched by `git diff <baseRef>`.
+ * Returns repo-relative POSIX-style paths.
+ */
+export declare function collectChangedFiles(args: {
+    cwd: string;
+    baseRef: string;
+}): string[];
+/**
+ * Extract every `<Semantic id="..." role="..." ...>` occurrence from a source file.
+ */
+export declare function extractSemanticHits(src: string, file: string): SemanticHit[];