@alignfirst/openclaw-test 0.20.0

package/dist/mock-cli-server.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Runner-side HTTP endpoint that the gateway-side shim calls.
+ *
+ *   POST /mock-cli/invoke
+ *     { cli, argv, cwd, stdin }
+ *   → { stdout, stderr, exitCode }
+ *
+ * A single in-flight registry is bound by the runner for the lifetime of one
+ * scenario. Per-conversation handler registries are populated by `ctx.mockCli`.
+ */
+import type { CliMockCall, CliMockHandler } from "./report.js";
+export declare const MOCK_CLI_PORT = 43124;
+export interface ConversationRegistry {
+    conversationId: string;
+    handlers: Map<string, CliMockHandler>;
+    /** Emit a `cliMock` event into the scenario's event stream. */
+    emitCliMock: (call: CliMockCall) => void;
+    /**
+     * `true` once the scenario has called `ctx.markScenarioAsEnded`. After that,
+     * the server bypasses handler dispatch: every incoming call is answered with
+     * the scenario-ended stub message so a lingering agent — still running while
+     * we move on — can read it and stop.
+     */
+    isScenarioEnded: () => boolean;
+}
+export interface MockCliServer {
+    /** Bind the in-flight conversation registry for the lifetime of one scenario. */
+    bind(reg: ConversationRegistry): void;
+    /**
+     * Release the binding. Waits for the gateway side to go quiet (no incoming
+     * call for `RELEASE_QUIET_MS`) before clearing, so any straggler from the
+     * just-ended scenario lands on its own registry and gets the scenario-ended
+     * stub — instead of leaking into the next scenario's bind window as a
+     * spurious `UnexpectedCall`. Bounded by `RELEASE_HARD_TIMEOUT_MS`.
+     */
+    release(): Promise<void>;
+    /** Stop the HTTP listener. */
+    close(): Promise<void>;
+}
+export declare function startMockCliServer(): MockCliServer;

package/dist/mock-cli-server.js

@@ -0,0 +1,194 @@
+/**
+ * Runner-side HTTP endpoint that the gateway-side shim calls.
+ *
+ *   POST /mock-cli/invoke
+ *     { cli, argv, cwd, stdin }
+ *   → { stdout, stderr, exitCode }
+ *
+ * A single in-flight registry is bound by the runner for the lifetime of one
+ * scenario. Per-conversation handler registries are populated by `ctx.mockCli`.
+ */
+import { createServer } from "node:http";
+import { Readable, Writable } from "node:stream";
+export const MOCK_CLI_PORT = 43124;
+const SCENARIO_ENDED_STUB_MESSAGE = "Stubbed call. If you see this, it means that you are in a test scenario. You should stop and acknowledge to the user.\n";
+const RELEASE_QUIET_MS = 500;
+const RELEASE_HARD_TIMEOUT_MS = 5_000;
+const RELEASE_POLL_MS = 100;
+export function startMockCliServer() {
+    let current;
+    let lastCallAtMs = 0;
+    const server = createServer(async (req, res) => {
+        if (req.method !== "POST" || req.url !== "/mock-cli/invoke") {
+            res.statusCode = 404;
+            res.end("not found");
+            return;
+        }
+        lastCallAtMs = Date.now();
+        await handleInvoke(req, res, current);
+    });
+    server.listen(MOCK_CLI_PORT, "0.0.0.0", () => {
+        console.log(`mock-cli server listening on 0.0.0.0:${MOCK_CLI_PORT}`);
+    });
+    return {
+        bind: (reg) => {
+            current = reg;
+        },
+        release: async () => {
+            const deadline = Date.now() + RELEASE_HARD_TIMEOUT_MS;
+            while (lastCallAtMs > 0 && Date.now() - lastCallAtMs < RELEASE_QUIET_MS) {
+                if (Date.now() >= deadline)
+                    break;
+                await new Promise((r) => setTimeout(r, RELEASE_POLL_MS));
+            }
+            current = undefined;
+            lastCallAtMs = 0;
+        },
+        close: () => new Promise((resolve) => {
+            server.close(() => resolve());
+            // Force-destroy lingering keep-alive sockets. A mocked CLI is invoked through the
+            // gateway shim over HTTP keep-alive; after the call the socket stays idle-open, and a
+            // bare `server.close()` waits for it indefinitely — the runner then hangs past the
+            // verdict instead of exiting. `closeAllConnections` lets `close()` complete at once.
+            server.closeAllConnections();
+        }),
+    };
+}
+async function handleInvoke(req, res, reg) {
+    let body;
+    try {
+        body = await readJsonBody(req);
+    }
+    catch (err) {
+        res.statusCode = 400;
+        res.end(`bad json: ${err.message}`);
+        return;
+    }
+    const invoke = normalizeInvoke(body);
+    if (!invoke) {
+        res.statusCode = 400;
+        res.end("missing cli");
+        return;
+    }
+    if (!reg) {
+        respondMissingScenario(res);
+        return;
+    }
+    const startedAt = new Date().toISOString();
+    const startedAtMs = Date.now();
+    if (reg.isScenarioEnded()) {
+        // Verdict is already decided. Don't dispatch to handlers and don't surface
+        // failures — just give the (still-running) agent a clear stop signal.
+        emitCallEvent(reg, invoke, { stdout: SCENARIO_ENDED_STUB_MESSAGE, stderr: "", exitCode: 0 }, startedAt, startedAtMs);
+        respondJson(res, { stdout: SCENARIO_ENDED_STUB_MESSAGE, stderr: "", exitCode: 0 });
+        return;
+    }
+    const handler = reg.handlers.get(invoke.cli);
+    if (!handler) {
+        const stderr = `mock-cli: unexpected call to ${invoke.cli}\n`;
+        emitCallEvent(reg, invoke, {
+            stdout: "",
+            stderr,
+            exitCode: 1,
+            handlerError: { name: "UnexpectedCall", message: `unexpected call to ${invoke.cli}` },
+        }, startedAt, startedAtMs);
+        respondJson(res, { stdout: "", stderr, exitCode: 1, missing: true });
+        return;
+    }
+    const outcome = await runHandler(handler, invoke);
+    emitCallEvent(reg, invoke, outcome, startedAt, startedAtMs);
+    respondJson(res, { stdout: outcome.stdout, stderr: outcome.stderr, exitCode: outcome.exitCode });
+}
+function normalizeInvoke(body) {
+    const cli = String(body.cli ?? "");
+    if (!cli)
+        return;
+    return {
+        cli,
+        argv: Array.isArray(body.argv) ? body.argv.map(String) : [],
+        cwd: String(body.cwd ?? ""),
+        stdin: String(body.stdin ?? ""),
+    };
+}
+async function runHandler(handler, invoke) {
+    const stdinStream = Readable.from([Buffer.from(invoke.stdin, "utf8")]);
+    const outCap = collectStream();
+    const errCap = collectStream();
+    const args = {
+        argv: invoke.argv,
+        cwd: invoke.cwd,
+        stdin: stdinStream,
+        stdout: outCap.writable,
+        stderr: errCap.writable,
+    };
+    let exitCode = 0;
+    let handlerError;
+    try {
+        const result = await handler(args);
+        exitCode = typeof result === "number" ? result : 0;
+    }
+    catch (err) {
+        const e = err;
+        handlerError = {
+            name: e?.name ?? "Error",
+            message: e?.message ?? String(err),
+            stack: e?.stack,
+        };
+        exitCode = 1;
+    }
+    outCap.writable.end();
+    errCap.writable.end();
+    return { stdout: outCap.read(), stderr: errCap.read(), exitCode, handlerError };
+}
+function emitCallEvent(reg, invoke, outcome, startedAt, startedAtMs) {
+    const call = {
+        cli: invoke.cli,
+        argv: invoke.argv,
+        cwd: invoke.cwd,
+        stdin: invoke.stdin,
+        stdout: outcome.stdout,
+        stderr: outcome.stderr,
+        exitCode: outcome.exitCode,
+        startedAt,
+        durationMs: Date.now() - startedAtMs,
+        ...(outcome.handlerError ? { handlerError: outcome.handlerError } : {}),
+    };
+    reg.emitCliMock(call);
+}
+function respondMissingScenario(res) {
+    const stderr = "mock-cli: no scenario currently owns this gateway\n";
+    respondJson(res, { stdout: "", stderr, exitCode: 1, missing: true });
+}
+function respondJson(res, body) {
+    res.setHeader("content-type", "application/json");
+    res.end(JSON.stringify(body));
+}
+function readJsonBody(req) {
+    return new Promise((resolve, reject) => {
+        const chunks = [];
+        req.on("data", (c) => chunks.push(c));
+        req.on("end", () => {
+            const raw = Buffer.concat(chunks).toString("utf8");
+            try {
+                resolve(raw ? JSON.parse(raw) : {});
+            }
+            catch (err) {
+                reject(err);
+            }
+        });
+        req.on("error", reject);
+    });
+}
+function collectStream() {
+    const chunks = [];
+    const writable = new Writable({
+        write(chunk, _enc, cb) {
+            chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
+            cb();
+        },
+    });
+    return {
+        writable,
+        read: () => Buffer.concat(chunks).toString("utf8"),
+    };
+}

package/dist/mock-cli-shim.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Mock-CLI shim. Consumers symlink it under `/opt/openclaw-test/mocks/bin/` for
+ * whichever commands they want to intercept (typically `claude`, `gh`, `glab`,
+ * etc.). Determines its invoked name from argv[1] basename, POSTs the call to
+ * the runner's /mock-cli/invoke endpoint, and replays the response locally.
+ */
+export {};

package/dist/mock-cli-shim.js

@@ -0,0 +1,89 @@
+/**
+ * Mock-CLI shim. Consumers symlink it under `/opt/openclaw-test/mocks/bin/` for
+ * whichever commands they want to intercept (typically `claude`, `gh`, `glab`,
+ * etc.). Determines its invoked name from argv[1] basename, POSTs the call to
+ * the runner's /mock-cli/invoke endpoint, and replays the response locally.
+ */
+import { request as httpRequest } from "node:http";
+import { basename } from "node:path";
+import { URL } from "node:url";
+function die(msg, code = 127) {
+    process.stderr.write(`mock-cli shim: ${msg}\n`);
+    process.exit(code);
+}
+async function readStdin() {
+    if (process.stdin.isTTY)
+        return "";
+    const chunks = [];
+    for await (const chunk of process.stdin) {
+        chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
+    }
+    return Buffer.concat(chunks).toString("utf8");
+}
+function post(urlStr, body) {
+    const url = new URL(urlStr);
+    return new Promise((resolve, reject) => {
+        const req = httpRequest({
+            method: "POST",
+            hostname: url.hostname,
+            port: url.port || 80,
+            path: url.pathname + url.search,
+            headers: {
+                "content-type": "application/json",
+                "content-length": Buffer.byteLength(body),
+            },
+        }, (res) => {
+            const chunks = [];
+            res.on("data", (c) => chunks.push(c));
+            res.on("end", () => {
+                const raw = Buffer.concat(chunks).toString("utf8");
+                if (res.statusCode !== 200) {
+                    reject(new Error(`runner returned HTTP ${res.statusCode}: ${raw}`));
+                    return;
+                }
+                try {
+                    resolve(JSON.parse(raw));
+                }
+                catch (err) {
+                    reject(new Error(`invalid JSON response: ${err.message}`));
+                }
+            });
+            res.on("error", reject);
+        });
+        req.on("error", reject);
+        req.write(body);
+        req.end();
+    });
+}
+async function main() {
+    // The sh wrapper at /opt/openclaw-test/mocks/bin/mock-cli-shim invokes us as
+    //   exec node mock-cli-shim.js "$0" "$@"
+    // so argv[2] is the symlink path used to call us (e.g. /opt/openclaw-test/mocks/bin/git)
+    // and argv.slice(3) is the original argv tail.
+    const invokedAs = basename(process.argv[2] ?? "");
+    if (!invokedAs)
+        die("could not determine invoked binary name from argv[2]");
+    const runnerUrl = process.env.OPENCLAW_TEST_RUNNER_URL;
+    if (!runnerUrl)
+        die("OPENCLAW_TEST_RUNNER_URL is not set");
+    const stdin = await readStdin();
+    const body = JSON.stringify({
+        cli: invokedAs,
+        argv: process.argv.slice(3),
+        cwd: process.cwd(),
+        stdin,
+    });
+    let res;
+    try {
+        res = await post(`${runnerUrl}/mock-cli/invoke`, body);
+    }
+    catch (err) {
+        die(`POST ${runnerUrl}/mock-cli/invoke failed: ${err.message}`);
+    }
+    if (res.stdout)
+        process.stdout.write(res.stdout);
+    if (res.stderr)
+        process.stderr.write(res.stderr);
+    process.exit(typeof res.exitCode === "number" ? res.exitCode : 1);
+}
+main().catch((err) => die(err.message ?? String(err)));

package/dist/models.d.ts

@@ -0,0 +1,23 @@
+export interface SelectedModel {
+    /** Bare id: the suffix after the last `/` of the full ref. */
+    id: string;
+    /** Full LiteLLM `provider/model` ref, as written in `OPENCLAW_TEST_MODELS`. */
+    ref: string;
+}
+/**
+ * Resolve the `--model` value against the configured model catalog.
+ *
+ * `OPENCLAW_TEST_MODELS` is a comma list of full `provider/model` refs — the only
+ * place the `provider/` prefix appears. The CLI value, `OPENCLAW_DEFAULT_TEST_MODEL`,
+ * and the recorded `model` are all bare ids (the suffix after the last `/`). A bare
+ * id resolves to its ref by suffix-match; zero or many matches is a hard error.
+ *
+ * The selection is `all` (whole catalog, sorted by id), a single bare id, or a comma
+ * list of bare ids (deduped, CLI order preserved); `undefined` falls back to
+ * `OPENCLAW_DEFAULT_TEST_MODEL`.
+ */
+export declare function resolveSelectedModels(params: {
+    selection: string | undefined;
+    modelsEnv: string | undefined;
+    defaultEnv: string | undefined;
+}): SelectedModel[];

package/dist/models.js

@@ -0,0 +1,78 @@
+/**
+ * Resolve the `--model` value against the configured model catalog.
+ *
+ * `OPENCLAW_TEST_MODELS` is a comma list of full `provider/model` refs — the only
+ * place the `provider/` prefix appears. The CLI value, `OPENCLAW_DEFAULT_TEST_MODEL`,
+ * and the recorded `model` are all bare ids (the suffix after the last `/`). A bare
+ * id resolves to its ref by suffix-match; zero or many matches is a hard error.
+ *
+ * The selection is `all` (whole catalog, sorted by id), a single bare id, or a comma
+ * list of bare ids (deduped, CLI order preserved); `undefined` falls back to
+ * `OPENCLAW_DEFAULT_TEST_MODEL`.
+ */
+export function resolveSelectedModels(params) {
+    const catalog = parseCatalog(params.modelsEnv);
+    if (params.selection === "all")
+        return [...catalog].sort((a, b) => a.id.localeCompare(b.id));
+    if (params.selection !== undefined)
+        return resolveIdList(catalog, params.selection);
+    if (!params.defaultEnv) {
+        throw new Error("run: no --model given and OPENCLAW_DEFAULT_TEST_MODEL is unset; pass --model <id|id,id,…|all> or set the default");
+    }
+    return [matchById(catalog, params.defaultEnv)];
+}
+function resolveIdList(catalog, selection) {
+    const ids = selection
+        .split(",")
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0);
+    if (ids.length === 0) {
+        throw new Error(`run: --model expects a non-empty id list, got ${JSON.stringify(selection)}`);
+    }
+    const seen = new Set();
+    const selected = [];
+    for (const id of ids) {
+        if (seen.has(id))
+            continue;
+        seen.add(id);
+        selected.push(matchById(catalog, id));
+    }
+    return selected;
+}
+function parseCatalog(modelsEnv) {
+    const refs = (modelsEnv ?? "")
+        .split(",")
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0);
+    if (refs.length === 0) {
+        throw new Error("run: OPENCLAW_TEST_MODELS is empty; set a comma list of provider/model refs");
+    }
+    const catalog = refs.map((ref) => ({ id: bareId(ref), ref }));
+    assertUniqueIds(catalog);
+    return catalog;
+}
+function bareId(ref) {
+    const id = ref.split("/").pop();
+    if (!id)
+        throw new Error(`run: invalid model ref ${JSON.stringify(ref)} in OPENCLAW_TEST_MODELS`);
+    return id;
+}
+function assertUniqueIds(catalog) {
+    const seen = new Set();
+    for (const { id } of catalog) {
+        if (seen.has(id)) {
+            throw new Error(`run: OPENCLAW_TEST_MODELS has two entries with bare id ${JSON.stringify(id)}`);
+        }
+        seen.add(id);
+    }
+}
+function matchById(catalog, id) {
+    const matches = catalog.filter((m) => m.id === id);
+    if (matches.length === 1)
+        return matches[0];
+    const known = catalog.map((m) => m.id).join(", ");
+    if (matches.length === 0) {
+        throw new Error(`run: model ${JSON.stringify(id)} not found in OPENCLAW_TEST_MODELS — known: ${known}`);
+    }
+    throw new Error(`run: model ${JSON.stringify(id)} is ambiguous in OPENCLAW_TEST_MODELS — known: ${known}`);
+}

package/dist/parse-tagged-json.d.ts

@@ -0,0 +1 @@
+export declare function extractTaggedBlock(raw: string, tagName: string): string;

package/dist/parse-tagged-json.js

@@ -0,0 +1,16 @@
+export function extractTaggedBlock(raw, tagName) {
+    const escaped = escapeRegex(tagName);
+    const open = new RegExp(`<${escaped}>`).exec(raw);
+    if (!open) {
+        throw new Error(`extractTaggedBlock: opening <${tagName}> not found. raw=${JSON.stringify(raw.slice(0, 200))}`);
+    }
+    const re = new RegExp(`<${escaped}>([\\s\\S]*?)</${escaped}>`);
+    const match = re.exec(raw);
+    if (!match) {
+        throw new Error(`extractTaggedBlock: closing </${tagName}> not found after opening tag. raw=${JSON.stringify(raw.slice(0, 200))}`);
+    }
+    return match[1].trim();
+}
+function escapeRegex(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}

package/dist/report.d.ts

@@ -0,0 +1,215 @@
+/**
+ * Scenario report typing for openclaw-test.
+ *
+ * Two artifacts per scenario run, under `<OPENCLAW_TEST_ARTIFACTS_DIR>/<baseStamp>/<scenario>-<channel>[-iter<n>]/`:
+ *
+ *   - `scenario-log.jsonl` — live append, one `ReportEntry` per line, written as
+ *                            things happen. Survives runner crash / hang.
+ *   - `report.json`        — final `ScenarioReport`, written once at end.
+ *
+ * `report.json` carries the same entries as `scenario-log.jsonl`, plus terminal
+ * data only computable at the end (status, durationMs, finishedAt, parsed
+ * agentToolCalls, cost).
+ *
+ * Each agent or scenario action is one entry. Assertions, scenario-log notes,
+ * and failures about that action live as nested fields on the entry, not as
+ * separate entries. Free-standing `scenarioLog` entries are the fallback for
+ * `ctx.log(...)` calls that are not bound to an action.
+ */
+import type { Readable, Writable } from "node:stream";
+export interface ScenarioReport {
+    schemaVersion: 4;
+    scenario: string;
+    channel: ChannelId;
+    model: string;
+    conversationId: string;
+    accountId: string;
+    startedAt: string;
+    finishedAt: string;
+    durationMs: number;
+    cost: CostBreakdown;
+    result: ScenarioResult;
+    entries: ReportEntry[];
+}
+export type ScenarioResult = PassScenarioResult | FailScenarioResult;
+export type FailScenarioResult = FailedEntryScenarioResult | ErrorScenarioResult;
+export interface PassScenarioResult {
+    verdict: "pass";
+}
+export interface FailedEntryScenarioResult {
+    verdict: "fail";
+    cause: "failedEntry";
+    /** Stable entry identifier shared by `scenario-log.jsonl` and `report.json`. */
+    entrySeq: number;
+    message: string;
+}
+export interface ErrorScenarioResult {
+    verdict: "fail";
+    cause: "error";
+    source: "assertion" | "judge" | "cliMock" | "timeout" | "scenarioThrow" | "runner";
+    errorName: string;
+    message: string;
+    stack?: string;
+}
+export type ChannelId = string;
+export type ReportEntry = ScenarioLogEntry | ActionEntry;
+export type ActionEntry = InboundSentEntry | OutboundReceivedEntry | CliMockEntry | AgentToolCallEntry;
+export interface ReportEntryBase {
+    ts: string;
+    /**
+     * Stable entry id shared across `scenario-log.jsonl` and `report.json`. In the
+     * jsonl it matches the line's emission order; in `report.json` entries are
+     * sorted by `ts` so `entrySeq` is not array position — it stays a cross-file
+     * reference.
+     */
+    entrySeq: number;
+}
+export interface ActionEntryBase extends ReportEntryBase {
+    scenarioLog?: ScenarioLogNote;
+    assertions?: AssertionRecord[];
+    failure?: ScenarioFailure;
+}
+export interface ScenarioLogNote {
+    label?: string;
+    ts: string;
+    extra?: unknown;
+}
+export interface ScenarioLogEntry extends ReportEntryBase {
+    kind: "scenarioLog";
+    message: string;
+}
+export interface InboundSentEntry extends ActionEntryBase {
+    kind: "inboundSent";
+    messageId: string;
+    text: string;
+    senderId: string;
+    senderName?: string;
+    threadId?: string;
+}
+export interface OutboundReceivedEntry extends ActionEntryBase {
+    kind: "outboundReceived";
+    messageId: string;
+    text: string;
+    threadId?: string;
+}
+export interface CliMockEntry extends ActionEntryBase {
+    kind: "cliMock";
+    call: CliMockCall;
+}
+export interface AgentToolCallEntry extends ActionEntryBase {
+    kind: "agentToolCall";
+    call: AgentToolCall;
+}
+export type AssertionRecord = {
+    label: string;
+    ok: true;
+    extra?: unknown;
+} | {
+    label: string;
+    ok: false;
+    detail: string;
+    extra?: unknown;
+};
+export interface CliMockCall {
+    cli: "git" | "npm" | "pnpm" | "yarn" | "claude" | (string & {});
+    argv: string[];
+    cwd: string;
+    stdin: string;
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+    startedAt: string;
+    durationMs: number;
+    handlerError?: {
+        name: string;
+        message: string;
+        stack?: string;
+    };
+}
+export interface AgentToolCall {
+    toolName: string;
+    toolUseId: string;
+    /**
+     * `sessionKey` of the session transcript the call was collected from — the
+     * OpenClaw session that made it (e.g. channel vs per-thread session).
+     */
+    sessionKey?: string;
+    input: unknown;
+    result?: {
+        isError: boolean;
+        /**
+         * Full tool result. Always present in `scenario-log.jsonl`. In `report.json`,
+         * replaced by `truncatedContent` when truncatable: a string longer than 60
+         * chars, or a `read` call's text content blocks (the file is identified by
+         * `input` and kept in full in the jsonl).
+         */
+        content?: unknown;
+        /** Only in `report.json`: rtrimmed first 60 chars + `…` of the truncatable text. */
+        truncatedContent?: string;
+    };
+    /**
+     * Timestamp of the transcript's assistant message that issued the call —
+     * per-message, so ordering `entries` on it is meaningful. Absent when the
+     * message carried no timestamp; such calls sort to the end of the timeline.
+     */
+    startedAt?: string;
+    /**
+     * Best-effort estimate of when the tool call actually executed, inferred by
+     * matching the call's leading CLI against an in-order `cliMock` entry from
+     * the same scenario. `startedAt` stamps the issuing assistant message, not
+     * the execution. Not used for `entries` ordering.
+     */
+    inferredStartedAt?: string;
+    turn?: number;
+}
+export interface ScenarioFailure {
+    name: string;
+    message: string;
+    stack?: string;
+    source: "assertion" | "judge" | "cliMock" | "timeout" | "scenarioThrow" | "runner";
+}
+export interface CostBreakdown {
+    agentUsd: number;
+    judgeUsd: number;
+    totalUsd: number;
+    agentTurns: number;
+}
+export interface CliMockHandlerArgs {
+    argv: string[];
+    cwd: string;
+    stdin: Readable;
+    stdout: Writable;
+    stderr: Writable;
+}
+export type CliMockHandler = (args: CliMockHandlerArgs) => number | undefined | Promise<number | undefined>;
+/**
+ * Patch describing the augmentation of an already-emitted entry. Carries only
+ * the new nested field so the live `scenario-log.jsonl` need not re-serialize
+ * the entry's full text/metadata on every assertion or annotation.
+ */
+export type AugmentPatch = {
+    kind: "scenarioLog";
+    scenarioLog: ScenarioLogNote;
+} | {
+    kind: "assertion";
+    assertion: AssertionRecord;
+} | {
+    kind: "failure";
+    failure: ScenarioFailure;
+};
+export type SinkEvent = {
+    type: "entry";
+    entry: ReportEntry;
+} | {
+    type: "augment";
+    entrySeq: number;
+    patch: AugmentPatch;
+};
+/**
+ * Callback the runner passes into `createContext` so live records can be
+ * appended to `scenario-log.jsonl` as they happen. First emission of an entry
+ * is `type: "entry"`. Subsequent nested-field additions (assertions, failure,
+ * scenarioLog) are `type: "augment"` with only the patch — readers reconstruct
+ * full state by folding augments onto entries by `seq`.
+ */
+export type EmitSink = (event: SinkEvent) => void;

package/dist/report.js

@@ -0,0 +1,19 @@
+/**
+ * Scenario report typing for openclaw-test.
+ *
+ * Two artifacts per scenario run, under `<OPENCLAW_TEST_ARTIFACTS_DIR>/<baseStamp>/<scenario>-<channel>[-iter<n>]/`:
+ *
+ *   - `scenario-log.jsonl` — live append, one `ReportEntry` per line, written as
+ *                            things happen. Survives runner crash / hang.
+ *   - `report.json`        — final `ScenarioReport`, written once at end.
+ *
+ * `report.json` carries the same entries as `scenario-log.jsonl`, plus terminal
+ * data only computable at the end (status, durationMs, finishedAt, parsed
+ * agentToolCalls, cost).
+ *
+ * Each agent or scenario action is one entry. Assertions, scenario-log notes,
+ * and failures about that action live as nested fields on the entry, not as
+ * separate entries. Free-standing `scenarioLog` entries are the fallback for
+ * `ctx.log(...)` calls that are not bound to an action.
+ */
+export {};