@alignfirst/openclaw-test 0.20.0

package/Dockerfile.base

@@ -0,0 +1,45 @@
+# Consumer-agnostic base for the openclaw-test runner stack. Built locally by the CLI
+# (`openclaw-test env build`) and tagged `paleo/openclaw-test-base:<pkg-version>`.
+# Ships: Node + `shadow` (for the UID/GID dance), the mock-cli shim binary (no
+# per-command symlinks — consumers add their own), and the exec RPC watcher.
+# The consumer's project `Dockerfile` does `FROM paleo/openclaw-test-base:${OPENCLAW_TEST_BASE_TAG}`
+# and adds whatever else the fixture needs (e.g. git, pnpm via corepack, mock symlinks).
+# Build context: this package's directory (so `dist/mock-cli-shim.js` is COPY-able).
+
+FROM node:26-alpine
+
+RUN apk add --no-cache shadow
+
+ARG ASSISTANT_UID=1000
+ARG ASSISTANT_GID=1000
+RUN (getent passwd ${ASSISTANT_UID} && userdel -r "$(getent passwd ${ASSISTANT_UID} | cut -d: -f1)" || true) && \
+    (getent group ${ASSISTANT_GID} && groupdel "$(getent group ${ASSISTANT_GID} | cut -d: -f1)" || true) && \
+    groupadd --gid ${ASSISTANT_GID} assistant && \
+    useradd --create-home --shell /bin/sh --uid ${ASSISTANT_UID} --gid ${ASSISTANT_GID} assistant && \
+    mkdir -p /home/assistant/.openclaw /opt/openclaw-test/src /opt/openclaw-test/artifacts && \
+    chown -R assistant:assistant /home/assistant /opt/openclaw-test/src /opt/openclaw-test/artifacts
+
+# Mock-CLI shim under /opt/openclaw-test/mocks. OpenClaw's exec tool runs commands via
+# `/bin/sh -lc`, which sources /etc/profile; Alpine's default resets PATH to a
+# "safe" set that excludes /opt/openclaw-test/mocks/bin, so we override it. Consumers
+# create the per-command symlinks (e.g. claude, gh) in their own Dockerfile.
+COPY --chown=assistant:assistant dist/mock-cli-shim.js /opt/openclaw-test/mocks/dist/mock-cli-shim.js
+RUN mkdir -p /opt/openclaw-test/mocks/bin && \
+    printf '#!/bin/sh\nexec node /opt/openclaw-test/mocks/dist/mock-cli-shim.js "$0" "$@"\n' > /opt/openclaw-test/mocks/bin/mock-cli-shim && \
+    chmod +x /opt/openclaw-test/mocks/bin/mock-cli-shim && \
+    chown -R assistant:assistant /opt/openclaw-test/mocks && \
+    printf 'export PATH="/opt/openclaw-test/mocks/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"\n' > /etc/profile
+
+# Exec RPC watcher. The gateway service spawns it in the background; the runner
+# pokes requests onto the shared `openclaw-test-ipc` volume via `ctx.execInGateway(...)`.
+# Pre-create the mountpoint with `assistant` ownership so a fresh named volume
+# inherits it — otherwise Docker creates it as root:root and the assistant-uid
+# processes get EACCES on the first write.
+COPY --chown=root:root exec-watcher.mjs /usr/local/bin/exec-watcher
+RUN chmod +x /usr/local/bin/exec-watcher && \
+    mkdir -p /var/run/openclaw-test-ipc && chown assistant:assistant /var/run/openclaw-test-ipc
+
+USER assistant
+WORKDIR /opt/openclaw-test/src
+
+CMD ["sleep", "infinity"]

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Paleo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,132 @@
+# @alignfirst/openclaw-test
+
+Dockerised regression-test harness for OpenClaw workspaces. Drives the agent through two synthetic channels (`discord-mock`, `slack-mock`) and asserts the results.
+
+Pair with [`@alignfirst/openclaw-channel-mock-core`](https://www.npmjs.com/package/@alignfirst/openclaw-channel-mock-core), [`@alignfirst/openclaw-discord-mock`](https://www.npmjs.com/package/@alignfirst/openclaw-discord-mock), [`@alignfirst/openclaw-slack-mock`](https://www.npmjs.com/package/@alignfirst/openclaw-slack-mock).
+
+For internals (topology, Dockerfile pair, mocked-CLI shim, channel plugin mechanics, OpenClaw quirks), see [openclaw-test-architecture.md](https://github.com/paleo/alignfirst/blob/main/docs/alignfirst-dev-kit/openclaw-test-architecture.md).
+
+## Install
+
+```sh
+npm i -D @alignfirst/openclaw-test @alignfirst/openclaw-channel-mock-core @alignfirst/openclaw-discord-mock @alignfirst/openclaw-slack-mock openclaw
+```
+
+Requires Docker Compose.
+
+## Init
+
+```sh
+npx @alignfirst/openclaw-test init <project-dir>
+```
+
+Adds the four `package.json` scripts (`env:build`, `env:up`, `env:down`, `e2e`) if missing, and drops four files:
+
+- `openclaw.json` — gateway config (mode `local`, both channel plugins enabled, main agent placeholder).
+- `.env.local.example` — copy to `.env.local` and fill in; its comments document every variable.
+- `docker-compose.yml` — thin overlay that `include:`s the base stack from `node_modules/`.
+- `Dockerfile` — consumer-owned; its comments document the common customizations (system tools, mock-CLI symlinks, fixtures, reset scripts).
+
+## Configure
+
+Edit `openclaw.json`:
+
+- `agents.entries.main.model` — default `provider/model` ref; `run --model` overrides it per run.
+- `agents.entries.main.workspace` — host path to your OpenClaw workspace. Field name is **`workspace`**, not `workspaceDir`.
+- `channels.slack-mock.replyToMode` — `"all"` (default) routes eligible roots and replies through
+  one thread session; `"off"` leaves root turns in the channel session and threads only explicit
+  replies. `blockStreaming: true` keeps streamed replies as one bus message.
+
+## Env vars (`.env.local`)
+
+```sh
+ANTHROPIC_API_KEY=sk-ant-… # Required for an Anthropic agent or judge.
+OPENROUTER_API_KEY=sk-or-… # Required for an OpenRouter agent or judge.
+OPENCLAW_WORKSPACE_DIR=/path/to/your/openclaw-workspace
+
+# Model catalog: full LiteLLM refs. `run --model` picks by bare id (suffix after the last "/").
+OPENCLAW_TEST_MODELS=anthropic/claude-sonnet-4-6,custom-openrouter/qwen/qwen3.6-plus
+OPENCLAW_DEFAULT_TEST_MODEL=claude-sonnet-4-6
+
+```
+
+See `.env.local.example` for the optional overrides (paths, raw stream log).
+
+## Scenarios
+
+Drop scenarios under `scenarios/<id>.ts`: each default-exports `async (ctx: ScenarioContext) => void`. They are loaded by Node's built-in TypeScript stripping: no `enum`, `namespace`, decorators, ctor parameter properties. Shared helpers must go under subdirectories (e.g. `scenarios/_lib/`).
+
+Project fixtures and their reset logic are consumer concerns — ship a reset script in your consumer image and invoke it via `ctx.execInGateway(...)`.
+
+`ScenarioContext` primitives (authoritative types: `src/context.ts`):
+
+- `channel`, `conversationId`, `accountId` — per-task isolation; never hard-code a conversation id.
+- `busUrl` — the bus the gateway talks to, for direct bus calls such as `failNextQaBusOperation`.
+- `sendInbound(input)` — push an inbound message on the bus.
+- `createThread(input)` — seed a thread the bot did not open, as a surface user creating one; returns its id for `sendInbound`.
+- `waitForOutbound(predicate, opts)` — await a matching outbound; fails fast on unmatched outbounds or mock-CLI silence.
+- `poll`, `expectNoOutbound`, `getCursor` — bus consumers.
+- `assertRegex`, `assertEqual`, `assertLength` — structural assertions.
+- `judgeLLM({ message, rubric, label, attachTo? })` — LLM judgement, bound to an action entry.
+- `mockCli(name, handler)` — intercept the gateway's CLI calls (`git`, `claude`, …); unregistered calls fail the scenario.
+- `execInGateway(argv, opts)` — run a command inside the gateway container.
+- `log(...)` — scenario log entry, free-standing or attached to an action.
+
+Prefer structural assertions over `judgeLLM`; reserve the judge for free-form content claims.
+
+Examples: [alignfirst-dev-kit-tests/scenarios](https://github.com/paleo/alignfirst/tree/main/alignfirst-dev-kit-tests/scenarios).
+
+## Run
+
+```sh
+npm run env:build                                                  # build base + consumer image
+npm run env:up                                                     # (optional) keep bus + gateway warm across iterative runs
+npm run e2e -- --channel all <scenario>                             # one scenario, both channels
+npm run e2e -- --channel all --all                                  # every scenario, both channels
+npm run e2e -- --channel discord-mock <scenario>                    # restrict to one channel
+npm run e2e -- --channel all --model qwen3.6-plus <scenario>        # pick a model by bare id
+npm run e2e -- --channel all --model claude-sonnet-4-6,qwen3.6-plus <s>  # a comma list of bare ids
+npm run e2e -- --channel all --model all <scenario>                 # run every model in OPENCLAW_TEST_MODELS
+npm run e2e -- --channel all --iterations 5 <scenario>              # repeat each (scenario, channel) pair 5×
+npm run e2e -- --channel all --iterations 5 --max-failures 1 <s>    # abort a pair after >1 failure
+npm run e2e -- --channel all --iterations 10 --parallel 4 <s>       # run up to 4 cells concurrently
+npm run e2e -- --channel discord-mock --reuse-stack <s>             # skip per-cell bus+gateway recreation
+npm run env:down                                                   # tear down all worker stacks
+```
+
+`run` auto-starts the worker stacks it needs and tears down the ones it started; an explicit `env:up` beforehand keeps them warm across runs.
+
+Rebuild (`npm run env:build`) after editing `openclaw.json` or the `Dockerfile`, or after bumping any `@alignfirst/openclaw-*` dependency.
+
+Cells run serially per worker stack. Exit 0 iff every pair passes. Artifacts land under `artifacts/<runStamp>/` — see the [architecture doc](https://github.com/paleo/alignfirst/blob/main/docs/alignfirst-dev-kit/openclaw-test-architecture.md).
+
+### Parallel runs
+
+`--parallel K` (default `OPENCLAW_TEST_PARALLEL` from `.env.local`, fallback 1; the flag wins) runs up to K cells concurrently, each on its own worker Compose stack — project `<project>-w<i>`, all sharing one image. Iterations of one pair parallelize too, so flakiness measurements (`--iterations N`) are the primary win. With K > 1, per-cell output is captured to `artifacts/<runStamp>/cells/<leaf>.log` and the console shows one compact line per cell event.
+
+Each worker gets its own gateway logs dir (`.gateway-logs/w<i>/`) and a private workspace copy under `.workers/`, refreshed from `OPENCLAW_WORKSPACE_DIR` before every cell. Add `.workers/` to your `.gitignore` and `.dockerignore`.
+
+`env up -- --parallel K` pre-warms K workers. `env down` takes no flag: it discovers every `<project>-w<N>` stack and tears them all down, including orphans from a crashed run.
+
+**Upgrading from a pre-parallel version:** stacks now run under per-worker Compose project names, so the old un-suffixed project is orphaned. Tear it down once with `docker compose down` from the project dir.
+
+## Channels
+
+- `discord-mock` — full Discord-shaped surface; no auto-thread.
+- `slack-mock` — Slack-shaped surface with `send`, `react`, `read`, `edit`, `delete`, `reactions`,
+  and `search`. It supports `replyToMode: "off" | "all"`; fake thread creation/rename actions stay
+  disabled.
+
+Assert on `conversation.id` / `threadId`, not envelope formatting.
+
+## Judge model
+
+Defaults to `anthropic/claude-haiku-4-5`. Override it via `OPENCLAW_TEST_JUDGE_MODEL` on
+the `runner` service (set in your consumer overlay). Direct Anthropic refs use
+`anthropic/<model>`; OpenRouter refs use `openrouter/<model>`, for example
+`openrouter/anthropic/claude-haiku-4.5`. The judge is **not** an OpenClaw agent — don't
+configure it in `openclaw.json`.
+
+## Attribution
+
+The runner package contains no upstream-adapted code. See sibling packages' `NOTICE.md` for OpenClaw attribution covering the channel plugins.

package/bin/cli.mjs

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+import { dispatch } from "../dist/cli.js";
+
+dispatch(process.argv.slice(2)).catch((err) => {
+  console.error("openclaw-test crash:", err);
+  process.exit(1);
+});

package/dist/bus.d.ts

@@ -0,0 +1 @@
+export declare function startBus(): void;

package/dist/bus.js

@@ -0,0 +1,20 @@
+import { createServer } from "node:http";
+import { createBus } from "@alignfirst/openclaw-channel-mock-core";
+const PORT = 43123;
+const HOST = "0.0.0.0";
+export function startBus() {
+    const { handler } = createBus();
+    const server = createServer(async (req, res) => {
+        const handled = await handler(req, res);
+        if (!handled) {
+            res.statusCode = 404;
+            res.end("not found");
+        }
+    });
+    server.listen(PORT, HOST, () => {
+        console.log(`channel-mock bus listening on ${HOST}:${PORT}`);
+    });
+}
+if (import.meta.url === `file://${process.argv[1]}`) {
+    startBus();
+}

package/dist/cell-result.d.ts

@@ -0,0 +1,25 @@
+import type { JudgeUsage } from "./judge.js";
+export interface CellResult {
+    schemaVersion: 3;
+    scenarioId: string;
+    channel: string;
+    model: string;
+    iterationIndex: number;
+    verdict: "pass" | "fail";
+    durationMs: number;
+    conversationId: string;
+    artifactDirName: string;
+    agentCostUsd: number;
+    agentTurns: number;
+    judgeUsd: number;
+    judgeUsages: JudgeUsage[];
+}
+export declare function cellLeafName(parts: {
+    scenarioId: string;
+    modelId: string;
+    channel: string;
+    iterationIndex: number;
+    iterationWidth: number;
+}): string;
+export declare function readCellResult(path: string): CellResult | undefined;
+export declare function writeCellResult(path: string, r: CellResult): void;

package/dist/cell-result.js

@@ -0,0 +1,32 @@
+import { readFileSync, writeFileSync } from "node:fs";
+export function cellLeafName(parts) {
+    const iterSuffix = parts.iterationWidth > 0
+        ? `-#${String(parts.iterationIndex).padStart(parts.iterationWidth, "0")}`
+        : "";
+    return `${parts.modelId}-${parts.scenarioId}-${parts.channel}${iterSuffix}`;
+}
+export function readCellResult(path) {
+    let raw;
+    try {
+        raw = readFileSync(path, "utf8");
+    }
+    catch {
+        return undefined;
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return undefined;
+    }
+    if (!parsed || typeof parsed !== "object")
+        return undefined;
+    const r = parsed;
+    if (r.schemaVersion !== 3)
+        return undefined;
+    return parsed;
+}
+export function writeCellResult(path, r) {
+    writeFileSync(path, JSON.stringify(r, null, 2));
+}

package/dist/cli.d.ts

@@ -0,0 +1,5 @@
+export declare function dispatch(argv: string[]): Promise<void>;
+export interface PackageJsonScripts {
+    scripts?: Record<string, string>;
+}
+export declare function addInitScripts(pkg: PackageJsonScripts): string[];

package/dist/cli.js

@@ -0,0 +1,110 @@
+import { copyFile, mkdir, readdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { startBus } from "./bus.js";
+import { envCommand, runCommand } from "./env-cli.js";
+import { main as runnerMain } from "./runner.js";
+// `dist/cli.js` ships under `<package>/dist/`; one level up from its dir is the package root.
+const PACKAGE_DIR = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const INIT_SCRIPTS = {
+    "env:build": "openclaw-test env build",
+    "env:up": "openclaw-test env up",
+    "env:down": "openclaw-test env down",
+    e2e: "openclaw-test run",
+};
+export async function dispatch(argv) {
+    const [cmd, ...rest] = argv;
+    switch (cmd) {
+        case "init":
+            await initCommand(rest[0]);
+            return;
+        case "env":
+            await envCommand(PACKAGE_DIR, rest);
+            return;
+        case "run":
+            await runCommand(PACKAGE_DIR, rest);
+            return;
+        case "bus":
+            startBus();
+            return;
+        case "runner":
+            await runnerMain(rest);
+            return;
+        default:
+            usage();
+    }
+}
+async function initCommand(targetDirRaw) {
+    if (!targetDirRaw)
+        usage();
+    const target = resolve(process.cwd(), targetDirRaw);
+    const templatesDir = join(PACKAGE_DIR, "templates");
+    await mkdir(target, { recursive: true });
+    const entries = await readdir(templatesDir);
+    for (const name of entries) {
+        await copyFile(join(templatesDir, name), join(target, name));
+        console.log(`copied ${name} -> ${join(target, name)}`);
+    }
+    await addScriptsToPackageJson(target);
+    printNextSteps();
+}
+async function addScriptsToPackageJson(target) {
+    const pkgPath = join(target, "package.json");
+    let raw;
+    try {
+        raw = await readFile(pkgPath, "utf8");
+    }
+    catch {
+        console.log("no package.json found — run `npm init`, then add the scripts manually (see README)");
+        return;
+    }
+    let pkg;
+    try {
+        pkg = JSON.parse(raw);
+    }
+    catch {
+        console.error(`${pkgPath} is not valid JSON — fix it, then add the scripts manually (see README)`);
+        return;
+    }
+    const added = addInitScripts(pkg);
+    if (added.length === 0) {
+        console.log("package.json scripts already present");
+        return;
+    }
+    await writeFile(pkgPath, `${JSON.stringify(pkg, undefined, 2)}\n`);
+    console.log(`added scripts to package.json: ${added.join(", ")}`);
+}
+export function addInitScripts(pkg) {
+    pkg.scripts ??= {};
+    const scripts = pkg.scripts;
+    const added = [];
+    for (const [name, command] of Object.entries(INIT_SCRIPTS)) {
+        if (scripts[name] !== undefined)
+            continue;
+        scripts[name] = command;
+        added.push(name);
+    }
+    return added;
+}
+function printNextSteps() {
+    console.log("\nNext steps:\n" +
+        "  1. npm i -D @alignfirst/openclaw-test @alignfirst/openclaw-channel-mock-core" +
+        " @alignfirst/openclaw-discord-mock @alignfirst/openclaw-slack-mock openclaw\n" +
+        "  2. cp .env.local.example .env.local   # then fill it in\n" +
+        "  3. npm run env:build");
+}
+function usage() {
+    console.error("usage: openclaw-test <init|env|run|bus|runner> [args]\n\n" +
+        "  init <target-dir>      copy templates into target dir\n" +
+        "  env <build|up|down>    drive the Compose stack (host-side)\n" +
+        "  run [flags] [...]      run scenarios against the stack (host-side)\n" +
+        "  bus                    start the bus HTTP server (inside container)\n" +
+        "  runner [flags] [...]   execute scenarios (inside container)\n");
+    process.exit(1);
+}
+if (import.meta.url === `file://${process.argv[1]}`) {
+    dispatch(process.argv.slice(2)).catch((err) => {
+        console.error("openclaw-test crash:", err);
+        process.exit(1);
+    });
+}

package/dist/context.d.ts

@@ -0,0 +1,220 @@
+import { type QaBusConversation, type QaBusMessage } from "@alignfirst/openclaw-channel-mock-core";
+import { type ExecInGatewayOptions, type ExecInGatewayResult } from "./exec-rpc.js";
+import { type JudgeUsage, type JudgeVerdict, type JudgeVerdictJson, type JudgeVerdictRaw } from "./judge.js";
+import type { ActionEntry, AgentToolCall, ChannelId, CliMockCall, CliMockEntry, CliMockHandler, EmitSink, InboundSentEntry, OutboundReceivedEntry, ReportEntry, ScenarioFailure, ScenarioResult } from "./report.js";
+export type { ChannelId } from "./report.js";
+export type Conversation = QaBusConversation;
+export type BusMessage = QaBusMessage;
+export interface PollResult {
+    messages: BusMessage[];
+    nextCursor: number;
+}
+export interface SendInboundResult {
+    message: BusMessage;
+    entry: InboundSentEntry;
+}
+export interface WaitForOutboundResult {
+    match: BusMessage;
+    entry: OutboundReceivedEntry;
+    nextCursor: number;
+}
+export type MockCliRegisterMode = "register" | "replace" | "ifAbsent";
+export interface MockCliRegisterOptions {
+    /**
+     * - `"register"` (default): throws if a handler is already registered for `name`.
+     * - `"replace"`: overwrites the existing handler; throws if none was registered.
+     * - `"ifAbsent"`: no-op if a handler is already registered.
+     */
+    mode?: MockCliRegisterMode;
+}
+export interface ScenarioContext {
+    channel: ChannelId;
+    conversationId: string;
+    accountId: ChannelId;
+    /** The bus the gateway's channel plugins talk to; for direct bus calls such as fault injection. */
+    busUrl: string;
+    /**
+     * The most recent agent-action entry (`outboundReceived` / `cliMock` /
+     * `agentToolCall`). Capture this synchronously after `await` resolves to
+     * pin a handle to the action before any further awaits could overwrite it.
+     * `inboundSent` is scenario-emitted and does not update this property.
+     */
+    readonly currentEntry: ActionEntry | undefined;
+    /**
+     * `true` once the scenario has called `markScenarioAsEnded`. After that, the
+     * mock-cli server stops dispatching to registered handlers — any incoming
+     * call (typically a lingering agent action firing after the verdict is in)
+     * is answered with a "scenario ended" stub message and recorded as a
+     * post-end cliMock entry that does **not** affect the result.
+     */
+    readonly isScenarioEnded: boolean;
+    /**
+     * Declare the scenario's verdict signals are all in. From this point on
+     * the runner stops attributing tool-call failures to this scenario.
+     * Optional `reason` is recorded in the scenarioLog (e.g. `"PASS"`).
+     */
+    markScenarioAsEnded(reason?: string): void;
+    log(message: string): void;
+    log(opts: {
+        attachTo: ActionEntry;
+        label?: string;
+        extra?: unknown;
+    }): void;
+    sendInbound(input: SendInboundInput): Promise<SendInboundResult>;
+    /**
+     * Seed a thread the bot did not open, as a human creating one on the surface. Returns its id,
+     * to be passed as `sendInbound`'s `threadId`.
+     */
+    createThread(input: CreateThreadInput): Promise<string>;
+    poll(opts: {
+        sinceCursor: number;
+        timeoutMs?: number;
+    }): Promise<PollResult>;
+    waitForOutbound(predicate: (m: BusMessage) => boolean, opts: WaitForOutboundOptions): Promise<WaitForOutboundResult>;
+    expectNoOutbound(predicate: (m: BusMessage) => boolean, opts: {
+        withinMs: number;
+        sinceCursor: number;
+    }): Promise<{
+        nextCursor: number;
+    }>;
+    assertRegex(actual: string, pattern: RegExp, label: string): void;
+    assertEqual<T>(actual: T, expected: T, label: string): void;
+    assertLength(value: {
+        length: number;
+    } | string | unknown[], expected: number, label: string): void;
+    /**
+     * Anthropic-direct judgement. The verdict is recorded as an `AssertionRecord`
+     * on the `attachTo` action entry — usually the entry returned by
+     * `waitForOutbound` / `sendInbound`, or a snapshot of `ctx.currentEntry`
+     * taken right after the relevant `await` resolves.
+     */
+    judgeLLM(p: {
+        attachTo: ActionEntry;
+        message: string;
+        rubric: string;
+        label: string;
+        maxTokens?: number;
+    }): Promise<JudgeVerdict>;
+    judgeLLMJson<T>(p: {
+        message: string;
+        prompt: string;
+        returnType: string;
+        label: string;
+        maxTokens?: number;
+    }): Promise<JudgeVerdictJson<T>>;
+    judgeLLMRaw(p: {
+        prompt: string;
+        label: string;
+        maxTokens?: number;
+    }): Promise<JudgeVerdictRaw>;
+    getCursor(): Promise<number>;
+    mockCli(name: string, handler: CliMockHandler, opts?: MockCliRegisterOptions): void;
+    /**
+     * Execute an arbitrary command inside the gateway container via the exec
+     * watcher RPC. Always resolves once the wrapped command finishes (with its
+     * exit code, stdout, and stderr — non-zero exits do NOT throw). Throws only
+     * on transport failure or hard timeout (`timeoutMs + 5s` headroom for the
+     * watcher to record a kill before this side gives up).
+     */
+    execInGateway(argv: string[], opts?: ExecInGatewayOptions): Promise<ExecInGatewayResult>;
+    /**
+     * Poll the session transcripts until an agent tool call matches `predicate`,
+     * then record a passing `AssertionRecord` on the current entry and return the
+     * call. On timeout, record a failing assertion and throw (hard-fail) — use it
+     * to assert the agent took a specific action (read a file, ran a command).
+     * Aggregates across all the conversation's sessions, so it sees thread and
+     * subagent tool calls; the transcript is appended per message, so calls from
+     * a turn still in flight are already visible.
+     */
+    waitForAgentToolCall(predicate: (call: AgentToolCall) => boolean, opts: WaitForAgentToolCallOptions): Promise<AgentToolCall>;
+    /**
+     * One-shot parse of the conversation's agent tool calls from the session
+     * transcripts — no waiting, no assertion recorded.
+     */
+    getAgentToolCalls(): Promise<AgentToolCall[]>;
+}
+export interface WaitForAgentToolCallOptions {
+    /** Assertion label recorded on the current entry. */
+    label: string;
+    /** Default 30_000. */
+    timeoutMs?: number;
+    /** Default 500. */
+    pollMs?: number;
+}
+export interface WaitForOutboundOptions {
+    sinceCursor: number;
+    timeoutMs?: number;
+    failFastUnmatchedOutbounds?: number | false;
+    /**
+     * Liveness fail-fast: throw when a mocked CLI invoked *during this wait* is
+     * followed by neither a CLI call nor any outbound for this long (default
+     * 60s). CLI calls from before the wait never arm it; any outbound disarms it
+     * until the next CLI call. `false` disables it — for waits where long
+     * non-CLI agent work (exec calls, delegation) legitimately follows a CLI
+     * call with no post in between.
+     */
+    failFastCliMockGraceMs?: number | false;
+}
+export interface ScenarioInternals {
+    finalize(opts?: {
+        failure?: ScenarioFailure;
+    }): {
+        entries: ReportEntry[];
+        judgeUsages: JudgeUsage[];
+        result: ScenarioResult;
+    };
+    emitOutboundReceived(m: BusMessage): void;
+    emitCliMock(call: CliMockCall): void;
+    getMockHandlers(): Map<string, CliMockHandler>;
+    peekEntries(): {
+        entries: ReportEntry[];
+    };
+    isScenarioEnded(): boolean;
+}
+export interface SendInboundInput {
+    senderId: string;
+    senderName?: string;
+    text: string;
+    threadId?: string;
+    /** Thread name carried by the message, as a surface reports it. */
+    threadTitle?: string;
+    conversation?: Conversation;
+}
+export interface CreateThreadInput {
+    title: string;
+    /** The surface user who opened it. Defaults to the bot, so a human thread must name one. */
+    createdBy?: string;
+}
+export declare class AssertionError extends Error {
+    constructor(msg: string);
+}
+export declare function createContext(params: {
+    channel: ChannelId;
+    conversationId: string;
+    /** Scenario start time; bounds the transcript window for `waitForAgentToolCall`. */
+    startedAtIso?: string;
+    emitSink?: EmitSink;
+}): {
+    ctx: ScenarioContext;
+    internals: ScenarioInternals;
+};
+export interface WaitForOutboundDeps {
+    accountId: ChannelId;
+    awaitEntry: (id: string) => Promise<OutboundReceivedEntry>;
+    getLastCliMock: () => {
+        atMs: number;
+        entry: CliMockEntry;
+    } | undefined;
+}
+export interface WaitForOutboundOpts {
+    sinceCursor: number;
+    timeoutMs?: number;
+    failFastUnmatchedOutbounds?: number | false;
+    failFastCliMockGraceMs?: number | false;
+    pollImpl?: (accountId: ChannelId, opts: {
+        sinceCursor: number;
+        timeoutMs?: number;
+    }) => Promise<PollResult>;
+    nowImpl?: () => number;
+}
+export declare function waitForOutbound(deps: WaitForOutboundDeps, predicate: (m: BusMessage) => boolean, opts: WaitForOutboundOpts): Promise<WaitForOutboundResult>;