@alignfirst/openclaw-test 0.20.0

package/dist/context.js

@@ -0,0 +1,479 @@
+import { createQaBusThread, getQaBusState, injectQaBusInboundMessage, pollQaBus, } from "@alignfirst/openclaw-channel-mock-core";
+import { judgeCostUsd } from "./cost.js";
+import { execInGateway } from "./exec-rpc.js";
+import { judgeLLM, judgeLLMJson, judgeLLMRaw, } from "./judge.js";
+import { parseAgentToolCalls } from "./transcript-log.js";
+const BUS_URL = process.env.OPENCLAW_TEST_BUS_URL ?? "http://bus:43123";
+export class AssertionError extends Error {
+    constructor(msg) {
+        super(msg);
+        this.name = "AssertionError";
+    }
+}
+export function createContext(params) {
+    const { channel, conversationId, emitSink } = params;
+    const startedAtIso = params.startedAtIso ?? new Date().toISOString();
+    const accountId = channel;
+    const entries = [];
+    const judgeUsages = [];
+    const mockHandlers = new Map();
+    const outboundWaiters = new Map();
+    const outboundByMessageId = new Map();
+    let nextEntrySeq = 0;
+    let currentEntry;
+    let lastCliMock;
+    let scenarioEnded = false;
+    const nextEntrySeqTs = () => ({
+        entrySeq: nextEntrySeq++,
+        ts: new Date().toISOString(),
+    });
+    const emit = (entry) => {
+        entries.push(entry);
+        emitSink?.({ type: "entry", entry });
+        return entry;
+    };
+    const emitAugment = (entrySeq, patch) => {
+        emitSink?.({ type: "augment", entrySeq, patch });
+    };
+    const setCurrentEntry = (entry) => {
+        currentEntry = entry;
+        if (entry.kind === "cliMock") {
+            lastCliMock = { atMs: Date.now(), entry: entry };
+        }
+        return entry;
+    };
+    const emitOutboundReceived = (m) => {
+        const entry = {
+            ...nextEntrySeqTs(),
+            kind: "outboundReceived",
+            messageId: m.id,
+            text: m.text,
+            ...(m.threadId !== undefined ? { threadId: m.threadId } : {}),
+        };
+        emit(entry);
+        setCurrentEntry(entry);
+        outboundByMessageId.set(m.id, entry);
+        const waiter = outboundWaiters.get(m.id);
+        if (waiter) {
+            outboundWaiters.delete(m.id);
+            waiter(entry);
+        }
+    };
+    const emitCliMock = (call) => {
+        const entry = { ...nextEntrySeqTs(), kind: "cliMock", call };
+        emit(entry);
+        setCurrentEntry(entry);
+        if (call.handlerError) {
+            const failure = {
+                name: call.handlerError.name,
+                message: call.handlerError.message,
+                ...(call.handlerError.stack ? { stack: call.handlerError.stack } : {}),
+                source: "cliMock",
+            };
+            entry.failure = failure;
+            emitAugment(entry.entrySeq, { kind: "failure", failure });
+        }
+    };
+    const ctx = {
+        channel,
+        conversationId,
+        accountId,
+        busUrl: BUS_URL,
+        get currentEntry() {
+            return currentEntry;
+        },
+        get isScenarioEnded() {
+            return scenarioEnded;
+        },
+        markScenarioAsEnded: (reason) => {
+            if (scenarioEnded)
+                return;
+            scenarioEnded = true;
+            const message = reason !== undefined && reason.length > 0 ? `scenario ended: ${reason}` : "scenario ended";
+            emit({ ...nextEntrySeqTs(), kind: "scenarioLog", message });
+        },
+        log: ((arg) => {
+            if (typeof arg === "string") {
+                emit({ ...nextEntrySeqTs(), kind: "scenarioLog", message: arg });
+                return;
+            }
+            const note = {
+                ts: new Date().toISOString(),
+                label: arg.label,
+                extra: arg.extra,
+            };
+            arg.attachTo.scenarioLog = note;
+            emitAugment(arg.attachTo.entrySeq, { kind: "scenarioLog", scenarioLog: note });
+        }),
+        sendInbound: (input) => sendInbound({ emit, nextEntrySeqTs }, accountId, conversationId, input),
+        createThread: (input) => createThread(accountId, conversationId, input),
+        poll: (opts) => poll(accountId, opts),
+        waitForOutbound: (predicate, opts) => waitForOutbound({
+            accountId,
+            awaitEntry: (id) => awaitOutboundEntry(id),
+            getLastCliMock: () => lastCliMock,
+        }, predicate, opts),
+        expectNoOutbound: (predicate, opts) => expectNoOutbound(accountId, predicate, opts),
+        assertRegex: (actual, pattern, label) => assertRegex({ getCurrentEntry: () => currentEntry, emitAugment }, actual, pattern, label),
+        assertEqual: (actual, expected, label) => assertEqual({ getCurrentEntry: () => currentEntry, emitAugment }, actual, expected, label),
+        assertLength: (value, expected, label) => assertLength({ getCurrentEntry: () => currentEntry, emitAugment }, value, expected, label),
+        judgeLLM: (p) => callJudgeVerdict({ emitAugment, judgeUsages }, p),
+        judgeLLMJson: (p) => callJudgeJson(judgeUsages, p),
+        judgeLLMRaw: (p) => callJudgeRaw(judgeUsages, p),
+        getCursor,
+        mockCli: (name, handler, opts) => registerMockCli(mockHandlers, name, handler, opts),
+        execInGateway: (argv, opts) => execInGateway(argv, opts),
+        waitForAgentToolCall: (predicate, opts) => waitForAgentToolCall({ conversationId, startedAtIso, getCurrentEntry: () => currentEntry, emitAugment }, predicate, opts),
+        getAgentToolCalls: () => parseAgentToolCalls({ conversationId, startedAtIso }),
+    };
+    const internals = {
+        finalize: (opts) => {
+            const result = computeResult(entries, currentEntry, emitAugment, opts?.failure);
+            return { entries, judgeUsages, result };
+        },
+        emitOutboundReceived,
+        emitCliMock,
+        getMockHandlers: () => mockHandlers,
+        peekEntries: () => ({ entries }),
+        isScenarioEnded: () => scenarioEnded,
+    };
+    function awaitOutboundEntry(messageId) {
+        const existing = outboundByMessageId.get(messageId);
+        if (existing)
+            return Promise.resolve(existing);
+        return new Promise((resolve) => {
+            outboundWaiters.set(messageId, resolve);
+        });
+    }
+    return { ctx, internals };
+}
+function computeResult(entries, currentEntry, emitAugment, scenarioFailure) {
+    if (scenarioFailure && currentEntry && !currentEntry.failure) {
+        currentEntry.failure = scenarioFailure;
+        emitAugment(currentEntry.entrySeq, { kind: "failure", failure: scenarioFailure });
+    }
+    const failedEntry = findFailedEntry(entries);
+    if (failedEntry) {
+        return {
+            verdict: "fail",
+            cause: "failedEntry",
+            entrySeq: failedEntry.entrySeq,
+            message: `[entry #${failedEntry.entrySeq}] ${failedEntry.failure?.message ?? ""}`,
+        };
+    }
+    if (!scenarioFailure)
+        return { verdict: "pass" };
+    const result = {
+        verdict: "fail",
+        cause: "error",
+        source: scenarioFailure.source,
+        errorName: scenarioFailure.name,
+        message: scenarioFailure.message,
+    };
+    if (scenarioFailure.stack)
+        result.stack = scenarioFailure.stack;
+    return result;
+}
+function findFailedEntry(entries) {
+    for (const e of entries) {
+        if (e.kind === "scenarioLog")
+            continue;
+        if (e.failure)
+            return e;
+    }
+    return;
+}
+function pushAssertion(deps, target, record) {
+    if (!target)
+        return;
+    if (!target.assertions)
+        target.assertions = [];
+    target.assertions.push(record);
+    deps.emitAugment(target.entrySeq, { kind: "assertion", assertion: record });
+}
+function registerMockCli(handlers, name, handler, opts) {
+    const mode = opts?.mode ?? "register";
+    const exists = handlers.has(name);
+    if (mode === "register") {
+        if (exists)
+            throw new Error(`mockCli: handler for "${name}" already registered`);
+        handlers.set(name, handler);
+        return;
+    }
+    if (mode === "replace") {
+        if (!exists)
+            throw new Error(`mockCli: no handler for "${name}" to replace`);
+        handlers.set(name, handler);
+        return;
+    }
+    // "ifAbsent"
+    if (!exists)
+        handlers.set(name, handler);
+}
+async function sendInbound(deps, accountId, conversationId, input) {
+    const conversation = input.conversation ?? {
+        kind: "channel",
+        id: conversationId,
+        title: conversationId,
+    };
+    const r = await injectQaBusInboundMessage({
+        baseUrl: BUS_URL,
+        input: {
+            accountId,
+            conversation,
+            senderId: input.senderId,
+            senderName: input.senderName,
+            text: input.text,
+            threadId: input.threadId,
+            threadTitle: input.threadTitle,
+        },
+    });
+    const entry = {
+        ...deps.nextEntrySeqTs(),
+        kind: "inboundSent",
+        messageId: r.message.id,
+        text: input.text,
+        senderId: input.senderId,
+        ...(input.senderName !== undefined ? { senderName: input.senderName } : {}),
+        ...(input.threadId !== undefined ? { threadId: input.threadId } : {}),
+    };
+    deps.emit(entry);
+    return { message: r.message, entry };
+}
+async function createThread(accountId, conversationId, input) {
+    const r = await createQaBusThread({
+        baseUrl: BUS_URL,
+        accountId,
+        conversationId,
+        title: input.title,
+        createdBy: input.createdBy,
+    });
+    return r.thread.id;
+}
+async function poll(accountId, opts) {
+    const timeoutMs = opts.timeoutMs ?? 1000;
+    // Client-side hard stop. The bus holds the long-poll for `timeoutMs` then answers, but a wedged
+    // bus or a half-open connection would hang this `fetch` forever — and callers loop on their own
+    // deadline (`waitForOutbound`), so a hung poll silently defeats that deadline and the runner never
+    // exits (the intermittent post-verdict hang). Abort a bit past the server hold so a stall surfaces
+    // as an empty poll and the caller's loop keeps ticking toward its real timeout.
+    const controller = new AbortController();
+    const abortTimer = setTimeout(() => controller.abort(), timeoutMs + 2_000);
+    let r;
+    try {
+        r = await pollQaBus({
+            baseUrl: BUS_URL,
+            accountId,
+            cursor: opts.sinceCursor,
+            timeoutMs,
+            signal: controller.signal,
+        });
+    }
+    catch (err) {
+        if (controller.signal.aborted)
+            return { messages: [], nextCursor: opts.sinceCursor };
+        throw err;
+    }
+    finally {
+        clearTimeout(abortTimer);
+    }
+    const messages = r.events
+        .filter((e) => e.kind === "outbound-message" || e.kind === "message-edited")
+        .map((e) => e.message);
+    return { messages, nextCursor: r.cursor };
+}
+export async function waitForOutbound(deps, predicate, opts) {
+    const timeoutMs = opts.timeoutMs ?? 30_000;
+    const maxUnmatched = opts.failFastUnmatchedOutbounds ?? 3;
+    // 60s: OpenClaw 2026.8 turns run noticeably longer between a CLI call and the
+    // turn-end post, and Slack sessions post nothing before turn end.
+    const cliMockGraceMs = opts.failFastCliMockGraceMs ?? 60_000;
+    const pollFn = opts.pollImpl ?? poll;
+    const now = opts.nowImpl ?? Date.now;
+    const deadline = now() + timeoutMs;
+    // The grace fail-fast is a liveness heuristic: it fires only when the newest
+    // activity this wait has observed is a mocked-CLI call that then went quiet.
+    // A CLI call from before the wait started never arms it, and any outbound
+    // observed after the call disarms it (the unmatched fail-fast polices those).
+    const baselineCliSeq = deps.getLastCliMock()?.entry.entrySeq ?? -1;
+    let lastOutboundAtMs;
+    let cursor = opts.sinceCursor;
+    let unmatched = 0;
+    let lastUnmatched;
+    while (now() < deadline) {
+        const remaining = Math.max(0, deadline - now());
+        const { messages, nextCursor } = await pollFn(deps.accountId, {
+            sinceCursor: cursor,
+            timeoutMs: Math.min(5000, remaining),
+        });
+        cursor = nextCursor;
+        const match = messages.find(predicate);
+        if (match) {
+            const entry = await deps.awaitEntry(match.id);
+            return { match, entry, nextCursor };
+        }
+        if (messages.length > 0) {
+            lastOutboundAtMs = now();
+            unmatched += messages.length;
+            lastUnmatched = messages[messages.length - 1];
+            if (maxUnmatched !== false && unmatched >= maxUnmatched) {
+                throw await unmatchedFastFailError(deps, unmatched, lastUnmatched);
+            }
+        }
+        if (cliMockGraceMs !== false) {
+            const last = deps.getLastCliMock();
+            const armed = last !== undefined &&
+                last.entry.entrySeq > baselineCliSeq &&
+                (lastOutboundAtMs === undefined || lastOutboundAtMs < last.atMs);
+            if (armed && now() - last.atMs >= cliMockGraceMs) {
+                throw cliMockGraceFastFailError(cliMockGraceMs, last.entry);
+            }
+        }
+    }
+    throw new AssertionError(`waitForOutbound timed out after ${timeoutMs}ms`);
+}
+function truncate(text, max = 80) {
+    const flat = text.replace(/\s+/g, " ");
+    if (flat.length <= max)
+        return flat;
+    return `${flat.slice(0, max - 1)}…`;
+}
+async function unmatchedFastFailError(deps, count, msg) {
+    const entry = await raceTimeout(deps.awaitEntry(msg.id), 50);
+    const seqOrId = entry ? `entrySeq=${entry.entrySeq}` : `msg=${msg.id}`;
+    const thread = msg.threadId !== undefined ? `threadId=${msg.threadId}` : "no threadId";
+    const text = truncate(msg.text);
+    return new AssertionError(`waitForOutbound: agent posted ${count} outbounds but none matched the predicate\n` +
+        `  observed: outbound ${seqOrId} (${thread}, text=${JSON.stringify(text)})`);
+}
+function cliMockGraceFastFailError(graceMs, entry) {
+    const cli = entry.call.cli;
+    const argvHead = truncate(entry.call.argv.join(" "));
+    return new AssertionError(`waitForOutbound: agent invoked a mocked CLI during this wait and posted no outbound for ${graceMs}ms\n` +
+        `  observed: cliMock ${cli} (argv head: ${JSON.stringify(argvHead)}), then silence`);
+}
+function raceTimeout(p, ms) {
+    return Promise.race([
+        p,
+        new Promise((resolve) => setTimeout(() => resolve(undefined), ms)),
+    ]);
+}
+async function expectNoOutbound(accountId, predicate, opts) {
+    const deadline = Date.now() + opts.withinMs;
+    let cursor = opts.sinceCursor;
+    while (Date.now() < deadline) {
+        const remaining = Math.max(0, deadline - Date.now());
+        const { messages, nextCursor } = await poll(accountId, {
+            sinceCursor: cursor,
+            timeoutMs: Math.min(remaining, 1000),
+        });
+        cursor = nextCursor;
+        const offender = messages.find(predicate);
+        if (offender) {
+            throw new AssertionError(`expectNoOutbound: forbidden message arrived: ${JSON.stringify({
+                id: offender.id,
+                text: offender.text,
+                threadId: offender.threadId,
+            })}`);
+        }
+    }
+    return { nextCursor: cursor };
+}
+async function waitForAgentToolCall(deps, predicate, opts) {
+    const timeoutMs = opts.timeoutMs ?? 30_000;
+    // Each poll spawns a full transcript dump inside the gateway container —
+    // keep the cadence low enough not to load the system under test.
+    const pollMs = opts.pollMs ?? 2_000;
+    const deadline = Date.now() + timeoutMs;
+    let calls = [];
+    while (Date.now() < deadline) {
+        calls = await parseAgentToolCalls({
+            conversationId: deps.conversationId,
+            startedAtIso: deps.startedAtIso,
+        });
+        const match = calls.find(predicate);
+        if (match) {
+            pushAssertion(deps, deps.getCurrentEntry(), { label: opts.label, ok: true });
+            return match;
+        }
+        await new Promise((r) => setTimeout(r, pollMs));
+    }
+    const detail = `no agent tool call matched within ${timeoutMs}ms; observed: ${summarizeToolCalls(calls)}`;
+    pushAssertion(deps, deps.getCurrentEntry(), { label: opts.label, ok: false, detail });
+    throw new AssertionError(`${opts.label}: ${detail}`);
+}
+function summarizeToolCalls(calls) {
+    if (calls.length === 0)
+        return "(none)";
+    return calls.map((c) => `${c.toolName}${toolCallHint(c.input)}`).join("; ");
+}
+function toolCallHint(input) {
+    if (!isRecord(input))
+        return "";
+    if (typeof input.path === "string")
+        return ` ${input.path}`;
+    if (typeof input.command === "string")
+        return ` ${input.command.slice(0, 60)}`;
+    return "";
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null;
+}
+function assertRegex(deps, actual, pattern, label) {
+    failingAssert(deps, pattern.test(actual), label, `value=${JSON.stringify(actual)} pattern=${pattern.source}`);
+}
+function assertEqual(deps, actual, expected, label) {
+    failingAssert(deps, actual === expected, label, `expected=${JSON.stringify(expected)} actual=${JSON.stringify(actual)}`);
+}
+function assertLength(deps, value, expected, label) {
+    failingAssert(deps, value.length === expected, label, `expected length=${expected} actual=${value.length}`);
+}
+function failingAssert(deps, ok, label, detail) {
+    if (ok)
+        return;
+    pushAssertion(deps, deps.getCurrentEntry(), { label, ok: false, detail });
+    throw new AssertionError(`${label}: ${detail}`);
+}
+async function callJudgeVerdict(deps, p) {
+    const verdict = await judgeLLM({
+        message: p.message,
+        rubric: p.rubric,
+        maxTokens: p.maxTokens,
+    });
+    deps.judgeUsages.push(verdict.usage);
+    const extra = {
+        judge: {
+            model: verdict.usage.model,
+            usage: {
+                inputTokens: verdict.usage.inputTokens,
+                outputTokens: verdict.usage.outputTokens,
+            },
+            costUsd: judgeCostUsd(verdict.usage),
+        },
+    };
+    if (verdict.verdict === "pass") {
+        pushAssertion(deps, p.attachTo, { label: p.label, ok: true, extra });
+        return verdict;
+    }
+    pushAssertion(deps, p.attachTo, { label: p.label, ok: false, detail: verdict.reasoning, extra });
+    throw new AssertionError(`${p.label}: ${verdict.reasoning}`);
+}
+async function callJudgeJson(judgeUsages, p) {
+    const result = await judgeLLMJson({
+        message: p.message,
+        prompt: p.prompt,
+        returnType: p.returnType,
+        maxTokens: p.maxTokens,
+    });
+    judgeUsages.push(result.usage);
+    return result;
+}
+async function callJudgeRaw(judgeUsages, p) {
+    const result = await judgeLLMRaw(p.prompt, { maxTokens: p.maxTokens });
+    judgeUsages.push(result.usage);
+    return result;
+}
+async function getCursor() {
+    const snap = await getQaBusState(BUS_URL);
+    return snap.cursor;
+}

package/dist/cost.d.ts

@@ -0,0 +1,2 @@
+import type { JudgeUsage } from "./judge.js";
+export declare function judgeCostUsd(usage: JudgeUsage): number;

package/dist/cost.js

@@ -0,0 +1,19 @@
+// USD per million tokens. Judge never uses prompt caching. Keys are LiteLLM-style
+// "provider/model" refs. Add models as needed.
+const JUDGE_PRICING = {
+    "anthropic/claude-haiku-4-5": { input: 1.0, output: 5.0 },
+    "openrouter/anthropic/claude-haiku-4.5": { input: 1.0, output: 5.0 },
+};
+const warnedUnknownModels = new Set();
+export function judgeCostUsd(usage) {
+    const key = usage.model.replace(/-\d{8}$/, "");
+    const price = JUDGE_PRICING[key];
+    if (!price) {
+        if (!warnedUnknownModels.has(key)) {
+            warnedUnknownModels.add(key);
+            console.warn(`cost: no JUDGE_PRICING entry for ${JSON.stringify(key)}; judge cost will report 0. Add it to src/cost.ts.`);
+        }
+        return 0;
+    }
+    return ((usage.inputTokens * price.input) / 1_000_000 + (usage.outputTokens * price.output) / 1_000_000);
+}

package/dist/env-cli.d.ts

@@ -0,0 +1,5 @@
+export declare function envCommand(packageDir: string, argv: string[]): Promise<never>;
+export declare function buildConsumerImageArgs(composeArgs: string[]): string[];
+export declare function runCommand(packageDir: string, argv: string[]): Promise<never>;
+/** Flag wins over the env var; fallback 1. Both must parse to an integer ≥ 1. */
+export declare function resolveParallel(flag: string | undefined, envValue: string | undefined): number;