@alignfirst/openclaw-test 0.21.1 → 0.23.0

package/README.md

@@ -45,8 +45,8 @@ OPENROUTER_API_KEY=sk-or-… # Required for an OpenRouter agent or judge.
 OPENCLAW_WORKSPACE_DIR=/path/to/your/openclaw-workspace
 
 # Model catalog: full LiteLLM refs. `run --model` picks by bare id (suffix after the last "/").
-OPENCLAW_TEST_MODELS=anthropic/claude-sonnet-4-6,custom-openrouter/qwen/qwen3.6-plus
-OPENCLAW_DEFAULT_TEST_MODEL=claude-sonnet-4-6
+OPENCLAW_TEST_MODELS=anthropic/claude-sonnet-5,custom-openrouter/qwen/qwen3.8-flash
+OPENCLAW_DEFAULT_TEST_MODEL=claude-sonnet-5
 
 ```
 
@@ -84,8 +84,8 @@ npm run env:up                                                     # (optional)
 npm run e2e -- --channel all <scenario>                             # one scenario, both channels
 npm run e2e -- --channel all --all                                  # every scenario, both channels
 npm run e2e -- --channel discord-mock <scenario>                    # restrict to one channel
-npm run e2e -- --channel all --model qwen3.6-plus <scenario>        # pick a model by bare id
-npm run e2e -- --channel all --model claude-sonnet-4-6,qwen3.6-plus <s>  # a comma list of bare ids
+npm run e2e -- --channel all --model qwen3.8-flash <scenario>        # pick a model by bare id
+npm run e2e -- --channel all --model claude-sonnet-5,qwen3.8-flash <s>  # a comma list of bare ids
 npm run e2e -- --channel all --model all <scenario>                 # run every model in OPENCLAW_TEST_MODELS
 npm run e2e -- --channel all --iterations 5 <scenario>              # repeat each (scenario, channel) pair 5×
 npm run e2e -- --channel all --iterations 5 --max-failures 1 <s>    # abort a pair after >1 failure
@@ -121,10 +121,10 @@ Assert on `conversation.id` / `threadId`, not envelope formatting.
 
 ## Judge model
 
-Defaults to `anthropic/claude-haiku-4-5`. Override it via `OPENCLAW_TEST_JUDGE_MODEL` on
-the `runner` service (set in your consumer overlay). Direct Anthropic refs use
-`anthropic/<model>`; OpenRouter refs use `openrouter/<model>`, for example
-`openrouter/anthropic/claude-haiku-4.5`. The judge is **not** an OpenClaw agent — don't
+Defaults to `openrouter/anthropic/claude-haiku-4.5`, billed to `OPENROUTER_API_KEY`. Override it
+via `OPENCLAW_TEST_JUDGE_MODEL` on the `runner` service (set in your consumer overlay). OpenRouter
+refs use `openrouter/<model>`; direct Anthropic refs use `anthropic/<model>`, for example
+`anthropic/claude-haiku-4-5`. The judge is **not** an OpenClaw agent — don't
 configure it in `openclaw.json`.
 
 ## Attribution

package/dist/env-cli.js

@@ -20,7 +20,7 @@ const RUN_USAGE = `usage: openclaw-test run --channel <id|id,id,…|all> [<scena
 
   Scenario selection is required: either a positional list or --all (mutually exclusive).
   --model <id|id,id,…|all>
-                      select the agent model(s): a bare id (e.g. claude-sonnet-4-6), a
+                      select the agent model(s): a bare id (e.g. claude-sonnet-5), a
                       comma list of bare ids, or "all". Defaults to OPENCLAW_DEFAULT_TEST_MODEL.
                       The catalog is OPENCLAW_TEST_MODELS (.env.local), a comma list of full
                       provider/model refs; the bare id is the suffix after the last "/".

package/dist/gateway-log.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Copies the gateway's file log into the cell's artifact dir as `gateway.log`.
+ * Under `--reuse-stack` the gateway outlives the cell, so the copy also holds
+ * earlier cells' lines. Best-effort: a failure is reported, never fatal.
+ */
+export declare function archiveGatewayLog(outDir: string): Promise<void>;

package/dist/gateway-log.js

@@ -0,0 +1,36 @@
+import { randomUUID } from "node:crypto";
+import { copyFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { execInGateway, IPC_DIR } from "./exec-rpc.js";
+// OpenClaw's rolling file log, `openclaw-<date>.log` under its temp root. It lives in
+// the gateway container, which the per-cell stack recreation destroys. Provider
+// failures such as an HTTP 401 are recorded there and nowhere in the transcripts.
+const GATEWAY_LOG_GLOB = "/tmp/openclaw/openclaw-*.log";
+/**
+ * Copies the gateway's file log into the cell's artifact dir as `gateway.log`.
+ * Under `--reuse-stack` the gateway outlives the cell, so the copy also holds
+ * earlier cells' lines. Best-effort: a failure is reported, never fatal.
+ */
+export async function archiveGatewayLog(outDir) {
+    const ipcPath = `${IPC_DIR}/${randomUUID()}.gateway.log`;
+    try {
+        const result = await execInGateway([
+            "sh",
+            "-c",
+            `cat ${GATEWAY_LOG_GLOB} > "$1"`,
+            "sh",
+            ipcPath,
+        ]);
+        if (result.exitCode !== 0) {
+            console.warn(`openclaw-test: gateway log copy failed (exit ${result.exitCode}): ${result.stderr}`);
+            return;
+        }
+        copyFileSync(ipcPath, join(outDir, "gateway.log"));
+    }
+    catch (err) {
+        console.warn(`openclaw-test: gateway log copy failed: ${String(err)}`);
+    }
+    finally {
+        rmSync(ipcPath, { force: true });
+    }
+}

package/dist/judge.js

@@ -1,7 +1,7 @@
 import Anthropic from "@anthropic-ai/sdk";
 import OpenAI from "openai";
 import { extractTaggedBlock } from "./parse-tagged-json.js";
-const DEFAULT_JUDGE_MODEL = "anthropic/claude-haiku-4-5";
+const DEFAULT_JUDGE_MODEL = "openrouter/anthropic/claude-haiku-4.5";
 const DEFAULT_MAX_TOKENS = 1024;
 const RESULT_TAG = "result-json";
 let cached;

package/dist/runner.js

@@ -5,6 +5,7 @@ import { basename, join } from "node:path";
 import { cellLeafName, writeCellResult } from "./cell-result.js";
 import { createContext, } from "./context.js";
 import { judgeCostUsd } from "./cost.js";
+import { archiveGatewayLog } from "./gateway-log.js";
 import { aggregateAgentToolCalls, fetchTranscriptSnapshot, readTranscriptCost, waitForTranscriptQuiescence, } from "./transcript-log.js";
 import { startMockCliServer } from "./mock-cli-server.js";
 import { parseArgs } from "./runner-args.js";
@@ -49,6 +50,7 @@ async function runCell(args) {
         // The per-session store dies with the stack recreation, so keep the raw
         // transcripts as a cell artifact for post-mortems.
         writeFileSync(join(outDir, "transcripts.json"), JSON.stringify(snapshot.sessions));
+        await archiveGatewayLog(outDir);
         const agentCalls = aggregateAgentToolCalls(snapshot.sessions);
         pairAgentCallsWithCliMocks(agentCalls, entries);
         if (agentCalls.length === 0 && (snapshot.error !== undefined || snapshot.databases === 0)) {

package/dist/transcript-store.js

@@ -1,4 +1,5 @@
 import { DatabaseSync } from "node:sqlite";
+import { zstdDecompressSync } from "node:zlib";
 export function readConversationSessions(dbPath, sinceMs, conversationId, threadIds) {
     let db;
     try {
@@ -51,19 +52,28 @@ function readNodeMessages(db, node, sinceMs) {
 }
 function readSessionMessages(db, sessionId, sinceMs) {
     const rows = db
-        .prepare("SELECT event_json FROM transcript_events WHERE session_id = ? AND created_at >= ? ORDER BY seq")
+        .prepare("SELECT event_json, event_zstd FROM transcript_events" +
+        " WHERE session_id = ? AND created_at >= ? ORDER BY seq")
         .all(sessionId, sinceMs);
     const messages = [];
     for (const row of rows) {
         let event;
         try {
-            event = JSON.parse(row.event_json);
+            event = JSON.parse(decodeTranscriptEvent(row));
         }
         catch {
             continue;
         }
-        if (event.type === "message" && event.message !== undefined)
+        if (event?.type === "message" && event.message !== undefined)
             messages.push(event.message);
     }
     return messages;
 }
+/** Since OpenClaw 2026.9.6, an event of 1 KiB or more is stored zstd-compressed, `event_json` NULL. */
+function decodeTranscriptEvent(row) {
+    if (row.event_json !== null)
+        return row.event_json;
+    if (row.event_zstd === null)
+        throw new Error("transcript event without a payload");
+    return zstdDecompressSync(row.event_zstd).toString("utf8");
+}

package/exec-watcher.mjs

@@ -12,6 +12,7 @@ const STALE_SUFFIXES = [
   ".res.json.tmp",
   ".transcript.json",
   ".transcript.json.tmp",
+  ".gateway.log",
 ];
 
 mkdirSync(IPC_DIR, { recursive: true });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alignfirst/openclaw-test",
-  "version": "0.21.1",
+  "version": "0.23.0",
   "description": "Dockerised regression-test framework for OpenClaw workspaces: bus, scenario driver, judge, Compose stack.",
   "keywords": [
     "openclaw",
@@ -44,15 +44,15 @@
     "openclaw": "*"
   },
   "dependencies": {
-    "@alignfirst/openclaw-channel-mock-core": "0.10.1",
-    "@alignfirst/openclaw-discord-mock": "0.6.1",
-    "@alignfirst/openclaw-slack-mock": "0.6.1",
+    "@alignfirst/openclaw-channel-mock-core": "0.10.3",
+    "@alignfirst/openclaw-discord-mock": "0.6.3",
+    "@alignfirst/openclaw-slack-mock": "0.6.3",
     "@anthropic-ai/sdk": "~0.127.0",
     "openai": "~7.8.0"
   },
   "devDependencies": {
     "@types/node": "~26.5.0",
-    "openclaw": "~2026.9.3",
+    "openclaw": "~2026.9.8",
     "rimraf": "~6.1.3",
     "typescript": "~7.0.2",
     "vitest": "~5.0.1"

package/templates/.env.local.example

@@ -7,8 +7,8 @@ OPENROUTER_API_KEY=
 # place the provider/ prefix appears. `run --model <id>` and OPENCLAW_DEFAULT_TEST_MODEL
 # use the bare id (the suffix after the last "/"). `--model all` runs every entry and
 # needs the API key for each referenced provider.
-OPENCLAW_TEST_MODELS=anthropic/claude-sonnet-4-6,custom-openrouter/qwen/qwen3.6-plus
-OPENCLAW_DEFAULT_TEST_MODEL=claude-sonnet-4-6
+OPENCLAW_TEST_MODELS=anthropic/claude-sonnet-5,custom-openrouter/qwen/qwen3.8-flash
+OPENCLAW_DEFAULT_TEST_MODEL=claude-sonnet-5
 
 # Required: host path to the OpenClaw workspace (mounted into the gateway).
 OPENCLAW_WORKSPACE_DIR=