@beryl-so/cli 0.77.1 → 0.78.0

package/dist/beryl-test-skill.js

@@ -425,6 +425,13 @@ observable proof the flow worked.
   chrome. Target the element that carries the text (the \`h1\`, the toast), assert the
   page with \`have_title\` (substring), or check "this string is visible somewhere"
   with \`expect_kind: "visible"\` + a \`text=…\` selector.
+- **For output that varies run to run** (a chatbot or AI assistant reply), assert its
+  meaning with \`expect_kind: "judged"\`: \`selector\` locates the reply (the LAST match
+  is judged, so a selector matching every assistant message reads the newest one) and
+  \`expect_text\` states the criteria in plain words ("Lists at least two decisions and
+  names the people involved; no error or refusal"). Beryl waits up to 120s for the text
+  to appear and stop changing, then a model passes or fails it and the failure carries
+  the model's reason. Write criteria a correct answer always meets, never exact wording.
 - **If you can't name a success signal, the flow is not test-worthy.** Don't bank a
   test that verifies nothing; explore a different flow.
 - **Don't work around a real app failure to make a test go green.** A genuinely broken

package/dist/commands/runs.js

@@ -152,9 +152,10 @@ export const runCommands = [
             "settled tests only: anything still running, healing or queued is left out. A result " +
             "carries `heal` when Beryl healed it: its steps were rewritten with the intent " +
             "unchanged, proved in a browser and it re-ran green; `version_no` is the revision it " +
-            "wrote (see `tests diff`) and `rationale` the agent's reason. A heal that did not turn " +
-            "the test green is not reported: a failure is a failure. A red that Beryl did not try " +
-            "to heal carries `heal_skip_reason` saying why. " +
+            "wrote (see `tests diff`) and `rationale` the agent's reason. A red that Beryl tried and " +
+            "failed to heal carries `heal_attempt`: `status` is how the heal ended, `reason` why, " +
+            "and `heal_job_id` the job. A red that Beryl did not try to heal carries " +
+            "`heal_skip_reason` saying why. " +
             "`runs report` returns the generated report document, `runs explain` an AI explanation " +
             "of one failed result.",
         scope: "project",

package/dist/commands/watch.js

@@ -46,6 +46,13 @@ export function findingsGateLine(gate) {
     }
     return `findings gate unknown: ${gate.reason ?? "not evaluated"}`;
 }
+export function healNote(r) {
+    if (r.heal_attempt) {
+        const why = r.heal_attempt.reason ? `: ${r.heal_attempt.reason.slice(0, 300)}` : "";
+        return `Auto-heal did not fix it (${r.heal_attempt.status})${why}`;
+    }
+    return r.heal_skip_reason ?? null;
+}
 // Auth, client and wrong-host errors fail identically on every retry; only a network
 // blip, a timeout, throttling or a 5xx is worth waiting out.
 function isRetryable(e) {
@@ -102,8 +109,9 @@ export async function watchRun(ctx, ws, project, runId, timeoutMinutes, opts = {
             ctx.err(red(detail.replace(/^/gm, "    ")));
             process.stdout.write(`::error title=${escapeAnnotationProp(r.title || r.id)}::${escapeAnnotationData(detail)}\n`);
         }
-        if (!ok && r.heal_skip_reason)
-            ctx.err(dim(`    ${r.heal_skip_reason}`));
+        const note = ok ? null : healNote(r);
+        if (note)
+            ctx.err(dim(`    ${note}`));
     };
     if (!ctx.json)
         ctx.err(dim(`watching run ${runId}`));
@@ -260,14 +268,17 @@ export async function watchRun(ctx, ws, project, runId, timeoutMinutes, opts = {
             `| ${counters.passed} | ${counters.failed} | ${counters.errored} | ${counters.cancelled} | ${counters.quarantined} | ${counters.total} |`,
             ...(gate ? ["", findingsGateLine(gate)] : []),
             ...(link ? ["", `[Open the run in Beryl](${link})`] : []),
-            ...failures.flatMap((r) => [
-                "",
-                `**✗ ${r.title || r.id}**`,
-                "```",
-                String(r.error_message ?? "").slice(0, CI_ERROR_MAX_LEN),
-                "```",
-                ...(r.heal_skip_reason ? ["", r.heal_skip_reason] : []),
-            ]),
+            ...failures.flatMap((r) => {
+                const note = healNote(r);
+                return [
+                    "",
+                    `**✗ ${r.title || r.id}**`,
+                    "```",
+                    String(r.error_message ?? "").slice(0, CI_ERROR_MAX_LEN),
+                    "```",
+                    ...(note ? ["", note] : []),
+                ];
+            }),
             "",
         ];
         try {

package/dist/schema.generated.js

@@ -85,7 +85,8 @@ export const ACTION_PLAN_SCHEMA = {
                 "have_count",
                 "persisted",
                 "gone",
-                "count_delta"
+                "count_delta",
+                "judged"
             ],
             "title": "ExpectKind",
             "type": "string"
@@ -835,6 +836,35 @@ export const ACTION_PLAN_SCHEMA = {
                         ]
                     }
                 },
+                {
+                    "if": {
+                        "properties": {
+                            "expect_kind": {
+                                "const": "judged"
+                            }
+                        },
+                        "required": [
+                            "expect_kind"
+                        ]
+                    },
+                    "then": {
+                        "properties": {
+                            "expect_text": {
+                                "not": {
+                                    "type": "null"
+                                }
+                            },
+                            "selector": {
+                                "minLength": 1,
+                                "type": "string"
+                            }
+                        },
+                        "required": [
+                            "expect_text",
+                            "selector"
+                        ]
+                    }
+                },
                 {
                     "if": {
                         "properties": {

package/dist/skill-tables.js

@@ -57,6 +57,7 @@ export const EXPECT_DOCS = {
     persisted: "The record just created is present — a visible match for the `{{unique}}`-named row",
     gone: "Zero matches — absent from the DOM, stronger than `hidden`",
     count_delta: "The match count moved by signed `expect_delta` vs the `capture_ref` baseline",
+    judged: "A model judges the LAST match's text, once it stops changing, against the plain-language criteria in `expect_text`: for answers that vary in wording (a chatbot reply)",
 };
 export const EXTRACT_DOCS = {
     code: "the one-time code",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@beryl-so/cli",
-  "version": "0.77.1",
+  "version": "0.78.0",
   "description": "Beryl on the command line: projects, tests, runs, and an MCP server over the same commands.",
   "license": "MIT",
   "type": "module",