@beryl-so/cli 0.24.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/mcp.js +111 -15
- package/dist/beryl-test-skill.js +241 -264
- package/dist/commands/accounts.js +8 -5
- package/dist/commands/config-vars.js +5 -0
- package/dist/commands/init.js +2 -2
- package/dist/commands/mailboxes.js +5 -1
- package/dist/commands/runs.js +19 -6
- package/dist/commands/tests.js +36 -6
- package/dist/skill-tables.js +74 -0
- package/package.json +1 -1
package/dist/adapters/mcp.js
CHANGED
|
@@ -6,6 +6,7 @@ import { loadConfig } from "../config.js";
|
|
|
6
6
|
import { createContext } from "../context.js";
|
|
7
7
|
import { CliError } from "../errors.js";
|
|
8
8
|
import { ApiClient } from "../http.js";
|
|
9
|
+
import { parseArgv } from "./cli.js";
|
|
9
10
|
import { commands } from "../registry/index.js";
|
|
10
11
|
import { cliVersion, warnIfStale } from "../version-check.js";
|
|
11
12
|
export function toolName(spec) {
|
|
@@ -31,15 +32,103 @@ export function toolInputSchema(spec) {
|
|
|
31
32
|
required.push(a.name);
|
|
32
33
|
}
|
|
33
34
|
for (const f of spec.flags ?? []) {
|
|
35
|
+
const withDefault = f.default !== undefined ? { default: f.default } : {};
|
|
34
36
|
properties[f.name] =
|
|
35
37
|
f.type === "strings"
|
|
36
|
-
? { type: "array", items: { type: "string" }, description: f.description }
|
|
37
|
-
: {
|
|
38
|
+
? { type: "array", items: { type: "string" }, description: f.description, ...withDefault }
|
|
39
|
+
: {
|
|
40
|
+
type: f.type,
|
|
41
|
+
description: f.description,
|
|
42
|
+
...(f.enum ? { enum: f.enum } : {}),
|
|
43
|
+
...withDefault,
|
|
44
|
+
};
|
|
38
45
|
if (f.required)
|
|
39
46
|
required.push(f.name);
|
|
40
47
|
}
|
|
41
48
|
return { type: "object", properties, ...(required.length ? { required } : {}) };
|
|
42
49
|
}
|
|
50
|
+
function shellTokens(text) {
|
|
51
|
+
const tokens = [];
|
|
52
|
+
let current = "";
|
|
53
|
+
let quote = null;
|
|
54
|
+
let pending = false;
|
|
55
|
+
for (const ch of text) {
|
|
56
|
+
if (quote) {
|
|
57
|
+
if (ch === quote)
|
|
58
|
+
quote = null;
|
|
59
|
+
else
|
|
60
|
+
current += ch;
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
if (ch === '"' || ch === "'") {
|
|
64
|
+
quote = ch;
|
|
65
|
+
pending = true;
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
if (/\s/.test(ch)) {
|
|
69
|
+
if (pending || current)
|
|
70
|
+
tokens.push(current);
|
|
71
|
+
current = "";
|
|
72
|
+
pending = false;
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
if (">|;#&".includes(ch))
|
|
76
|
+
return null;
|
|
77
|
+
current += ch;
|
|
78
|
+
pending = true;
|
|
79
|
+
}
|
|
80
|
+
if (quote)
|
|
81
|
+
return null;
|
|
82
|
+
if (pending || current)
|
|
83
|
+
tokens.push(current);
|
|
84
|
+
return tokens;
|
|
85
|
+
}
|
|
86
|
+
/** A CLI example translated to the JSON args the MCP tool takes, or null when it doesn't
|
|
87
|
+
* translate cleanly (shell syntax, another command's example, nothing beyond defaults) —
|
|
88
|
+
* agents must see tool args as JSON, never `--flag` syntax. */
|
|
89
|
+
export function exampleArgs(spec, example) {
|
|
90
|
+
const prefix = `beryl ${spec.name}`;
|
|
91
|
+
if (example !== prefix && !example.startsWith(`${prefix} `))
|
|
92
|
+
return null;
|
|
93
|
+
const tokens = shellTokens(example.slice(prefix.length));
|
|
94
|
+
if (!tokens)
|
|
95
|
+
return null;
|
|
96
|
+
let parsed;
|
|
97
|
+
try {
|
|
98
|
+
parsed = parseArgv(spec, tokens);
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
if (parsed.help)
|
|
104
|
+
return null;
|
|
105
|
+
const byName = new Map((spec.flags ?? []).map((f) => [f.name, f]));
|
|
106
|
+
const out = {};
|
|
107
|
+
for (const [name, value] of Object.entries(parsed.input.args)) {
|
|
108
|
+
if (value === undefined || (Array.isArray(value) && value.length === 0))
|
|
109
|
+
continue;
|
|
110
|
+
out[name] = value;
|
|
111
|
+
}
|
|
112
|
+
for (const [name, value] of Object.entries(parsed.input.flags)) {
|
|
113
|
+
const f = byName.get(name);
|
|
114
|
+
// parseArgv fills declared defaults in; only what the example explicitly set teaches.
|
|
115
|
+
if (value === undefined || value === f?.default)
|
|
116
|
+
continue;
|
|
117
|
+
out[name] =
|
|
118
|
+
f?.type === "number" && typeof value === "string" && Number.isFinite(Number(value))
|
|
119
|
+
? Number(value)
|
|
120
|
+
: value;
|
|
121
|
+
}
|
|
122
|
+
return Object.keys(out).length ? out : null;
|
|
123
|
+
}
|
|
124
|
+
export function toolDescription(spec) {
|
|
125
|
+
const base = spec.description ? `${spec.summary}. ${spec.description}` : spec.summary;
|
|
126
|
+
const lines = (spec.examples ?? [])
|
|
127
|
+
.map((e) => exampleArgs(spec, e))
|
|
128
|
+
.filter((a) => a !== null)
|
|
129
|
+
.map((a) => `Example: ${JSON.stringify(a)}`);
|
|
130
|
+
return lines.length ? `${base}\n${lines.join("\n")}` : base;
|
|
131
|
+
}
|
|
43
132
|
export function toolResult(result, lines) {
|
|
44
133
|
const parts = [...lines];
|
|
45
134
|
if (result.data !== undefined)
|
|
@@ -102,29 +191,36 @@ export function currentAuth(fallback) {
|
|
|
102
191
|
export function __resetAuthCacheForTests() {
|
|
103
192
|
authCache = undefined;
|
|
104
193
|
}
|
|
194
|
+
// The running version is stated up front because this server is long-lived and never
|
|
195
|
+
// hot-reloads: a session can sit on a days-old build while `@latest` has moved, and
|
|
196
|
+
// "that tool doesn't exist for me" is indistinguishable from a bug without it. Saying
|
|
197
|
+
// it here means the model knows without spending a `version` tool call.
|
|
198
|
+
export function mcpInstructions() {
|
|
199
|
+
const version = cliVersion();
|
|
200
|
+
return (`Beryl CLI v${version} (call the \`version\` tool for the API URL, Node ` +
|
|
201
|
+
"version, and whether this build is behind npm's latest). " +
|
|
202
|
+
"Beryl authors, runs, and heals end-to-end tests for any web app: tests are JSON " +
|
|
203
|
+
"action plans replayed in real cloud browsers, signing in as a durable test " +
|
|
204
|
+
"account whose mail arrives at the project's own mailbox — so signup/OTP/" +
|
|
205
|
+
"magic-link flows are self-contained, with no human login needed. " +
|
|
206
|
+
"The full authoring guide (plan shape, outcome assertions, email/OTP wiring, " +
|
|
207
|
+
"run-fix loop) ships as both the beryl-test skill and the `guide` tool — same " +
|
|
208
|
+
`content. If a beryl-test skill stating v${version} is already loaded, do not ` +
|
|
209
|
+
"call `guide`; if no beryl-test skill is available or it states another version, " +
|
|
210
|
+
"call `guide` before authoring your first test plan.");
|
|
211
|
+
}
|
|
105
212
|
export async function serveMcp(baseCtx) {
|
|
106
213
|
// Fire-and-forget staleness warning: a stale MCP server silently exposes fewer
|
|
107
214
|
// tools, and stderr is the one channel a stdio MCP server can safely log to.
|
|
108
215
|
void warnIfStale(cliVersion(), (msg) => console.error(msg));
|
|
109
216
|
const server = new Server({ name: "beryl", version: cliVersion() }, {
|
|
110
217
|
capabilities: { tools: {} },
|
|
111
|
-
|
|
112
|
-
// hot-reloads: a session can sit on a days-old build while `@latest` has moved, and
|
|
113
|
-
// "that tool doesn't exist for me" is indistinguishable from a bug without it. Saying
|
|
114
|
-
// it here means the model knows without spending a `version` tool call.
|
|
115
|
-
instructions: `Beryl CLI v${cliVersion()} (call the \`version\` tool for the API URL, Node ` +
|
|
116
|
-
"version, and whether this build is behind npm's latest). " +
|
|
117
|
-
"Beryl authors, runs, and heals end-to-end tests for any web app: tests are JSON " +
|
|
118
|
-
"action plans replayed in real cloud browsers, signing in as a durable test " +
|
|
119
|
-
"account whose mail arrives at the project's own mailbox — so signup/OTP/" +
|
|
120
|
-
"magic-link flows are self-contained, with no human login needed. Before " +
|
|
121
|
-
"authoring your first test plan, call the `guide` tool — it returns the full " +
|
|
122
|
-
"authoring guide (plan shape, outcome assertions, email/OTP wiring, run-fix loop).",
|
|
218
|
+
instructions: mcpInstructions(),
|
|
123
219
|
});
|
|
124
220
|
server.setRequestHandler(ListToolsRequestSchema, () => ({
|
|
125
221
|
tools: mcpTools().map((spec) => ({
|
|
126
222
|
name: toolName(spec),
|
|
127
|
-
description: spec
|
|
223
|
+
description: toolDescription(spec),
|
|
128
224
|
inputSchema: toolInputSchema(spec),
|
|
129
225
|
})),
|
|
130
226
|
}));
|
package/dist/beryl-test-skill.js
CHANGED
|
@@ -5,6 +5,12 @@
|
|
|
5
5
|
// embedded string so it ships in the published package (`files: ["dist"]`) with no
|
|
6
6
|
// build-time asset copy, and so there is ONE source for the guidance — not a copy in
|
|
7
7
|
// the CLI and another in the docs. Edit here; `init` writes it verbatim.
|
|
8
|
+
import { actionTable, expectTable, extractLine } from "./skill-tables.js";
|
|
9
|
+
import { cliVersion } from "./version-check.js";
|
|
10
|
+
const indent = (text) => text
|
|
11
|
+
.split("\n")
|
|
12
|
+
.map((line) => ` ${line}`)
|
|
13
|
+
.join("\n");
|
|
8
14
|
export const BERYL_TEST_SKILL_FILENAME = "SKILL.md";
|
|
9
15
|
export const BERYL_TEST_SKILL_DIR = "beryl-test";
|
|
10
16
|
// The example plan the skill shows verbatim. Exported so the test suite lints it with
|
|
@@ -48,23 +54,21 @@ description: Author durable, healable end-to-end tests for a web app with Beryl.
|
|
|
48
54
|
|
|
49
55
|
# Authoring Beryl tests
|
|
50
56
|
|
|
51
|
-
|
|
52
|
-
asserts the flow reached its meaningful outcome. \`beryl init\` has already wired two MCP
|
|
53
|
-
servers for you — **beryl** (create/list/run tests) and **playwright** (drive a real
|
|
54
|
-
browser). Your job is to author tests that keep passing as the app's markup drifts,
|
|
55
|
-
because Beryl can **heal** them — but only when you give it what it needs to.
|
|
57
|
+
beryl-test v${cliVersion()} — installed by \`beryl init\`; the \`guide\` tool always returns the current version of this guide.
|
|
56
58
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
+
A Beryl test drives a real browser through a flow and asserts the flow reached its
|
|
60
|
+
meaningful outcome. \`beryl init\` wired two MCP servers: **beryl** (create/list/run
|
|
61
|
+
tests) and **playwright** (drive a real browser). Beryl **heals** tests as the app's
|
|
62
|
+
markup drifts — but only when you supply the three things that make a test durable: a
|
|
63
|
+
real **outcome assertion**, a strong **natural-language intent**, and the **local
|
|
64
|
+
run-fix loop**.
|
|
59
65
|
|
|
60
66
|
## 0. Start here
|
|
61
67
|
|
|
62
|
-
**Everything happens on your machine first
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
server-side instead (create tells you when), and \`--no-verify\` banks unproven — avoid
|
|
67
|
-
it. So a broken local browser is not a detail you can skip past; it is the whole loop.
|
|
68
|
+
**Everything happens on your machine first** — you drive the flow here, and
|
|
69
|
+
\`tests create\` proves the plan here too, so a working local browser IS the loop.
|
|
70
|
+
Avoid \`--no-verify\` (banks unproven); replay mechanics and the cloud-session
|
|
71
|
+
exception are in \`tests create\`'s own description.
|
|
68
72
|
|
|
69
73
|
Before the first plan:
|
|
70
74
|
|
|
@@ -78,10 +82,9 @@ beryl accounts check <id> # does that stored sign-in
|
|
|
78
82
|
beryl mailbox get # the address they receive mail at (§5)
|
|
79
83
|
\`\`\`
|
|
80
84
|
|
|
81
|
-
\`accounts list\` comes FIRST
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
account's login plan has to be stored before you can create the test at all.
|
|
85
|
+
\`accounts list\` comes FIRST: standing account vs new identity changes the plan you
|
|
86
|
+
write (discovering it afterwards means rewriting), and an authenticated test cannot be
|
|
87
|
+
created until its account's login plan is stored.
|
|
85
88
|
|
|
86
89
|
Then, per test:
|
|
87
90
|
|
|
@@ -89,9 +92,9 @@ Then, per test:
|
|
|
89
92
|
imagination (§2).
|
|
90
93
|
2. **Write the ActionPlan**, with one real outcome assertion (§2).
|
|
91
94
|
3. \`beryl tests lint --file plan.json\` — schema check, offline, no network.
|
|
92
|
-
4. \`beryl tests create --title … --file … --description "<the intent>"\` — replays the
|
|
93
|
-
locally
|
|
94
|
-
|
|
95
|
+
4. \`beryl tests create --title … --file … --description "<the intent>"\` — replays the
|
|
96
|
+
plan locally, banks it only on green; a red replay hands back the failure — fix and
|
|
97
|
+
re-run (§3 for the intent, §4 for the loop).
|
|
95
98
|
5. \`beryl runs local\` — re-run banked tests on your machine while iterating.
|
|
96
99
|
6. \`beryl runs trigger\` — hand it to Beryl's cloud, on demand or on a schedule.
|
|
97
100
|
|
|
@@ -107,40 +110,41 @@ admin qa+admin@x7k2p9.email.beryl.so beryl otp ready
|
|
|
107
110
|
\`\`\`
|
|
108
111
|
|
|
109
112
|
Author the sign-in as ordinary opening steps citing the reserved handles — fill
|
|
110
|
-
\`{{login_email}}\`, fill \`{{login_password}}\`, submit, assert the logged-in shell.
|
|
111
|
-
\`requires_auth: true\` AND \`auth_mode\` — both, always
|
|
112
|
-
a \`requires_auth\` plan without it
|
|
113
|
-
signs itself in like this
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
across environments.
|
|
113
|
+
\`{{login_email}}\`, fill \`{{login_password}}\`, submit, assert the logged-in shell.
|
|
114
|
+
Set \`requires_auth: true\` AND \`auth_mode\` — both, always; \`auth_mode\` has NO
|
|
115
|
+
default and create rejects a \`requires_auth\` plan without it. \`"inline"\` = the plan
|
|
116
|
+
signs itself in like this; \`"session"\` = no sign-in steps, rides the account's
|
|
117
|
+
once-per-run session (§ Session mode). \`auth_label\` picks a non-default identity
|
|
118
|
+
(the \`*\` row is the default). Handles resolve at run time to the named account, so
|
|
119
|
+
one plan stays correct across environments.
|
|
118
120
|
|
|
119
121
|
**The account needs a stored login plan before any authenticated test can be created.**
|
|
120
|
-
|
|
122
|
+
A run replays that plan to produce the session its tests ride, and replays it again
|
|
121
123
|
when the session expires — so \`tests create\` rejects a \`requires_auth\` plan whose
|
|
122
|
-
account has none,
|
|
123
|
-
\`beryl accounts set-login\` (§ Session mode)
|
|
124
|
+
account has none, naming the fix. Store it once per account with
|
|
125
|
+
\`beryl accounts set-login\` (§ Session mode).
|
|
124
126
|
|
|
125
|
-
**NEVER paste a real email or password into a plan.**
|
|
127
|
+
**NEVER paste a real email or password into a plan.** Handles resolve at run time —
|
|
126
128
|
the password never lands in the rendered spec and is scrubbed from artifacts. A pasted
|
|
127
|
-
value is baked
|
|
129
|
+
value is baked in forever and rots on the next rotation.
|
|
128
130
|
|
|
129
131
|
### Why this matters for coverage
|
|
130
132
|
|
|
131
|
-
The account persists between runs
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
data inside the test.
|
|
133
|
+
The account persists between runs and accumulates real data — that is the point. It
|
|
134
|
+
reaches what a fresh signup never could: a populated list, filters with something to
|
|
135
|
+
filter, run history, a dashboard with numbers. **A flow that needs pre-existing data is
|
|
136
|
+
the signal to use the test account**, not to build the data inside the test.
|
|
136
137
|
|
|
137
138
|
The trade is drift — run 200 has 200 of everything run 1 created. So:
|
|
138
139
|
|
|
139
140
|
- **Assert relatively, never absolutely.** "The row I just created is present"
|
|
140
|
-
(\`expect.persisted\` on a \`{{unique}}\`-named row), not "there are 3 rows"
|
|
141
|
-
assertion
|
|
142
|
-
- **
|
|
143
|
-
on pass AND
|
|
141
|
+
(\`expect.persisted\` on a \`{{unique}}\`-named row), not "there are 3 rows" — a count
|
|
142
|
+
assertion goes stale by itself.
|
|
143
|
+
- **Own the resource lifecycle, both directions.** Creating? Delete it in \`after\`
|
|
144
|
+
(runs on pass AND fail, unlike a trailing step inside \`steps\`). Deleting? Create
|
|
145
|
+
your own victim first — in \`before\` or the opening steps, \`{{unique}}\`-named —
|
|
146
|
+
then delete it and assert \`gone\`. Never aim a delete at standing data: run 2 would
|
|
147
|
+
have nothing to delete, and it eats the account's accumulated coverage.
|
|
144
148
|
|
|
145
149
|
### If there is no account yet
|
|
146
150
|
|
|
@@ -153,32 +157,26 @@ An empty \`accounts list\` means nothing is set up. In order of preference:
|
|
|
153
157
|
\`beryl accounts create --type beryl\`, then prove it with
|
|
154
158
|
\`beryl accounts provision <id> --file <plan.json>\`. The plan is any ActionPlan that
|
|
155
159
|
ends LOGGED IN — a signup filling \`{{mailbox_address}}\` and \`{{login_password}}\`
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
+
for a new account, or a **sign-in** for one that already exists (hand-made, or the
|
|
161
|
+
record was lost); both prove the same thing. No password sign-in? Pass
|
|
162
|
+
\`--login-method otp\` (or \`magic_link\`) and let the plan \`await_email\` through —
|
|
163
|
+
later sign-ins read the same mailbox.
|
|
160
164
|
3. **Neither** → the flow is not testable authenticated. Say so rather than guessing.
|
|
161
165
|
|
|
162
|
-
A project with no accounts
|
|
163
|
-
\`LOGIN_PASSWORD\` secret
|
|
166
|
+
A project with no accounts falls back to the \`LOGIN_EMAIL\` variable +
|
|
167
|
+
\`LOGIN_PASSWORD\` secret with the same handle resolution, so existing tests are
|
|
164
168
|
unaffected — but new work should create an account.
|
|
165
169
|
|
|
166
170
|
### Session mode: sign in once per run, not once per test
|
|
167
171
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
the
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
flow it actually tests begins — no login preamble in the plan, in the replay, or in the
|
|
177
|
-
failure evidence. Nothing has to be turned on for it: every run signs in its accounts and
|
|
178
|
-
hands out the sessions. What a session-mode test DOES need is a stored login plan on its
|
|
179
|
-
account — with none, there is nothing to sign in with, and the test fails at setup with
|
|
180
|
-
\`SESSION_NO_LOGIN_PLAN\` rather than falling back, because it has no sign-in steps of its
|
|
181
|
-
own to fall back to.
|
|
172
|
+
\`"inline"\` re-types the plan's own sign-in steps every run; always available.
|
|
173
|
+
|
|
174
|
+
\`"session"\` moves the sign-in out of the test: the account signs in ONCE at the start
|
|
175
|
+
of the run, the session is proved live, and every \`requires_auth\` test rides it
|
|
176
|
+
(inline ones just start already signed in). No login preamble in the plan, the replay,
|
|
177
|
+
or the failure evidence; nothing to turn on. A session-mode test has no sign-in steps
|
|
178
|
+
to fall back to, so an account with no stored login plan fails at setup with
|
|
179
|
+
\`SESSION_NO_LOGIN_PLAN\`.
|
|
182
180
|
|
|
183
181
|
Set it up once per account:
|
|
184
182
|
|
|
@@ -187,82 +185,89 @@ beryl accounts set-login <id> --file signin.json --probe probe.json
|
|
|
187
185
|
beryl accounts check <id> # signs in NOW and proves it — do not skip this
|
|
188
186
|
\`\`\`
|
|
189
187
|
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
- **\`--probe\`** is two steps: goto a gated page, then a **positive** assertion that only
|
|
194
|
-
holds when signed in — \`expect.visible\` on the account menu or a "Sign out" control.
|
|
195
|
-
It is required. \`hidden\`, \`count 0\` and a URL match on a redirect ALL pass against a
|
|
196
|
-
logged-out page, so without a positive signal a dead session runs every test logged-out
|
|
197
|
-
and the run still reports green.
|
|
188
|
+
What \`--file\` and \`--probe\` must contain, and the \`--base-hash\` read-before-write
|
|
189
|
+
rule for later edits, are in \`accounts set-login\` / \`accounts get-login\`'s own
|
|
190
|
+
descriptions.
|
|
198
191
|
|
|
199
|
-
Then write the tests with \`requires_auth: true
|
|
192
|
+
Then write the tests with \`requires_auth: true\`, \`auth_mode: "session"\`, and NO
|
|
200
193
|
sign-in steps.
|
|
201
194
|
|
|
202
|
-
**Creates share the sign-in.** \`tests create\` replays
|
|
203
|
-
account's established session
|
|
204
|
-
|
|
205
|
-
|
|
195
|
+
**Creates share the sign-in.** \`tests create\` replays session-mode plans against the
|
|
196
|
+
account's established session (reused while it stays live), so a batch of creates costs
|
|
197
|
+
at most one sign-in — but still create one at a time: racing creates race the same
|
|
198
|
+
mailbox for mail.
|
|
206
199
|
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
200
|
+
An expired session is never your problem: every run proves the stored session and, on a
|
|
201
|
+
miss, signs in again from the login plan. A "reconnect" message only appears when there
|
|
202
|
+
is no stored login plan to refresh from.
|
|
210
203
|
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
What to expect when it does not work: \`accounts check\` returning
|
|
217
|
-
\`SESSION_LOGIN_FAILED\` means the sign-in plan itself is wrong — fix it.
|
|
218
|
-
\`SESSION_PROOF_FAILED\` means signing in worked but the session could not be carried into
|
|
219
|
-
a fresh browser, because this app keeps its credential somewhere unextractable. That is not
|
|
220
|
-
your bug, but it IS your move: the account is marked unsupported, and its session-mode
|
|
221
|
-
tests fail at setup with \`SESSION_UNSUPPORTED\` on every run until you re-author them
|
|
222
|
-
with \`auth_mode: "inline"\` and their own sign-in steps. Inline tests are unaffected.
|
|
223
|
-
|
|
224
|
-
\`beryl runs local\` works on session-mode tests too — same shape as the cloud: one
|
|
225
|
-
sign-in per invocation, shared by every session-mode test.
|
|
204
|
+
When \`accounts check\` fails, its description explains which half broke and each
|
|
205
|
+
\`SESSION_*\` reason. The one that changes your authoring: an account marked
|
|
206
|
+
unsupported (its session cannot survive a fresh browser) is not your bug, but it IS
|
|
207
|
+
your move — re-author its session-mode tests with \`auth_mode: "inline"\` and their own
|
|
208
|
+
sign-in steps.
|
|
226
209
|
|
|
227
210
|
### Two identities in one test
|
|
228
211
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
"the share link is issued") — a real, strong outcome. While AUTHORING you can
|
|
232
|
-
full handshake live:
|
|
233
|
-
second mailbox (\`mailbox create --label invitee
|
|
212
|
+
Invite-a-teammate-and-accept-as-them needs two identities mid-flow — not bankable as
|
|
213
|
+
one test. Bank the half the app shows to account A ("the invitation is listed as
|
|
214
|
+
pending", "the share link is issued") — a real, strong outcome. While AUTHORING you can
|
|
215
|
+
drive the full handshake live: a second account (\`accounts create --label member\`), or
|
|
216
|
+
a second mailbox (\`mailbox create --label invitee\`, read with \`mailbox read\`).
|
|
217
|
+
|
|
218
|
+
SSO-only sites (no email+password form at all) ride a captured session instead: a human
|
|
219
|
+
signs in once through a live browser view in the Beryl webapp, Beryl stores the session
|
|
220
|
+
encrypted server-side and injects it into cloud runs — \`tests create\` verifies such
|
|
221
|
+
plans server-side, \`runs trigger\` runs them, \`runs local\` skips them (the session
|
|
222
|
+
never leaves Beryl's cloud).
|
|
234
223
|
|
|
235
|
-
SSO-only sites (no email+password form at all) remain webapp territory.
|
|
236
224
|
## 2. Author locally over the Playwright MCP
|
|
237
225
|
|
|
238
|
-
1. **Drive the flow in a real browser first.**
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
selector. Watch what actually happens; don't author from imagination.
|
|
226
|
+
1. **Drive the flow in a real browser first.** Walk it by hand over the Playwright MCP
|
|
227
|
+
— log in, fill, submit — acting on elements by their accessibility ref from the
|
|
228
|
+
latest page snapshot, never a guessed selector. Watch what actually happens.
|
|
242
229
|
2. **Write it as an ActionPlan** — a JSON object whose \`steps\` are
|
|
243
|
-
\`{action, selector, url, value, ...}\`. Two structural rules
|
|
244
|
-
- the **first executed step is a \`goto
|
|
230
|
+
\`{action, selector, url, value, ...}\`. Two structural rules:
|
|
231
|
+
- the **first executed step is a \`goto\`**, and
|
|
245
232
|
- **at least one step is an \`expect\`** (a test that asserts nothing is not a test).
|
|
246
233
|
**A \`goto\` at your own app is a PATH, never a full URL** — \`/pricing\`, not
|
|
247
|
-
\`https://app.example.com/pricing\`. The origin comes from the environment's root
|
|
248
|
-
one plan runs against prod, staging and a preview
|
|
249
|
-
silently ignored
|
|
250
|
-
|
|
251
|
-
Optional \`before\` / \`after\` arrays hold setup and teardown; \`after\` runs even
|
|
252
|
-
main step fails, so a create
|
|
234
|
+
\`https://app.example.com/pricing\`. The origin comes from the environment's root
|
|
235
|
+
URL, so one plan runs against prod, staging and a preview; bake the origin in and
|
|
236
|
+
\`--env\` is silently ignored. Absolute URLs stay legal for OTHER origins (an OAuth
|
|
237
|
+
handoff, a magic link on another domain).
|
|
238
|
+
Optional \`before\` / \`after\` arrays hold setup and teardown; \`after\` runs even
|
|
239
|
+
when a main step fails, so a create flow can clean up its record — and a deletion
|
|
240
|
+
flow creates its own record in \`before\` first.
|
|
241
|
+
|
|
242
|
+
The full action vocabulary:
|
|
243
|
+
|
|
244
|
+
${indent(actionTable())}
|
|
245
|
+
|
|
246
|
+
The routinely mis-written ones — exact semantics:
|
|
247
|
+
- \`upload\`'s \`value\` names a file in the project's CONFIG-FILE store
|
|
248
|
+
(\`config files upload\`), never a filesystem path.
|
|
249
|
+
- \`dialog\` arms a one-shot accept/dismiss handler for the dialog the NEXT step
|
|
250
|
+
triggers (armed before the click, or Playwright auto-dismisses it). No selector;
|
|
251
|
+
optional \`dialog_expect_text\` asserts the dialog's message.
|
|
252
|
+
- \`switch_tab\` / \`close_tab\`: \`value\` is a 0-based tab index OR a URL substring.
|
|
253
|
+
- \`scroll\` with no selector scrolls the PAGE — \`value\` is \`bottom\`, \`top\`,
|
|
254
|
+
\`up\`, \`down\`, or a signed pixel count; default one viewport down. With a
|
|
255
|
+
selector it scrolls that element into view.
|
|
256
|
+
|
|
257
|
+
Any element action can reach inside an iframe (Stripe/Adyen card fields) via the
|
|
258
|
+
\`iframe\` field — a URL-substring match on the frame's \`src\`; ignored by
|
|
259
|
+
page-level verbs.
|
|
253
260
|
|
|
254
261
|
An \`expect\` step's shape is \`{action: "expect", expect_kind, selector, expect_text}\`:
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
- \`
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
\`count_delta\` needs \`capture_ref\` + \`expect_delta\` (vs a baseline banked by an
|
|
265
|
-
earlier \`capture_count\` step).
|
|
262
|
+
|
|
263
|
+
${indent(expectTable())}
|
|
264
|
+
|
|
265
|
+
- \`expect_text\` is an exact match on the element for \`have_text\` / \`have_value\`,
|
|
266
|
+
a substring match on the page for \`have_url\` / \`have_title\` (page-level kinds
|
|
267
|
+
take no selector). There is no \`value\` field on an expect and no bare \`text\` /
|
|
268
|
+
\`url\` kind.
|
|
269
|
+
- \`have_count\` needs \`expect_count\`; \`count_delta\` needs \`capture_ref\` +
|
|
270
|
+
\`expect_delta\` (vs a baseline banked by an earlier \`capture_count\` step).
|
|
266
271
|
|
|
267
272
|
A fully valid minimal plan ("the pricing page renders"):
|
|
268
273
|
|
|
@@ -278,89 +283,79 @@ ${JSON.stringify(BERYL_TEST_SKILL_EXAMPLE_PLAN, null, 2)
|
|
|
278
283
|
beryl tests create --title "Log in" --file plan.json \\
|
|
279
284
|
--description "<the intent — see §3>"
|
|
280
285
|
\`\`\`
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
replay is imported as the test's first run (\`--no-sync\` to skip); \`--no-verify\` banks
|
|
287
|
-
unproven. Inspect the exact spec that would run with \`beryl tests script --file
|
|
288
|
-
plan.json\`. The full ActionPlan JSON Schema is at
|
|
286
|
+
\`create\` banks the plan only on a green local replay; a red one hands back the
|
|
287
|
+
failure — fix and re-run. Replay mechanics (where it runs over MCP, \`--no-sync\`,
|
|
288
|
+
\`--no-verify\`, the cloud-session exception) are in \`tests create\`'s own
|
|
289
|
+
description. Inspect the exact spec with \`beryl tests script --file plan.json\`.
|
|
290
|
+
Full ActionPlan JSON Schema:
|
|
289
291
|
https://api.beryl.so/api/v1/schemas/action-plan.schema.json.
|
|
290
292
|
|
|
291
293
|
### The outcome assertion is the whole game
|
|
292
294
|
|
|
293
295
|
A flow is only worth banking if you can point at the **success signal** — the one
|
|
294
|
-
observable proof the flow worked.
|
|
295
|
-
|
|
296
|
-
- The signal must be **true only if the flow succeeded
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
destination's own distinctive content.
|
|
296
|
+
observable proof the flow worked.
|
|
297
|
+
|
|
298
|
+
- The signal must be **true only if the flow succeeded**: a confirmation that appeared,
|
|
299
|
+
an element that showed up or disappeared, content unique to where the flow landed.
|
|
300
|
+
- **Never assert global chrome** — nav bar, logo, footer, cookie banner are on every
|
|
301
|
+
page, so asserting them tests nothing. "Was there anyway" means site-wide chrome, NOT
|
|
302
|
+
the destination's own distinctive content.
|
|
302
303
|
- For a **navigation** flow, the strongest signal is that the destination actually
|
|
303
|
-
**rendered**:
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
element
|
|
310
|
-
disappeared — e.g. a spinner, or the item you just deleted). §2 has the full
|
|
311
|
-
\`expect_kind\` list and the step shape.
|
|
304
|
+
**rendered**: its unique heading or page-specific content (for \`/pricing\`, the
|
|
305
|
+
"Pricing" H1 or a plan name). A bare "the URL is /pricing" passes even on a blank,
|
|
306
|
+
broken page — reserve URL-only assertions for when the URL *is* the outcome (a form
|
|
307
|
+
landing on \`/thank-you\`) and no distinctive content exists.
|
|
308
|
+
- The usual outcome kinds: \`visible\` (the success element showed up), \`have_text\`
|
|
309
|
+
(an element's text matches), \`have_url\` (the URL contains a value), \`gone\` (an
|
|
310
|
+
element disappeared — a spinner, the item you just deleted).
|
|
312
311
|
- **\`have_text\` is an EXACT full-text match on the selector's element** — asserting
|
|
313
|
-
\`have_text: "Documentation"\` on \`body\` fails
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
- **If you can't name a success signal, the flow is not test-worthy.** Don't bank a
|
|
318
|
-
that verifies nothing
|
|
319
|
-
- **Don't work around a real app failure to make a test go green.**
|
|
320
|
-
|
|
321
|
-
with a weaker assertion.
|
|
312
|
+
\`have_text: "Documentation"\` on \`body\` fails because \`body\` includes all the nav
|
|
313
|
+
chrome. Target the element that carries the text (the \`h1\`, the toast), assert the
|
|
314
|
+
page with \`have_title\` (substring), or check "this string is visible somewhere"
|
|
315
|
+
with \`expect_kind: "visible"\` + a \`text=…\` selector.
|
|
316
|
+
- **If you can't name a success signal, the flow is not test-worthy.** Don't bank a
|
|
317
|
+
test that verifies nothing; explore a different flow.
|
|
318
|
+
- **Don't work around a real app failure to make a test go green.** A genuinely broken
|
|
319
|
+
flow is a finding to report, not something to paper over with a weaker assertion.
|
|
322
320
|
|
|
323
321
|
### Traps when authoring against a real app
|
|
324
322
|
|
|
325
323
|
Every one of these has produced a wrong plan or a red \`tests create\`. Check them.
|
|
326
324
|
|
|
327
325
|
1. **Your browser may already be signed in.** The Playwright MCP keeps a persistent
|
|
328
|
-
profile
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
2. **The sign-UP flow is not the sign-IN flow.** A brand-new address often gets an
|
|
332
|
-
"create your account" step
|
|
333
|
-
|
|
334
|
-
it twice: once to create the account, once to see signing in again.
|
|
326
|
+
profile. Author while signed in and you never see the gate — you will mark gated
|
|
327
|
+
pages as public. **Log out first**, then confirm the gated page really redirects to
|
|
328
|
+
the login.
|
|
329
|
+
2. **The sign-UP flow is not the sign-IN flow.** A brand-new address often gets an
|
|
330
|
+
extra "create your account" step a returning address skips. A stored login plan must
|
|
331
|
+
be the **returning** path — drive it twice: once to create, once to sign in again.
|
|
335
332
|
3. **Read the DOM, not just the accessibility tree, before picking a selector.** Two
|
|
336
|
-
buttons can share a
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
\`input[autocomplete=one-time-code]\`)
|
|
340
|
-
4. **A single-page app can redirect after the first \`goto\`.** If \`/dashboard\`
|
|
341
|
-
redirects to \`/dashboard/<id>\`, a click fired straight after the goto
|
|
342
|
-
pre-redirect render and
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
permanent explainer, a control that is always there.
|
|
333
|
+
buttons can share a label ("Continue" / "Continue with Google"), and Beryl relaxes
|
|
334
|
+
\`text=X\` to a case-insensitive SUBSTRING — \`text=Continue\` is ambiguous. Use
|
|
335
|
+
something unique (\`button[type=submit]\`, \`input[name=email]\`,
|
|
336
|
+
\`input[autocomplete=one-time-code]\`).
|
|
337
|
+
4. **A single-page app can redirect after the first \`goto\`.** If \`/dashboard\`
|
|
338
|
+
client-side redirects to \`/dashboard/<id>\`, a click fired straight after the goto
|
|
339
|
+
lands on the pre-redirect render and is discarded on re-render. Put a \`wait_for\`
|
|
340
|
+
on something that exists only AFTER the redirect, then act.
|
|
341
|
+
5. **Never bake an id into a URL.** \`goto /projects/8ab46d63-.../settings\` breaks for
|
|
342
|
+
any other account. Navigate to the stable entry point and click through
|
|
343
|
+
(\`a[href$='/settings']\`) — the plan is about the app, not your row.
|
|
344
|
+
6. **Assert durable content, not the empty state.** "No tests yet" is true today and
|
|
345
|
+
false the moment anything exists. Prefer what is structural — a section heading, a
|
|
346
|
+
permanent explainer, an always-present control.
|
|
351
347
|
|
|
352
348
|
## 3. Writing the natural-language intent
|
|
353
349
|
|
|
354
|
-
Pass the intent as \`--description\` on \`beryl tests create\` (or \`tests set-plan\`
|
|
355
|
-
re-author). 1–3 sentences. This is the immutable anchor from §6
|
|
350
|
+
Pass the intent as \`--description\` on \`beryl tests create\` (or \`tests set-plan\`
|
|
351
|
+
when you re-author). 1–3 sentences. This is the immutable anchor from §6.
|
|
356
352
|
|
|
357
353
|
- **State the purpose, not the steps.** Not "clicks Sign in, types email and password,
|
|
358
|
-
clicks submit" —
|
|
359
|
-
|
|
360
|
-
- **Name the one observable outcome** that is true only if the flow worked —
|
|
361
|
-
success signal
|
|
362
|
-
- **Never describe global chrome.**
|
|
363
|
-
outcome, not "the header is present".
|
|
354
|
+
clicks submit" — the trajectory will change. Instead: *what does a green run prove is
|
|
355
|
+
true about the app?*
|
|
356
|
+
- **Name the one observable outcome** that is true only if the flow worked — §2's
|
|
357
|
+
success signal, in words.
|
|
358
|
+
- **Never describe global chrome.**
|
|
364
359
|
|
|
365
360
|
Good:
|
|
366
361
|
> "Proves a returning user can sign in: after submitting valid credentials, the
|
|
@@ -371,79 +366,63 @@ Weak (describes steps + asserts nothing meaningful):
|
|
|
371
366
|
|
|
372
367
|
## 4. The local run-fix loop
|
|
373
368
|
|
|
374
|
-
Iterate on your machine before you rely on the cloud. \`beryl runs local\`
|
|
375
|
-
|
|
376
|
-
for a scheduled run.
|
|
369
|
+
Iterate on your machine before you rely on the cloud. \`beryl runs local\` runs banked
|
|
370
|
+
tests with your local \`@playwright/test\` — no cloud, no waiting for a scheduled run.
|
|
377
371
|
|
|
378
372
|
\`\`\`
|
|
379
|
-
npm i -D @playwright/test && npx playwright install chromium # once
|
|
380
373
|
beryl runs local <test-id> --no-sync --url-override http://localhost:3000 --dir ./beryl-local
|
|
381
374
|
beryl runs local # the whole suite, results recorded in Beryl
|
|
382
375
|
\`\`\`
|
|
383
376
|
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
or assertion failed and why, fix the plan, \`beryl tests set-plan\`, run again.
|
|
393
|
-
- It exits **0** if every test passed, **1** on a failure — so it drops straight into a
|
|
394
|
-
run-fix-run loop.
|
|
395
|
-
- \`await_email\` steps work locally: the CLI answers them over the API against the same
|
|
396
|
-
mailbox the cloud runner would use, exactly as it would.
|
|
397
|
-
- **Authenticated tests work locally.** A plan that signs itself in by filling
|
|
398
|
-
\`{{login_email}}\` / \`{{login_password}}\` runs fine: the email is baked into the fetched
|
|
399
|
-
spec and the password is revealed once over the logged secret-reveal route, then
|
|
400
|
-
scrubbed from any uploaded error text or DOM snapshot. If the test account it names has
|
|
401
|
-
no password stored, the test is skipped with the exact fix-it command.
|
|
402
|
-
- A test that depends on a session Beryl holds server-side, rather than signing itself in,
|
|
403
|
-
is skipped locally with a note — run those with \`beryl runs trigger\`.
|
|
404
|
-
|
|
405
|
-
Once the test passes locally against a real outcome, it's ready to bank and let Beryl run
|
|
406
|
-
and heal it.
|
|
377
|
+
The loop: draft → \`tests lint\` → \`tests create\` → \`runs local --no-sync\` while
|
|
378
|
+
iterating (a synced run lands in the project's run history) → read \`--dir\`'s
|
|
379
|
+
\`report.json\` to see which step or assertion failed and why → fix the plan →
|
|
380
|
+
\`tests set-plan\` → run again. Flag semantics, session-mode behavior, \`await_email\`
|
|
381
|
+
and authenticated-test handling, and exit codes are in \`runs local\`'s own description.
|
|
382
|
+
|
|
383
|
+
Once the test passes locally against a real outcome, it's ready to bank and let Beryl
|
|
384
|
+
run and heal it.
|
|
407
385
|
|
|
408
386
|
## 5. Testing an OTP / signup flow (\`await_email\`)
|
|
409
387
|
|
|
410
|
-
A flow that emails the user — a signup verification code, a magic sign-in link, a
|
|
411
|
-
— is testable with the \`await_email\` action. No setup, no
|
|
388
|
+
A flow that emails the user — a signup verification code, a magic sign-in link, a
|
|
389
|
+
receipt — is testable with the \`await_email\` action. No setup, no configuration, no
|
|
412
390
|
flag to turn on.
|
|
413
391
|
|
|
414
|
-
**The project has one permanent mailbox and all of its mail arrives there.** Two
|
|
415
|
-
put an address on the page
|
|
392
|
+
**The project has one permanent mailbox and all of its mail arrives there.** Two
|
|
393
|
+
handles put an address on the page; the one you cite decides which *identity* the test
|
|
394
|
+
acts as:
|
|
416
395
|
|
|
417
396
|
| Handle | Renders as | Use it for |
|
|
418
397
|
|---|---|---|
|
|
419
398
|
| \`{{mailbox_address}}\` | the mailbox's own address, the same every run | signing in as the project's standing test account (§1) |
|
|
420
399
|
| \`{{inbox_address}}\` | a \`+tag\` alias of it, fresh every run | tests whose subject IS getting a NEW identity — a signup, an invited teammate |
|
|
421
400
|
|
|
422
|
-
An alias is a real address the site has never issued, so a signup is repeatable run
|
|
423
|
-
run; the mail
|
|
424
|
-
Nothing expires
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
account would be rejected — a signup form, or an invite you must accept as a second person.
|
|
401
|
+
An alias is a real address the site has never issued, so a signup is repeatable run
|
|
402
|
+
after run; the mail lands in the same mailbox and Beryl reads only the alias's own
|
|
403
|
+
mail. Nothing expires; there is no second inbox to manage. Default to
|
|
404
|
+
\`{{mailbox_address}}\`; reach for \`{{inbox_address}}\` only when an existing account
|
|
405
|
+
would be rejected — a signup form, an invite accepted as a second person.
|
|
428
406
|
|
|
429
407
|
The wiring is a three-part chain:
|
|
430
408
|
|
|
431
|
-
1. **Type the address into the app** — a \`fill\` with \`value: "{{inbox_address}}"\`
|
|
432
|
-
\`{{mailbox_address}}\`).
|
|
433
|
-
construction
|
|
434
|
-
must-not-collide values like a username
|
|
409
|
+
1. **Type the address into the app** — a \`fill\` with \`value: "{{inbox_address}}"\`
|
|
410
|
+
(or \`{{mailbox_address}}\`). The alias is fresh every run, so a signup is
|
|
411
|
+
repeatable by construction — no \`{{unique}}\` needed for the email itself; use it
|
|
412
|
+
for other must-not-collide values like a username. The generator handles
|
|
413
|
+
\`{{unique}}\`, \`{{uuid}}\`, \`{{timestamp}}\` are each minted once per execution;
|
|
414
|
+
\`{{timestamp}}\` is second-resolution (a time, NOT a uniqueness guarantee —
|
|
415
|
+
\`{{unique}}\` is).
|
|
435
416
|
2. **Await the mail and bank the extracted value** — an \`await_email\` step with:
|
|
436
|
-
- \`extract\` (required):
|
|
437
|
-
\`pattern\` (your own regex in \`extract_pattern\`, exactly one capture group).
|
|
417
|
+
- \`extract\` (required): ${extractLine()}.
|
|
438
418
|
- \`capture_as\` (required): the handle name the extracted string is banked under.
|
|
439
|
-
- \`subject_contains\` / \`from_contains\` (optional): match the right mail when the
|
|
440
|
-
sends more than one.
|
|
441
|
-
- \`wait_s\` (optional, 1–50, default 30):
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
literal text, and the linter rejects it there.
|
|
419
|
+
- \`subject_contains\` / \`from_contains\` (optional): match the right mail when the
|
|
420
|
+
app sends more than one.
|
|
421
|
+
- \`wait_s\` (optional, 1–50, default 30): seconds to block waiting for the mail.
|
|
422
|
+
3. **Use the banked value** — cite \`{{<capture_as>}}\` in a later step's \`value\`
|
|
423
|
+
(fill the code) or \`url\` (goto the magic link). A captured handle is legal
|
|
424
|
+
**only** in \`value\`/\`url\`; in a \`selector\`, \`option\`, or \`expect_text\` it
|
|
425
|
+
would be literal text, and the linter rejects it there.
|
|
447
426
|
|
|
448
427
|
A fully valid signup-with-OTP plan:
|
|
449
428
|
|
|
@@ -457,34 +436,32 @@ For a magic-link flow, replace the code steps with
|
|
|
457
436
|
|
|
458
437
|
Two caveats:
|
|
459
438
|
|
|
460
|
-
- \`beryl tests create\` verifies an \`await_email\` plan like any other
|
|
461
|
-
replay receives at the project mailbox and answers each step over the API, so the
|
|
462
|
-
mail really is received and extracted before the test is accepted.
|
|
463
|
-
signs up / sends mail
|
|
464
|
-
unwanted.
|
|
465
|
-
|
|
466
|
-
-
|
|
467
|
-
|
|
439
|
+
- \`beryl tests create\` verifies an \`await_email\` plan like any other: the local
|
|
440
|
+
replay receives at the project mailbox and answers each step over the API, so the
|
|
441
|
+
app's mail really is received and extracted before the test is accepted. The replay
|
|
442
|
+
signs up / sends mail FOR REAL — pass \`--no-verify\` only if that side effect is
|
|
443
|
+
unwanted. \`beryl runs local\` serves \`await_email\` the same way, so the local loop
|
|
444
|
+
covers OTP/signup flows end to end.
|
|
445
|
+
- §2's outcome discipline still applies: the green signal is the post-verification
|
|
446
|
+
state (the welcome screen, the dashboard), not "an email arrived".
|
|
468
447
|
|
|
469
448
|
## 6. Why this shape: durable and healable
|
|
470
449
|
|
|
471
|
-
Beryl's cloud runs your test on a schedule. When
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
- **The
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
can't tell a real regression from cosmetic drift, so it either heals over real breakage or
|
|
489
|
-
fails on noise.
|
|
450
|
+
Beryl's cloud runs your test on a schedule. When markup drifts and a selector stops
|
|
451
|
+
matching, a heal-vs-fail agent decides whether to **heal** (silently re-derive the
|
|
452
|
+
selector/trajectory, stay green) or **fail** (surface a real regression) — judged
|
|
453
|
+
against your test's **intent**:
|
|
454
|
+
|
|
455
|
+
- **The intent is the immutable anchor; Beryl never rewrites it.** It states what the
|
|
456
|
+
test proves — what every future run is judged against.
|
|
457
|
+
- **Selectors and the trajectory are the healable "how"** — a moved button, a renamed
|
|
458
|
+
class, an extra click. Beryl re-derives those because your intent says what the flow
|
|
459
|
+
is *for*.
|
|
460
|
+
- **A failed outcome assertion is a real regression Beryl will NOT heal green.** If the
|
|
461
|
+
success signal stops holding, the app broke, and the test fails loudly. That is the
|
|
462
|
+
point.
|
|
463
|
+
|
|
464
|
+
So a test is *healable* exactly when it has **a strong intent + a real outcome
|
|
465
|
+
assertion**. With a vague intent and a chrome-only assertion Beryl can't tell
|
|
466
|
+
regression from cosmetic drift — it either heals over real breakage or fails on noise.
|
|
490
467
|
`;
|
|
@@ -172,9 +172,10 @@ export const testAccountCommands = [
|
|
|
172
172
|
"--probe is the liveness check: a two-step plan (goto a gated page, then a POSITIVE " +
|
|
173
173
|
"assertion that only holds when signed in — the account menu, a 'Sign out' control). " +
|
|
174
174
|
"It is replayed in a fresh browser carrying only the captured session. It is " +
|
|
175
|
-
"REQUIRED, and not a formality: assertions like `hidden
|
|
176
|
-
"against a logged-out page, so without a positive
|
|
177
|
-
"every test logged-out and still report the run
|
|
175
|
+
"REQUIRED, and not a formality: assertions like `hidden`, `count 0`, and a URL " +
|
|
176
|
+
"match on a redirect all pass against a logged-out page, so without a positive " +
|
|
177
|
+
"signal a dead session would run every test logged-out and still report the run " +
|
|
178
|
+
"green.\n\n" +
|
|
178
179
|
"Storing only stores. Run `beryl accounts check` to prove it against the live app.",
|
|
179
180
|
scope: "project",
|
|
180
181
|
args: [
|
|
@@ -264,8 +265,10 @@ export const testAccountCommands = [
|
|
|
264
265
|
"itself did not complete (fix the plan); SESSION_PROOF_FAILED means the sign-in " +
|
|
265
266
|
"worked but the session did not survive the move to a fresh browser, so this app " +
|
|
266
267
|
"keeps its credential somewhere that cannot be carried (a service worker, a " +
|
|
267
|
-
"WebAuthn binding). In that case the account is marked unsupported
|
|
268
|
-
"
|
|
268
|
+
"WebAuthn binding). In that case the account is marked unsupported: its " +
|
|
269
|
+
"session-mode tests fail at setup with SESSION_UNSUPPORTED on every run until " +
|
|
270
|
+
"re-authored with auth_mode \"inline\" and their own sign-in steps; inline tests " +
|
|
271
|
+
"are unaffected.\n\n" +
|
|
269
272
|
"Blocks for two browser replays.",
|
|
270
273
|
scope: "project",
|
|
271
274
|
args: [{ name: "account-id", description: "Account id", required: true }],
|
|
@@ -57,6 +57,9 @@ export const configCommands = [
|
|
|
57
57
|
{
|
|
58
58
|
name: "config vars get",
|
|
59
59
|
summary: "Show one config variable",
|
|
60
|
+
description: "Returns the variable row with its value in plaintext (variables are not secret) — " +
|
|
61
|
+
"a sensitive value lives in `config secrets`, readable only via `config secrets " +
|
|
62
|
+
"get --reveal`.",
|
|
60
63
|
scope: "project",
|
|
61
64
|
args: [{ name: "key", description: "Variable key or id", required: true }],
|
|
62
65
|
async run(ctx, input) {
|
|
@@ -113,6 +116,8 @@ export const configCommands = [
|
|
|
113
116
|
{
|
|
114
117
|
name: "config secrets get",
|
|
115
118
|
summary: "Show one secret's metadata, or reveal its value with --reveal",
|
|
119
|
+
description: "Returns metadata only by default; --reveal is a logged, member-gated decrypt — " +
|
|
120
|
+
"unlike `config vars get`, which returns its value in plaintext.",
|
|
116
121
|
scope: "project",
|
|
117
122
|
args: [{ name: "key", description: "Secret key or id", required: true }],
|
|
118
123
|
flags: [
|
package/dist/commands/init.js
CHANGED
|
@@ -285,8 +285,8 @@ export const initCommands = [
|
|
|
285
285
|
description: "The full guide to authoring durable, healable tests: the ActionPlan shape, outcome " +
|
|
286
286
|
"assertions, natural-language intent, test accounts and the project mailbox " +
|
|
287
287
|
"({{login_email}}, {{mailbox_address}}, {{inbox_address}} + await_email), and the " +
|
|
288
|
-
"local run-fix loop.
|
|
289
|
-
|
|
288
|
+
"local run-fix loop. Same content as the beryl-test skill `beryl init` installs — " +
|
|
289
|
+
`skip if a loaded beryl-test skill states v${cliVersion()}; else call this first ` +
|
|
290
290
|
"(works without logging in).",
|
|
291
291
|
examples: ["beryl guide"],
|
|
292
292
|
async run() {
|
|
@@ -91,7 +91,9 @@ export const mailboxCommands = [
|
|
|
91
91
|
"the server caps the wait at 50s — re-run to keep waiting). With --extract-code, " +
|
|
92
92
|
"also pulls the one-time code (4-8 digits) out of the body/subject. Use " +
|
|
93
93
|
"--recipient-contains to read only one `+tag` alias's mail when several identities " +
|
|
94
|
-
"share the mailbox. Exits non-zero if nothing arrives before the timeout."
|
|
94
|
+
"share the mailbox. Exits non-zero if nothing arrives before the timeout. Waits for " +
|
|
95
|
+
"and returns ONE latest matching email — `mailbox emails` lists what has already " +
|
|
96
|
+
"arrived, without waiting.",
|
|
95
97
|
scope: "project",
|
|
96
98
|
args: [{ name: "mailbox-id", description: "Mailbox id from `beryl mailbox list`", required: true }],
|
|
97
99
|
flags: [
|
|
@@ -139,6 +141,8 @@ export const mailboxCommands = [
|
|
|
139
141
|
{
|
|
140
142
|
name: "mailbox emails",
|
|
141
143
|
summary: "List the emails a mailbox has received",
|
|
144
|
+
description: "Returns the already-received emails without waiting — `mailbox read` blocks for " +
|
|
145
|
+
"a matching one and returns just it.",
|
|
142
146
|
scope: "project",
|
|
143
147
|
args: [{ name: "mailbox-id", description: "Mailbox id from `beryl mailbox list`", required: true }],
|
|
144
148
|
flags: [
|
package/dist/commands/runs.js
CHANGED
|
@@ -102,17 +102,19 @@ export const runCommands = [
|
|
|
102
102
|
"says so once instead of failing every test (on a terminal the CLI offers to install " +
|
|
103
103
|
"whichever half is missing; over MCP it prints the exact install commands). " +
|
|
104
104
|
"Signup/OTP flows work: the CLI answers the spec's await_email steps over the API " +
|
|
105
|
-
"against the same mailbox the cloud runner uses. Authenticated tests work too: a plan " +
|
|
106
|
-
"that signs itself in with {{login_email}}/{{login_password}}
|
|
105
|
+
"against the same mailbox the cloud runner uses. Authenticated tests work too: for a plan " +
|
|
106
|
+
"that signs itself in with {{login_email}}/{{login_password}}, the email is baked into " +
|
|
107
|
+
"the fetched spec and the password is fetched " +
|
|
107
108
|
"once over the logged secret-reveal route, handed to the spec the way the cloud runner " +
|
|
108
|
-
"does, and scrubbed from any error text or DOM snapshot before results upload
|
|
109
|
+
"does, and scrubbed from any error text or DOM snapshot before results upload; a test " +
|
|
110
|
+
"whose account has no stored password is skipped with the exact fix-it command. When the " +
|
|
109
111
|
"run finishes, the results and replay artifacts are imported into Beryl as a normal run " +
|
|
110
112
|
"(trigger source `local`) — history, replay, and reports all work; pass --no-sync to " +
|
|
111
113
|
"keep a run entirely off the record while iterating. Session-mode tests behave as " +
|
|
112
114
|
"in the cloud: their account signs in once per invocation and every session-mode " +
|
|
113
115
|
"test rides that session; a failed sign-in fails those tests with the same " +
|
|
114
116
|
"SESSION_* reason a cloud run reports. Only a test that depends on a session Beryl " +
|
|
115
|
-
"captured server-side is skipped, with a note
|
|
117
|
+
"captured server-side is skipped, with a note — run those with `runs trigger`. Point " +
|
|
116
118
|
"--url-override at a local dev server or preview, and --dir to keep specs, artifacts, " +
|
|
117
119
|
"and reports on disk. Exits 0 only if every executed test passed.",
|
|
118
120
|
scope: "project",
|
|
@@ -134,7 +136,12 @@ export const runCommands = [
|
|
|
134
136
|
type: "string",
|
|
135
137
|
description: "Run against this base URL instead of the environment's (e.g. http://localhost:3000)",
|
|
136
138
|
},
|
|
137
|
-
{
|
|
139
|
+
{
|
|
140
|
+
name: "env",
|
|
141
|
+
type: "string",
|
|
142
|
+
description: "Environment id to run against and attach the imported run to — use for a " +
|
|
143
|
+
"standing environment; --url-override is for a throwaway host",
|
|
144
|
+
},
|
|
138
145
|
{
|
|
139
146
|
name: "sync",
|
|
140
147
|
type: "boolean",
|
|
@@ -464,7 +471,9 @@ export const runCommands = [
|
|
|
464
471
|
summary: "Show one run with its per-test results",
|
|
465
472
|
description: "Over MCP the failure screenshots come back as viewable image content, so an agent can " +
|
|
466
473
|
"look at the page that broke instead of guessing from the error string. Set screenshots " +
|
|
467
|
-
"to false to skip fetching them. Ignored outside MCP — the terminal cannot show an image."
|
|
474
|
+
"to false to skip fetching them. Ignored outside MCP — the terminal cannot show an image. " +
|
|
475
|
+
"Returns the run row with its per-test results — `runs report` returns the generated " +
|
|
476
|
+
"report document, `runs explain` an AI explanation of one failed result.",
|
|
468
477
|
scope: "project",
|
|
469
478
|
args: [{ name: "run-id", description: "Run id", required: true }],
|
|
470
479
|
flags: [
|
|
@@ -516,6 +525,8 @@ export const runCommands = [
|
|
|
516
525
|
{
|
|
517
526
|
name: "runs report",
|
|
518
527
|
summary: "Show the generated report for a run",
|
|
528
|
+
description: "Returns the run's stored generated report (404 until it has been generated) — " +
|
|
529
|
+
"`runs get` returns the raw run row with per-test results.",
|
|
519
530
|
scope: "project",
|
|
520
531
|
args: [{ name: "run-id", description: "Run id", required: true }],
|
|
521
532
|
async run(ctx, input) {
|
|
@@ -595,6 +606,8 @@ export const runCommands = [
|
|
|
595
606
|
{
|
|
596
607
|
name: "runs explain",
|
|
597
608
|
summary: "Explain, with AI, why a test result failed",
|
|
609
|
+
description: "Takes a single test-RESULT id (not a run id) and returns an AI failure " +
|
|
610
|
+
"explanation for that result — `runs get` lists a run's results and their ids.",
|
|
598
611
|
scope: "project",
|
|
599
612
|
args: [{ name: "result-id", description: "Test result id (from `beryl runs get`)", required: true }],
|
|
600
613
|
async run(ctx, input) {
|
package/dist/commands/tests.js
CHANGED
|
@@ -128,6 +128,9 @@ export const testCommands = [
|
|
|
128
128
|
{
|
|
129
129
|
name: "tests get",
|
|
130
130
|
summary: "Show one test",
|
|
131
|
+
description: "Returns the test's metadata row (status, flags, per-environment last result), not " +
|
|
132
|
+
"the plan — `tests plan` prints the stored JSON plan, `tests script` the rendered " +
|
|
133
|
+
"Playwright spec.",
|
|
131
134
|
scope: "project",
|
|
132
135
|
args: [{ name: "test-id", description: "Test id", required: true }],
|
|
133
136
|
async run(ctx, input) {
|
|
@@ -138,6 +141,8 @@ export const testCommands = [
|
|
|
138
141
|
{
|
|
139
142
|
name: "tests plan",
|
|
140
143
|
summary: "Print a test's current step plan (JSON)",
|
|
144
|
+
description: "Returns the stored json_plan of the test's current version — `tests get` returns " +
|
|
145
|
+
"the metadata row, `tests script` the rendered Playwright spec.",
|
|
141
146
|
scope: "project",
|
|
142
147
|
args: [{ name: "test-id", description: "Test id", required: true }],
|
|
143
148
|
async run(ctx, input) {
|
|
@@ -164,11 +169,16 @@ export const testCommands = [
|
|
|
164
169
|
"image content), you fix the plan file and re-run. The proving run is imported as the " +
|
|
165
170
|
"test's first run (--no-sync to skip). A plan that signs in with a session Beryl captured " +
|
|
166
171
|
"server-side cannot replay locally (that session never leaves Beryl's cloud) — it falls " +
|
|
167
|
-
"back to server-side verification automatically. A session-mode plan replays locally " +
|
|
172
|
+
"back to server-side verification automatically, and says so. A session-mode plan replays locally " +
|
|
168
173
|
"fine: the server renders it with its account's stored sign-in steps in front, so " +
|
|
169
174
|
"the same identity is exercised on your machine. Optional `before` and `after` arrays hold setup and teardown " +
|
|
170
175
|
"steps: `after` runs even when a main step fails, which is how a create/update/delete test " +
|
|
171
|
-
"cleans up the record it made on the runs that go red."
|
|
176
|
+
"cleans up the record it made on the runs that go red. " +
|
|
177
|
+
"Recovery: a 409 `duplicate_title` carries existing_test_id + existing_plan_hash — " +
|
|
178
|
+
"reconcile with that test (`tests get` / `tests set-plan`), don't rename-and-retry; a " +
|
|
179
|
+
"409 `plan_hash_mismatch` means the submitted plan is not the bytes that were replayed " +
|
|
180
|
+
"— re-run `tests create`; a 429 with Retry-After 30 means the verify slots are " +
|
|
181
|
+
"saturated — wait and retry.",
|
|
172
182
|
scope: "project",
|
|
173
183
|
flags: [
|
|
174
184
|
{ name: "title", type: "string", required: true, description: "Title for the new test" },
|
|
@@ -421,7 +431,9 @@ export const testCommands = [
|
|
|
421
431
|
description: "Accepts the same plan shape as `tests create`, including the optional `before` and " +
|
|
422
432
|
"`after` sections — `after` runs on pass and on fail, so cleanup happens even when the " +
|
|
423
433
|
"test goes red. Pass `--description` when the re-authored plan changes what the test " +
|
|
424
|
-
"proves; omit it to keep the test's existing intent."
|
|
434
|
+
"proves; omit it to keep the test's existing intent. Saves the edit with NO replay — " +
|
|
435
|
+
"it rides into the next run unproven; `tests recompile` is the verify-first " +
|
|
436
|
+
"alternative.",
|
|
425
437
|
scope: "project",
|
|
426
438
|
args: [{ name: "test-id", description: "Test id", required: true }],
|
|
427
439
|
flags: [
|
|
@@ -464,9 +476,12 @@ export const testCommands = [
|
|
|
464
476
|
name: "tests quarantine",
|
|
465
477
|
summary: "Mute a flaky test: it keeps running, but its failures stop failing the run",
|
|
466
478
|
description: "A quarantined test still executes and its result is still recorded and visible — its " +
|
|
467
|
-
"red
|
|
479
|
+
"red lands in the run's quarantined_count and gates neither the run's verdict nor exit " +
|
|
480
|
+
"codes, so it can't red-light a deploy. Use " +
|
|
468
481
|
"it on a persistently flaky test instead of deleting it (which destroys the history) or " +
|
|
469
|
-
"asking support to deactivate it (which stops it running at all).
|
|
482
|
+
"asking support to deactivate it (which stops it running at all). After 5 consecutive " +
|
|
483
|
+
"clean passes the test reports rehab_ready — advisory only, nothing un-quarantines " +
|
|
484
|
+
"itself. `off` un-quarantines.",
|
|
470
485
|
scope: "project",
|
|
471
486
|
args: [
|
|
472
487
|
{ name: "test-id", description: "Test id", required: true },
|
|
@@ -501,6 +516,13 @@ export const testCommands = [
|
|
|
501
516
|
{
|
|
502
517
|
name: "tests recompile",
|
|
503
518
|
summary: "Validate + verify an edited plan against the live site before persisting",
|
|
519
|
+
description: "Unlike `tests set-plan` (which saves the edit and lets it ride into the next run), " +
|
|
520
|
+
"this replays the edited plan against the live site before anything persists. A " +
|
|
521
|
+
"deterministic replay failure (verdict `drop`) REJECTS the edit — persisted:false, " +
|
|
522
|
+
"the prior plan stays live — and returns the failure evidence (over MCP the " +
|
|
523
|
+
"screenshot is image content). Verdict `flag` (the runner errored, no verdict on " +
|
|
524
|
+
"the flow) persists the plan but reports it unverified. Returns 429 with " +
|
|
525
|
+
"Retry-After 30 when the 2 inline-verify slots are saturated — wait and retry.",
|
|
504
526
|
scope: "project",
|
|
505
527
|
args: [{ name: "test-id", description: "Test id", required: true }],
|
|
506
528
|
flags: [
|
|
@@ -569,6 +591,9 @@ export const testCommands = [
|
|
|
569
591
|
{
|
|
570
592
|
name: "tests restore",
|
|
571
593
|
summary: "Restore a test to an earlier version",
|
|
594
|
+
description: "Copies the named older version's plan forward as a NEW head version — unlike " +
|
|
595
|
+
"`tests reset`, which flips authored_by back to `system` and leaves the plan " +
|
|
596
|
+
"untouched.",
|
|
572
597
|
scope: "project",
|
|
573
598
|
args: [
|
|
574
599
|
{ name: "test-id", description: "Test id", required: true },
|
|
@@ -584,6 +609,9 @@ export const testCommands = [
|
|
|
584
609
|
{
|
|
585
610
|
name: "tests reset",
|
|
586
611
|
summary: "Discard user edits and return the test to its latest system-authored version",
|
|
612
|
+
description: "Flips authored_by back to `system` WITHOUT changing the plan (the next " +
|
|
613
|
+
"regeneration overwrites it) — unlike `tests restore`, which copies an older " +
|
|
614
|
+
"version's plan forward as a new version.",
|
|
587
615
|
scope: "project",
|
|
588
616
|
args: [{ name: "test-id", description: "Test id", required: true }],
|
|
589
617
|
async run(ctx, input) {
|
|
@@ -632,7 +660,9 @@ export const testCommands = [
|
|
|
632
660
|
summary: "Print the rendered Playwright spec for a test (or an unbanked plan file)",
|
|
633
661
|
description: "With a test id, fetches the banked test's rendered .spec.ts. With --file, compiles a " +
|
|
634
662
|
"plan JSON that has NOT been banked yet — the same render `tests create` proves locally — " +
|
|
635
|
-
"so you can inspect exactly what would run before creating anything."
|
|
663
|
+
"so you can inspect exactly what would run before creating anything. This returns the " +
|
|
664
|
+
"executable spec — `tests plan` returns the stored JSON plan it is rendered from, " +
|
|
665
|
+
"`tests get` the metadata row.",
|
|
636
666
|
scope: "project",
|
|
637
667
|
args: [{ name: "test-id", description: "Test id (omit when passing --file)" }],
|
|
638
668
|
flags: [
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
// Table rows derive from the schema snapshot at module load, so the skill can never
|
|
2
|
+
// drift from the plan language; only the one-liners are hand-written, and parity
|
|
3
|
+
// tests fail when a schema enum gains or loses a member the docs don't cover.
|
|
4
|
+
import { ACTION_PLAN_SCHEMA } from "./schema.generated.js";
|
|
5
|
+
const defs = ACTION_PLAN_SCHEMA.$defs;
|
|
6
|
+
export const ACTION_TYPES = defs.ActionType.enum;
|
|
7
|
+
export const EXPECT_KINDS = defs.ExpectKind.enum;
|
|
8
|
+
export const EMAIL_EXTRACTS = defs.EmailExtract.enum;
|
|
9
|
+
function actionRequires(action) {
|
|
10
|
+
const rule = defs.PlanStep.allOf.find((r) => r.if.properties.action?.const === action && !r.if.properties.extract);
|
|
11
|
+
return rule?.then.required ?? [];
|
|
12
|
+
}
|
|
13
|
+
function expectRequires(kind) {
|
|
14
|
+
const rule = defs.PlanStep.allOf.find((r) => r.if.properties.expect_kind?.const === kind);
|
|
15
|
+
return rule?.then.required ?? [];
|
|
16
|
+
}
|
|
17
|
+
export const ACTION_DOCS = {
|
|
18
|
+
goto: "Navigate to `url` — a path on your app, absolute only for another origin",
|
|
19
|
+
fill: "Type `value` into the `selector` element",
|
|
20
|
+
click: "Click the `selector` element",
|
|
21
|
+
press: "Press keyboard `key` (e.g. `Enter`) on the `selector` element",
|
|
22
|
+
check: "Check the `selector` checkbox",
|
|
23
|
+
uncheck: "Uncheck the `selector` checkbox",
|
|
24
|
+
select: "Choose `option` in the `selector` dropdown",
|
|
25
|
+
hover: "Hover the `selector` element",
|
|
26
|
+
scroll: "Scroll the page (no selector) or bring `selector` into view — see below",
|
|
27
|
+
wait_for: "Wait for the `selector` element to appear",
|
|
28
|
+
expect: "Assert — kinds in the expect table below",
|
|
29
|
+
capture_count: "Bank the live count of `selector` matches under `capture_as` for a later `count_delta`",
|
|
30
|
+
await_email: "Await mail in the run inbox, bank the extracted value under `capture_as` (§5)",
|
|
31
|
+
upload: "Set a file input: `value` names a project config file — see below",
|
|
32
|
+
dialog: "Arm a one-shot accept/dismiss handler for the NEXT step's dialog — see below",
|
|
33
|
+
switch_tab: "Make another open tab the active page; `value` picks it — see below",
|
|
34
|
+
close_tab: "Close the active tab (optional `value` picks one) and fall back to the previous",
|
|
35
|
+
drag: "Drag the `selector` element onto the `value` TARGET selector",
|
|
36
|
+
};
|
|
37
|
+
export const EXPECT_DOCS = {
|
|
38
|
+
visible: "The element is visible",
|
|
39
|
+
attached: "In the DOM, maybe not shown — for carousel/slider content where `visible` is timing-flaky",
|
|
40
|
+
hidden: "The element is not visible",
|
|
41
|
+
checked: "The checkbox/radio is checked",
|
|
42
|
+
enabled: "The element is enabled",
|
|
43
|
+
disabled: "The element is disabled",
|
|
44
|
+
have_text: "The element's text EXACTLY equals `expect_text`",
|
|
45
|
+
have_value: "The input's value equals `expect_text`",
|
|
46
|
+
have_url: "The page URL contains `expect_text` (page-level, no selector)",
|
|
47
|
+
have_title: "The page title contains `expect_text` (page-level, no selector)",
|
|
48
|
+
have_count: "Exactly `expect_count` elements match `selector`",
|
|
49
|
+
persisted: "The record just created is present — a visible match for the `{{unique}}`-named row",
|
|
50
|
+
gone: "Zero matches — absent from the DOM, stronger than `hidden`",
|
|
51
|
+
count_delta: "The match count moved by signed `expect_delta` vs the `capture_ref` baseline",
|
|
52
|
+
};
|
|
53
|
+
export const EXTRACT_DOCS = {
|
|
54
|
+
code: "the one-time code",
|
|
55
|
+
link: "the sign-in/verify URL",
|
|
56
|
+
pattern: "your own regex in `extract_pattern`, exactly one capture group",
|
|
57
|
+
};
|
|
58
|
+
function table(header, rows) {
|
|
59
|
+
return [
|
|
60
|
+
`| ${header} | required fields | meaning |`,
|
|
61
|
+
"|---|---|---|",
|
|
62
|
+
...rows.map(([name, req, doc]) => `| \`${name}\` | ${req} | ${doc} |`),
|
|
63
|
+
].join("\n");
|
|
64
|
+
}
|
|
65
|
+
const fields = (names) => names.map((f) => `\`${f}\``).join(", ") || "—";
|
|
66
|
+
export function actionTable() {
|
|
67
|
+
return table("action", ACTION_TYPES.map((a) => [a, fields(actionRequires(a)), ACTION_DOCS[a] ?? ""]));
|
|
68
|
+
}
|
|
69
|
+
export function expectTable() {
|
|
70
|
+
return table("expect_kind", EXPECT_KINDS.map((k) => [k, fields(expectRequires(k)), EXPECT_DOCS[k] ?? ""]));
|
|
71
|
+
}
|
|
72
|
+
export function extractLine() {
|
|
73
|
+
return EMAIL_EXTRACTS.map((e) => `\`${e}\` (${EXTRACT_DOCS[e] ?? ""})`).join(", ");
|
|
74
|
+
}
|
package/package.json
CHANGED