@beryl-so/cli 0.48.0 → 0.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/beryl-test-skill.js +113 -112
- package/dist/commands/init.js +45 -8
- package/dist/commands/tests.js +5 -5
- package/dist/lint.js +6 -33
- package/dist/schema.generated.js +58 -0
- package/dist/skill-tables.js +2 -0
- package/package.json +1 -1
package/dist/beryl-test-skill.js
CHANGED
|
@@ -48,7 +48,7 @@ export const BERYL_COPILOT_SECTION = `${BERYL_COPILOT_MARK_BEGIN}
|
|
|
48
48
|
${BERYL_GUIDE_POINTER}
|
|
49
49
|
${BERYL_COPILOT_MARK_END}`;
|
|
50
50
|
// The example plan the skill shows verbatim. Exported so the test suite lints it with
|
|
51
|
-
// lintPlan
|
|
51
|
+
// lintPlan; the skill's documented shape must pass `tests_lint` on the first try.
|
|
52
52
|
export const BERYL_TEST_SKILL_EXAMPLE_PLAN = {
|
|
53
53
|
steps: [
|
|
54
54
|
{ action: "goto", url: "/pricing" },
|
|
@@ -63,7 +63,7 @@ export const BERYL_TEST_SKILL_EXAMPLE_PLAN = {
|
|
|
63
63
|
};
|
|
64
64
|
// The OTP/signup example the skill shows verbatim (§5). Exported so the test suite lints
|
|
65
65
|
// it with lintPlan, same as the minimal example — the documented await_email shape must
|
|
66
|
-
// pass `
|
|
66
|
+
// pass `tests_lint` on the first try.
|
|
67
67
|
export const BERYL_TEST_SKILL_OTP_EXAMPLE_PLAN = {
|
|
68
68
|
steps: [
|
|
69
69
|
{ action: "goto", url: "/signup" },
|
|
@@ -89,7 +89,7 @@ export const BERYL_TEST_SKILL_OTP_EXAMPLE_PLAN = {
|
|
|
89
89
|
};
|
|
90
90
|
export const BERYL_TEST_SKILL = `---
|
|
91
91
|
name: beryl-test
|
|
92
|
-
description: Author durable, healable end-to-end tests for a web app with Beryl. Use when writing, running, or fixing a Beryl test locally with your own coding agent
|
|
92
|
+
description: Author durable, healable end-to-end tests for a web app with Beryl. Use when writing, running, or fixing a Beryl test locally with your own coding agent, drafting the plan over the Playwright MCP, writing the natural-language intent, and running it with \`runs_local\`.
|
|
93
93
|
---
|
|
94
94
|
|
|
95
95
|
# Authoring Beryl tests
|
|
@@ -97,52 +97,50 @@ description: Author durable, healable end-to-end tests for a web app with Beryl.
|
|
|
97
97
|
beryl-test v${cliVersion()} — installed by \`beryl init\`; the \`guide\` tool always returns the current version of this guide.
|
|
98
98
|
|
|
99
99
|
A Beryl test drives a real browser through a flow and asserts the flow reached its
|
|
100
|
-
meaningful outcome. \`beryl init\` wired two MCP servers: **beryl** (
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
100
|
+
meaningful outcome. \`beryl init\` wired two MCP servers: **beryl** (the tools you author
|
|
101
|
+
with: \`accounts_list\`, \`tests_create\`, \`runs_local\`, and the rest) and **playwright**
|
|
102
|
+
(drive a real browser). You do everything below by calling those tools. Beryl **heals**
|
|
103
|
+
tests as the app's markup drifts, but only when you supply the three things that make a
|
|
104
|
+
test durable: a real **outcome assertion**, a strong **natural-language intent**, and the
|
|
105
|
+
**local run-fix loop**.
|
|
105
106
|
|
|
106
107
|
## 0. Start here
|
|
107
108
|
|
|
108
|
-
**Everything happens on your machine first
|
|
109
|
-
\`
|
|
110
|
-
|
|
111
|
-
exception are in \`
|
|
109
|
+
**Everything happens on your machine first**: you drive the flow here, and
|
|
110
|
+
\`tests_create\` proves the plan here too, so a working local browser IS the loop.
|
|
111
|
+
Keep \`tests_create\`'s verify on (banking unproven is a trap); replay mechanics and the
|
|
112
|
+
cloud-session exception are in the \`tests_create\` tool's own description.
|
|
112
113
|
|
|
113
|
-
|
|
114
|
+
Set up (once): if you cannot call the Beryl tools yet, run \`beryl init\` in a terminal to
|
|
115
|
+
install the browser and wire the tools. Then, before the first plan:
|
|
114
116
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
beryl accounts set-login <id> --file … --probe … # store its sign-in — required (§1)
|
|
121
|
-
beryl accounts check <id> # does that stored sign-in still work? (§1)
|
|
122
|
-
beryl mailbox get # the address they receive mail at (§5)
|
|
123
|
-
\`\`\`
|
|
117
|
+
- \`envs_list\`: the root URL gotos resolve against (§2). If it is empty, set it with
|
|
118
|
+
\`envs_update\`.
|
|
119
|
+
- \`accounts_list\`: who do authenticated tests sign in as (§1)? Store its sign-in with
|
|
120
|
+
\`accounts_set_login\`, prove it with \`accounts_check\`.
|
|
121
|
+
- \`mailbox_get\`: the address the account receives mail at (§5).
|
|
124
122
|
|
|
125
|
-
\`
|
|
123
|
+
\`accounts_list\` comes FIRST: standing account vs new identity changes the plan you
|
|
126
124
|
write (discovering it afterwards means rewriting), and an authenticated test cannot be
|
|
127
125
|
created until its account's login plan is stored.
|
|
128
126
|
|
|
129
127
|
Then, per test:
|
|
130
128
|
|
|
131
|
-
1. **Drive the flow** in a real browser over the Playwright MCP
|
|
132
|
-
imagination (§2).
|
|
129
|
+
1. **Drive the flow** in a real browser over the Playwright MCP, never from imagination (§2).
|
|
133
130
|
2. **Write the ActionPlan**, with one real outcome assertion (§2).
|
|
134
|
-
3.
|
|
135
|
-
|
|
136
|
-
4. \`
|
|
137
|
-
|
|
138
|
-
replay hands back the failure
|
|
139
|
-
|
|
140
|
-
|
|
131
|
+
3. **Lint it** with \`tests_lint\` (the plan plus its \`read\`/\`write\` access): schema and
|
|
132
|
+
access check, offline.
|
|
133
|
+
4. **Bank it** with \`tests_create\` (the title, the plan, its \`access\`, and the
|
|
134
|
+
\`description\` intent): it replays the plan locally, banks only on green, records no
|
|
135
|
+
run; a red replay hands back the failure to fix and re-run (§3 for the intent, §4 for
|
|
136
|
+
the loop).
|
|
137
|
+
5. When the batch is banked, run the new ids together ONCE with \`runs_local\` (or
|
|
138
|
+
\`runs_trigger\` for the cloud) so they land as one run with real first results.
|
|
141
139
|
|
|
142
140
|
## 1. First: who do your tests sign in as?
|
|
143
141
|
|
|
144
142
|
An authenticated test **signs in as an account that already exists** — it does not sign
|
|
145
|
-
one up. Start every authoring session with \`
|
|
143
|
+
one up. Start every authoring session with \`accounts_list\`:
|
|
146
144
|
|
|
147
145
|
\`\`\`
|
|
148
146
|
label email type login status
|
|
@@ -161,9 +159,9 @@ one plan stays correct across environments.
|
|
|
161
159
|
|
|
162
160
|
**The account needs a stored login plan before any authenticated test can be created.**
|
|
163
161
|
A run replays that plan to produce the session its tests ride, and replays it again
|
|
164
|
-
when the session expires
|
|
162
|
+
when the session expires, so \`tests_create\` rejects a \`requires_auth\` plan whose
|
|
165
163
|
account has none, naming the fix. Store it once per account with
|
|
166
|
-
\`
|
|
164
|
+
\`accounts_set_login\` (§ Session mode).
|
|
167
165
|
|
|
168
166
|
**NEVER paste a real email or password into a plan.** Handles resolve at run time —
|
|
169
167
|
the password never lands in the rendered spec and is scrubbed from artifacts. A pasted
|
|
@@ -181,46 +179,60 @@ The trade is drift — run 200 has 200 of everything run 1 created. So:
|
|
|
181
179
|
- **Assert relatively, never absolutely.** "The row I just created is present"
|
|
182
180
|
(\`expect.persisted\` on a \`{{unique}}\`-named row), not "there are 3 rows" — a count
|
|
183
181
|
assertion goes stale by itself.
|
|
184
|
-
- **
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
182
|
+
- **Cleanup lives in \`after\`, and it returns the account to the exact state you found
|
|
183
|
+
it.** \`after\` runs on pass AND fail; a trailing step inside \`steps\` does not, so
|
|
184
|
+
never put cleanup in \`steps\`.
|
|
185
|
+
- **Created something?** Delete it in \`after\`.
|
|
186
|
+
- **Deleting something?** Create your own victim first (in \`before\` or the opening
|
|
187
|
+
steps, \`{{unique}}\`-named), then delete it and prove \`gone\`; when the deleted thing
|
|
188
|
+
must persist, re-create it in \`after\` as it was. Never aim a delete at standing data:
|
|
189
|
+
run 2 has nothing to delete, and it eats the account's accumulated coverage.
|
|
190
|
+
- **Changed a value (rename, toggle)?** Restore it in \`after\`: set it back to what it
|
|
191
|
+
was. Capture the original BEFORE you change it with \`capture_text\` (or \`capture_value\`
|
|
192
|
+
for a form input), banking it under \`capture_as\`, then \`fill {{handle}}\` in \`after\`
|
|
193
|
+
to put it back.
|
|
194
|
+
- **You label each test \`read\` or \`write\`; Beryl takes your word for it.** Set its
|
|
195
|
+
\`access\` (default \`read\`). Beryl does NOT infer it from your selectors, so YOU
|
|
196
|
+
decide, by one test: **would the app be different after this test runs?**
|
|
197
|
+
- **\`write\`** = it leaves a change behind: it creates, edits, deletes, renames, moves,
|
|
198
|
+
uploads or toggles a record or setting that is still different once the test ends. A
|
|
199
|
+
\`write\` must undo itself: end \`after\` with an \`expect\` proving the undo: \`gone\`,
|
|
200
|
+
\`hidden\`, \`have_count\` or \`count_delta\` for a removal, \`have_text\`,
|
|
201
|
+
\`have_value\` or \`persisted\` for a restore.
|
|
202
|
+
- **\`read\`** = it only observes: navigating, searching, filtering, sorting, opening a
|
|
203
|
+
menu or dialog and reading it, or clicking a destructive button and then CANCELLING.
|
|
204
|
+
Nothing is committed. Opening a confirm dialog and cancelling is a \`read\`; a submit
|
|
205
|
+
that stays disabled is a \`read\`.
|
|
206
|
+
A flow that both changes and reads is a \`write\`. Only the \`write\` label is checked
|
|
207
|
+
(its closing teardown proof, and a \`write\` whose teardown removes nothing fails the
|
|
208
|
+
run); \`read\` is taken at your word, so a real write you mislabel \`read\` silently
|
|
209
|
+
drifts the account. Label deliberately, and flip a banked test with \`tests_set_access\`.
|
|
197
210
|
|
|
198
211
|
### If there is no account yet
|
|
199
212
|
|
|
200
|
-
An empty \`
|
|
213
|
+
An empty \`accounts_list\` means nothing is set up. In order of preference:
|
|
201
214
|
|
|
202
|
-
1. **The user has a dedicated test account** → bank it
|
|
203
|
-
|
|
204
|
-
account only — never a real user's.
|
|
215
|
+
1. **The user has a dedicated test account** → bank it with \`accounts_create\` (email +
|
|
216
|
+
password). A DEDICATED test account only, never a real user's.
|
|
205
217
|
2. **Only Google (or another SSO) sign-in, no email+password form** → the same path:
|
|
206
|
-
ask for a DEDICATED Google test account and bank it with \`
|
|
207
|
-
|
|
218
|
+
ask for a DEDICATED Google test account and bank it with \`accounts_create\`. Its
|
|
219
|
+
sign-in plan runs through Google's own pages: click the app's
|
|
208
220
|
Google button, fill \`#identifierId\` with \`{{login_email}}\`, click
|
|
209
221
|
\`#identifierNext\`, fill \`input[name=Passwd]\` with \`{{login_password}}\`, click
|
|
210
|
-
\`#passwordNext\`, then assert the app's signed-in shell. Store it with
|
|
211
|
-
|
|
222
|
+
\`#passwordNext\`, then assert the app's signed-in shell. Store it with
|
|
223
|
+
\`accounts_set_login\`, prove it with \`accounts_check\`. Author its tests with \`auth_mode:
|
|
212
224
|
"session"\` only, so Google is signed in once per run, never per test. Google may
|
|
213
225
|
still challenge a scripted sign-in now and then ("Verify it's you"); the captured
|
|
214
226
|
session (end of §1) is the fallback for that, not the default.
|
|
215
227
|
3. **No credentials, but the app has a signup form** → let Beryl make one.
|
|
216
|
-
\`
|
|
217
|
-
|
|
228
|
+
\`accounts_create\` (type \`beryl\`), then sign it up with \`accounts_signup\` (its
|
|
229
|
+
signup plan). The plan is the app's own
|
|
218
230
|
signup form as an ActionPlan, ending LOGGED IN, filling \`{{mailbox_address}}\` and
|
|
219
231
|
\`{{login_password}}\`. It stays on the account, so a new environment signs the same
|
|
220
232
|
identity up again by itself. An account that already exists on the site (hand-made,
|
|
221
|
-
or the record was lost) skips signup: store its sign-in with \`
|
|
222
|
-
No password sign-in?
|
|
223
|
-
\`await_email\` through; later sign-ins read the same mailbox.
|
|
233
|
+
or the record was lost) skips signup: store its sign-in with \`accounts_set_login\`.
|
|
234
|
+
No password sign-in? Set the account's login method to \`otp\` (or \`magic_link\`) and
|
|
235
|
+
let the plan \`await_email\` through; later sign-ins read the same mailbox.
|
|
224
236
|
4. **Neither** → the flow is not testable authenticated. Say so rather than guessing.
|
|
225
237
|
|
|
226
238
|
A project with no accounts falls back to the \`LOGIN_EMAIL\` variable +
|
|
@@ -238,21 +250,17 @@ or the failure evidence; nothing to turn on. A session-mode test has no sign-in
|
|
|
238
250
|
to fall back to, so an account with no stored login plan fails at setup with
|
|
239
251
|
\`SESSION_NO_LOGIN_PLAN\`.
|
|
240
252
|
|
|
241
|
-
Set it up once per account:
|
|
242
|
-
|
|
243
|
-
\`\`\`
|
|
244
|
-
beryl accounts set-login <id> --file signin.json --probe probe.json
|
|
245
|
-
beryl accounts check <id> # signs in NOW and proves it — do not skip this
|
|
246
|
-
\`\`\`
|
|
253
|
+
Set it up once per account: \`accounts_set_login\` (the sign-in plan and a probe), then
|
|
254
|
+
\`accounts_check\` (signs in NOW and proves it; do not skip this).
|
|
247
255
|
|
|
248
|
-
What
|
|
249
|
-
|
|
256
|
+
What the sign-in plan and the probe must contain, and the read-before-write rule for
|
|
257
|
+
later edits, are in the \`accounts_set_login\` / \`accounts_get_login\` tools' own
|
|
250
258
|
descriptions.
|
|
251
259
|
|
|
252
260
|
Then write the tests with \`requires_auth: true\`, \`auth_mode: "session"\`, and NO
|
|
253
261
|
sign-in steps.
|
|
254
262
|
|
|
255
|
-
**Creates share the sign-in.** \`
|
|
263
|
+
**Creates share the sign-in.** \`tests_create\` replays session-mode plans against the
|
|
256
264
|
account's established session (reused while it stays live), so a batch of creates costs
|
|
257
265
|
at most one sign-in — but still create one at a time: racing creates race the same
|
|
258
266
|
mailbox for mail.
|
|
@@ -261,7 +269,7 @@ An expired session is never your problem: every run proves the stored session an
|
|
|
261
269
|
miss, signs in again from the login plan. A "reconnect" message only appears when there
|
|
262
270
|
is no stored login plan to refresh from.
|
|
263
271
|
|
|
264
|
-
When \`
|
|
272
|
+
When \`accounts_check\` fails, its description explains which half broke and each
|
|
265
273
|
\`SESSION_*\` reason. The one that changes your authoring: an account marked
|
|
266
274
|
unsupported (its session cannot survive a fresh browser) is not your bug, but it IS
|
|
267
275
|
your move — re-author its session-mode tests with \`auth_mode: "inline"\` and their own
|
|
@@ -272,15 +280,15 @@ sign-in steps.
|
|
|
272
280
|
Invite-a-teammate-and-accept-as-them needs two identities mid-flow — not bankable as
|
|
273
281
|
one test. Bank the half the app shows to account A ("the invitation is listed as
|
|
274
282
|
pending", "the share link is issued") — a real, strong outcome. While AUTHORING you can
|
|
275
|
-
drive the full handshake live: a second account (\`
|
|
276
|
-
|
|
283
|
+
drive the full handshake live: a second account (\`accounts_create\`, labelled), or a
|
|
284
|
+
second mailbox (\`mailbox_create\`, read with \`mailbox_read\`).
|
|
277
285
|
|
|
278
286
|
SSO-only sites are not a dead end: with a dedicated Google test account they follow §1
|
|
279
287
|
item 2 like any password account. The captured session is the FALLBACK — no test account
|
|
280
288
|
can be provided, or Google challenges the scripted sign-in: a human signs in once
|
|
281
289
|
through a live browser view in the Beryl webapp, Beryl stores the session encrypted
|
|
282
|
-
server-side and injects it into cloud runs
|
|
283
|
-
server-side, \`
|
|
290
|
+
server-side and injects it into cloud runs, and \`tests_create\` verifies such plans
|
|
291
|
+
server-side, \`runs_trigger\` runs them, \`runs_local\` skips them (the session never
|
|
284
292
|
leaves Beryl's cloud).
|
|
285
293
|
|
|
286
294
|
## 2. Author locally over the Playwright MCP
|
|
@@ -294,9 +302,9 @@ leaves Beryl's cloud).
|
|
|
294
302
|
- **at least one step is an \`expect\`** (a test that asserts nothing is not a test).
|
|
295
303
|
**A \`goto\` at your own app is a PATH, never a full URL** — \`/pricing\`, not
|
|
296
304
|
\`https://app.example.com/pricing\`. The origin comes from the environment's root
|
|
297
|
-
URL, so one plan runs against prod, staging and a preview; bake the origin in and
|
|
298
|
-
|
|
299
|
-
handoff, a magic link on another domain).
|
|
305
|
+
URL, so one plan runs against prod, staging and a preview; bake the origin in and the
|
|
306
|
+
run's environment is silently ignored. Absolute URLs stay legal for OTHER origins (an
|
|
307
|
+
OAuth handoff, a magic link on another domain).
|
|
300
308
|
Optional \`before\` / \`after\` arrays hold setup and teardown; \`after\` runs even
|
|
301
309
|
when a main step fails, so a create flow can clean up its record — and a deletion
|
|
302
310
|
flow creates its own record in \`before\` first. A step in \`after\` may carry
|
|
@@ -345,21 +353,17 @@ ${JSON.stringify(BERYL_TEST_SKILL_EXAMPLE_PLAN, null, 2)
|
|
|
345
353
|
.map((line) => ` ${line}`)
|
|
346
354
|
.join("\n")}
|
|
347
355
|
\`\`\`
|
|
348
|
-
3. **Validate offline, then create
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
\`create\` banks the plan only on a green local replay; a red one hands back the
|
|
355
|
-
failure — fix and re-run. It records no run. Replay mechanics (where it runs over
|
|
356
|
-
MCP, \`--no-verify\`, the cloud-session exception) are in \`tests create\`'s own
|
|
357
|
-
description. Inspect the exact spec with \`beryl tests script --file plan.json\`.
|
|
356
|
+
3. **Validate offline, then create.** Lint with \`tests_lint\` (schema check, no network),
|
|
357
|
+
then bank with \`tests_create\` (the title, the plan, and the \`description\` intent, see
|
|
358
|
+
§3). \`tests_create\` banks the plan only on a green local replay; a red one hands back
|
|
359
|
+
the failure to fix and re-run. It records no run. Replay mechanics (where it runs over
|
|
360
|
+
MCP, its verify option, the cloud-session exception) are in the \`tests_create\` tool's
|
|
361
|
+
own description. Inspect the exact spec with \`tests_script\`.
|
|
358
362
|
Full ActionPlan JSON Schema:
|
|
359
363
|
https://api.beryl.so/api/v1/schemas/action-plan.schema.json.
|
|
360
|
-
Groups are optional and never block: after the test is banked, if \`
|
|
361
|
-
shows any, offer them once (\`
|
|
362
|
-
|
|
364
|
+
Groups are optional and never block: after the test is banked, if \`groups_list\`
|
|
365
|
+
shows any, offer them once (\`tests_set_groups\`); no answer = leave it ungrouped and
|
|
366
|
+
move on. Never create a group yourself, that is the user's call.
|
|
363
367
|
|
|
364
368
|
### The outcome assertion is the whole game
|
|
365
369
|
|
|
@@ -391,7 +395,7 @@ observable proof the flow worked.
|
|
|
391
395
|
|
|
392
396
|
### Traps when authoring against a real app
|
|
393
397
|
|
|
394
|
-
Every one of these has produced a wrong plan or a red \`
|
|
398
|
+
Every one of these has produced a wrong plan or a red \`tests_create\`. Check them.
|
|
395
399
|
|
|
396
400
|
1. **Your browser may already be signed in.** The Playwright MCP keeps a persistent
|
|
397
401
|
profile. Author while signed in and you never see the gate — you will mark gated
|
|
@@ -423,8 +427,8 @@ Every one of these has produced a wrong plan or a red \`tests create\`. Check th
|
|
|
423
427
|
|
|
424
428
|
## 3. Writing the natural-language intent
|
|
425
429
|
|
|
426
|
-
|
|
427
|
-
when you re-author). 1
|
|
430
|
+
Give the intent as the \`description\` of \`tests_create\` (or \`tests_set_plan\`
|
|
431
|
+
when you re-author). 1-3 sentences. This is the immutable anchor from §6.
|
|
428
432
|
|
|
429
433
|
- **State the purpose, not the steps.** Not "clicks Sign in, types email and password,
|
|
430
434
|
clicks submit" — the trajectory will change. Instead: *what does a green run prove is
|
|
@@ -442,20 +446,17 @@ Weak (describes steps + asserts nothing meaningful):
|
|
|
442
446
|
|
|
443
447
|
## 4. The local run-fix loop
|
|
444
448
|
|
|
445
|
-
Iterate on your machine before you rely on the cloud. \`
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
beryl runs local <test-id> --no-sync --url-override http://localhost:3000 --dir ./beryl-local
|
|
450
|
-
beryl runs local # the whole suite, results recorded in Beryl
|
|
451
|
-
\`\`\`
|
|
449
|
+
Iterate on your machine before you rely on the cloud. \`runs_local\` runs banked tests
|
|
450
|
+
with the Playwright runner bundled in the CLI: no cloud, no waiting for a scheduled run.
|
|
451
|
+
Run it un-synced with a report directory against one test id while you iterate, or with
|
|
452
|
+
no id to run the whole suite and record the results in Beryl.
|
|
452
453
|
|
|
453
|
-
The loop: draft → \`
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
454
|
+
The loop: draft → \`tests_lint\` → \`tests_create\` (proves, records nothing) → \`runs_local\`
|
|
455
|
+
un-synced while iterating → read the run's \`report.json\` to see which step or assertion
|
|
456
|
+
failed and why → fix the plan → \`tests_set_plan\` → run again. When the batch is done, one
|
|
457
|
+
synced \`runs_local\` (or \`runs_trigger\`) records it as a single run. Its options,
|
|
458
|
+
session-mode behavior, and \`await_email\` / authenticated-test handling are in the
|
|
459
|
+
\`runs_local\` tool's own description.
|
|
459
460
|
|
|
460
461
|
## 5. Testing an OTP / signup flow (\`await_email\`)
|
|
461
462
|
|
|
@@ -466,7 +467,7 @@ flag to turn on.
|
|
|
466
467
|
**The project has one permanent mailbox and all of its mail arrives there.** The
|
|
467
468
|
standing account is used one way only, \`requires_auth: true\` (its session is injected
|
|
468
469
|
before the first step); a test never types its address (the save refuses it), only the
|
|
469
|
-
account's own
|
|
470
|
+
account's own signup and sign-in plans fill \`{{mailbox_address}}\`. Every flow that
|
|
470
471
|
sends mail (a signup, a sign-in code, a wrong code, a lockout, a resend timer, an
|
|
471
472
|
invitation) cites \`{{inbox_address}}\`, a \`+tag\` alias minted per execution and read by
|
|
472
473
|
this test alone.
|
|
@@ -476,7 +477,7 @@ this test alone.
|
|
|
476
477
|
| creates an account, or signs in as one it created (a returning-user test: \`before\` signs up the alias and logs out, the steps sign in as it) | an \`after\` section that deletes it through the app, its steps marked \`optional\`; a plan that reads its signup mail is refused at save time without one |
|
|
477
478
|
| only makes mail arrive, read or not, no account created | nothing to clean up; mail is kept for 48 hours |
|
|
478
479
|
|
|
479
|
-
Every alias minted is listed by \`
|
|
480
|
+
Every alias minted is listed by \`mailbox_identities\` with how its teardown
|
|
480
481
|
ended; the run detail shows the same outcome on each result.
|
|
481
482
|
|
|
482
483
|
The wiring is a three-part chain:
|
|
@@ -511,11 +512,11 @@ For a magic-link flow, replace the code steps with
|
|
|
511
512
|
|
|
512
513
|
Two caveats:
|
|
513
514
|
|
|
514
|
-
- \`
|
|
515
|
+
- \`tests_create\` verifies an \`await_email\` plan like any other: the local
|
|
515
516
|
replay receives at the project mailbox and answers each step over the API, so the
|
|
516
517
|
app's mail really is received and extracted before the test is accepted. The replay
|
|
517
|
-
signs up / sends mail FOR REAL
|
|
518
|
-
unwanted. \`
|
|
518
|
+
signs up / sends mail FOR REAL, so turn \`tests_create\`'s verify off only if that side
|
|
519
|
+
effect is unwanted. \`runs_local\` serves \`await_email\` the same way, so the local loop
|
|
519
520
|
covers OTP/signup flows end to end.
|
|
520
521
|
- §2's outcome discipline still applies: the green signal is the post-verification
|
|
521
522
|
state (the welcome screen, the dashboard), not "an email arrived".
|
package/dist/commands/init.js
CHANGED
|
@@ -89,6 +89,22 @@ function claudeAddHint(name, entry) {
|
|
|
89
89
|
function cursorUserConfigPath() {
|
|
90
90
|
return path.join(os.homedir(), ".cursor", "mcp.json");
|
|
91
91
|
}
|
|
92
|
+
// VS Code's user-scope MCP config lives in the profile dir, NOT ~/.vscode (which VS Code
|
|
93
|
+
// ignores): macOS ~/Library/Application Support/Code/User, Linux ~/.config/Code/User,
|
|
94
|
+
// Windows %APPDATA%/Code/User. Pure resolver so all three platforms are testable.
|
|
95
|
+
export function vscodeUserConfigDirFor(platform, home, env) {
|
|
96
|
+
return platform === "darwin"
|
|
97
|
+
? path.join(home, "Library", "Application Support", "Code", "User")
|
|
98
|
+
: platform === "win32"
|
|
99
|
+
? path.join(env.APPDATA ?? path.join(home, "AppData", "Roaming"), "Code", "User")
|
|
100
|
+
: path.join(env.XDG_CONFIG_HOME ?? path.join(home, ".config"), "Code", "User");
|
|
101
|
+
}
|
|
102
|
+
// Returns null when that dir is absent (VS Code not installed there, Insiders / VSCodium,
|
|
103
|
+
// portable mode) so the caller falls back to a repo-local file.
|
|
104
|
+
function vscodeUserConfigDir() {
|
|
105
|
+
const dir = vscodeUserConfigDirFor(process.platform, os.homedir(), process.env);
|
|
106
|
+
return fs.existsSync(dir) ? dir : null;
|
|
107
|
+
}
|
|
92
108
|
// Write generated content that ships with the CLI and changes every release, so anything
|
|
93
109
|
// that differs is overwritten: a stale copy silently feeds agents months-old guidance,
|
|
94
110
|
// which is worse than losing a local edit the user can redo (and `beryl guide` still prints
|
|
@@ -218,7 +234,8 @@ export const initCommands = [
|
|
|
218
234
|
"instead. With --cursor the same two servers are also wired into Cursor and a repo-local " +
|
|
219
235
|
".cursor/rules/beryl.mdc authoring rule is written (Cursor's equivalent of the skill, " +
|
|
220
236
|
"since it reads neither skills dir); --vscode does the same for VS Code / GitHub Copilot " +
|
|
221
|
-
"(.vscode/mcp.json plus a marked
|
|
237
|
+
"(VS Code's user mcp.json, or .vscode/mcp.json with --scope project, plus a marked " +
|
|
238
|
+
"section in .github/copilot-instructions.md). No " +
|
|
222
239
|
"workspace/project pin and no URL prompt: ask Claude to write tests for " +
|
|
223
240
|
"your site and it resolves the workspace, project, and URL. Safe to re-run: every run " +
|
|
224
241
|
"refreshes the authoring skill to this CLI's version (overwriting an older or edited " +
|
|
@@ -244,8 +261,8 @@ export const initCommands = [
|
|
|
244
261
|
{
|
|
245
262
|
name: "vscode",
|
|
246
263
|
type: "boolean",
|
|
247
|
-
description: "Also wire the two MCP servers into .vscode/mcp.json
|
|
248
|
-
".github/copilot-instructions.md for
|
|
264
|
+
description: "Also wire the two MCP servers into VS Code's user config (or .vscode/mcp.json with " +
|
|
265
|
+
"--scope project) and add a Beryl section to .github/copilot-instructions.md for Copilot",
|
|
249
266
|
},
|
|
250
267
|
],
|
|
251
268
|
examples: [
|
|
@@ -341,16 +358,30 @@ export const initCommands = [
|
|
|
341
358
|
// marked section in `.github/copilot-instructions.md` for the guide pointer. VS Code's
|
|
342
359
|
// per-user MCP config is managed through its UI, not a portable file, so scope is moot.
|
|
343
360
|
if (input.flags.vscode) {
|
|
344
|
-
|
|
361
|
+
// User scope (default) targets VS Code's real profile mcp.json, so it works in
|
|
362
|
+
// every project. Project scope, or a missing profile dir, falls back to the repo.
|
|
363
|
+
const userDir = scope === "user" ? vscodeUserConfigDir() : null;
|
|
364
|
+
const file = userDir
|
|
365
|
+
? path.join(userDir, "mcp.json")
|
|
366
|
+
: path.join(cwd, ".vscode", "mcp.json");
|
|
345
367
|
const wrote = mergeMcpConfig(file, "servers", {
|
|
346
368
|
beryl: VSCODE_MCP_SERVER_ENTRY,
|
|
347
369
|
playwright: VSCODE_PLAYWRIGHT_SERVER_ENTRY,
|
|
348
370
|
});
|
|
349
|
-
const where = path.relative(cwd, file);
|
|
371
|
+
const where = userDir ? file : path.relative(cwd, file);
|
|
350
372
|
report("vscode beryl MCP", wrote.beryl, where);
|
|
351
373
|
report("vscode playwright MCP", wrote.playwright, where);
|
|
374
|
+
if (scope === "user" && !userDir) {
|
|
375
|
+
ctx.err(dim(" • VS Code's user config dir was not found, so this went to a repo-local " +
|
|
376
|
+
".vscode/mcp.json. Run --vscode from your project, or add the server via " +
|
|
377
|
+
'VS Code\'s "MCP: Add Server".'));
|
|
378
|
+
}
|
|
352
379
|
const inst = writeCopilotInstructions(cwd);
|
|
353
380
|
ctx.err(`${green("✓")} copilot instructions ${outcomeLabel(inst.outcome)} ${dim(path.relative(cwd, inst.file))}`);
|
|
381
|
+
if (!fs.existsSync(path.join(cwd, ".git"))) {
|
|
382
|
+
ctx.err(dim(" • copilot-instructions is repo-local; this dir is not a git repo, so " +
|
|
383
|
+
"Copilot may not read it. Run --vscode from your project root."));
|
|
384
|
+
}
|
|
354
385
|
}
|
|
355
386
|
const skills = writeSkills(cwd, scope);
|
|
356
387
|
for (const skill of skills) {
|
|
@@ -367,10 +398,16 @@ export const initCommands = [
|
|
|
367
398
|
ctx.err(`${green("✓")} Beryl authoring skill ${what} ${dim(where)}`);
|
|
368
399
|
}
|
|
369
400
|
await ensureLocalPlaywright(ctx, cwd);
|
|
370
|
-
|
|
371
|
-
|
|
401
|
+
// Name the editor's own agent, so the message speaks to what the user just set up.
|
|
402
|
+
const ed = input.flags.vscode
|
|
403
|
+
? { open: "open VS Code", agent: "Copilot" }
|
|
404
|
+
: input.flags.cursor
|
|
405
|
+
? { open: "open Cursor", agent: "Cursor" }
|
|
406
|
+
: { open: "open your editor", agent: "Claude" };
|
|
407
|
+
const nextSteps = `\n${bold(`Beryl is set up. Now ${ed.open} and ask ${ed.agent} to write tests.`)}\n` +
|
|
408
|
+
` ${dim(`• Say: "write tests for https://your-app.com" and ${ed.agent} picks your workspace/project`)}\n` +
|
|
372
409
|
` ${dim(" and sets the URL for you (no pin, no prompt).")}\n` +
|
|
373
|
-
` ${dim(
|
|
410
|
+
` ${dim(`• Playwright MCP is wired, so ${ed.agent} can drive a real browser to author from your plan.`)}\n` +
|
|
374
411
|
`\nAuthor against the ActionPlan JSON Schema: ${cyan(ACTION_PLAN_SCHEMA_URL)}`;
|
|
375
412
|
// The .mcp.json entry is pinned to @latest, but a global install / old npx cache
|
|
376
413
|
// still wins resolution — so tell the user when the CLI they just ran is stale.
|
package/dist/commands/tests.js
CHANGED
|
@@ -110,8 +110,8 @@ export const testCommands = [
|
|
|
110
110
|
{
|
|
111
111
|
name: "access",
|
|
112
112
|
type: "string",
|
|
113
|
-
description: "The label the plan will be banked with
|
|
114
|
-
"
|
|
113
|
+
description: "The label the plan will be banked with: read (default) or write. It is yours to " +
|
|
114
|
+
"set; Beryl does not infer it. Only a write is checked (its closing teardown proof)",
|
|
115
115
|
},
|
|
116
116
|
],
|
|
117
117
|
examples: [
|
|
@@ -580,9 +580,9 @@ export const testCommands = [
|
|
|
580
580
|
{
|
|
581
581
|
name: "tests set-access",
|
|
582
582
|
summary: "Mark a test read or write",
|
|
583
|
-
description: "read: the test only reads the app. write:
|
|
584
|
-
"
|
|
585
|
-
"the
|
|
583
|
+
description: "read: the test only reads the app. write: it creates, edits, renames, deletes or " +
|
|
584
|
+
"toggles something that persists. The label is yours to set; Beryl does not infer it " +
|
|
585
|
+
"from the plan. A write test must clean up in `after`, ending with a proof expect.",
|
|
586
586
|
scope: "project",
|
|
587
587
|
args: [
|
|
588
588
|
{ name: "test-id", description: "Test id", required: true },
|
package/dist/lint.js
CHANGED
|
@@ -9,9 +9,6 @@ export const MAILBOX_ADDRESS_IN_TEST = "This plan types the standing account's a
|
|
|
9
9
|
"and rides the account's session; one that needs an address the site can mail cites " +
|
|
10
10
|
"{{inbox_address}}, an alias only this test reads. Only the account's own signup and " +
|
|
11
11
|
"sign-in plans (accounts signup, accounts set-login) may fill {{mailbox_address}}.";
|
|
12
|
-
export const WRITE_NEEDS_LABEL = (label, detail) => `${label} (${detail}) changes the app, so this test must be marked access 'write' ` +
|
|
13
|
-
"(tests create --access write, or tests set-access <id> write) and clean up in " +
|
|
14
|
-
"'after'. A 'read' test only reads.";
|
|
15
12
|
export const WRITE_NEEDS_CLOSING_TEARDOWN = (detail) => "A 'write' test must end with an 'after' section that undoes what it changed and " +
|
|
16
13
|
"proves it: the last step of 'after' is an expect with expect_kind 'gone', 'hidden', " +
|
|
17
14
|
"'have_count', 'count_delta' (something removed) or 'have_text', 'have_value', " +
|
|
@@ -177,31 +174,6 @@ function identityIssues(plan, provisioning) {
|
|
|
177
174
|
? [{ path: "(root)", message: MINTED_IDENTITY_NEEDS_TEARDOWN }]
|
|
178
175
|
: [];
|
|
179
176
|
}
|
|
180
|
-
// Same verb list as the server's write_signal (api/app/ta_shared/schemas.py).
|
|
181
|
-
const WRITE_VERBS_RE = /\b(create|save|delete|remove|rename|revoke|enable|disable|invite|add|publish|archive|restore|quarantine|un-quarantine|turn on|turn off|join|decline)\b/i;
|
|
182
|
-
function writeSignal(plan) {
|
|
183
|
-
if (typeof plan !== "object" || plan === null)
|
|
184
|
-
return null;
|
|
185
|
-
const { before, steps: main, after } = plan;
|
|
186
|
-
if (steps(after).length > 0)
|
|
187
|
-
return ["after", "a teardown means something was created"];
|
|
188
|
-
const sectioned = steps(before).length > 0 || steps(after).length > 0;
|
|
189
|
-
const label = (section, i) => (sectioned ? `${section}[${i}]` : `step[${i}]`);
|
|
190
|
-
const all = [
|
|
191
|
-
...steps(before).map((s, i) => [label("before", i), s]),
|
|
192
|
-
...steps(main).map((s, i) => [label("steps", i), s]),
|
|
193
|
-
];
|
|
194
|
-
for (const [where, s] of all) {
|
|
195
|
-
if (s.action === "upload")
|
|
196
|
-
return [where, "upload"];
|
|
197
|
-
if (s.action === "click" && typeof s.selector === "string") {
|
|
198
|
-
const hit = WRITE_VERBS_RE.exec(s.selector);
|
|
199
|
-
if (hit)
|
|
200
|
-
return [where, `click on '${hit[0]}'`];
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
return null;
|
|
204
|
-
}
|
|
205
177
|
const TEARDOWN_PROOF_KINDS = new Set([
|
|
206
178
|
"gone",
|
|
207
179
|
"hidden",
|
|
@@ -211,11 +183,11 @@ const TEARDOWN_PROOF_KINDS = new Set([
|
|
|
211
183
|
"have_value",
|
|
212
184
|
"persisted",
|
|
213
185
|
]);
|
|
186
|
+
// The read/write label is the author's call; Beryl no longer infers it from the plan.
|
|
187
|
+
// Only a `write` label is checked here: it must close its teardown with a proof.
|
|
214
188
|
function accessIssues(plan, access) {
|
|
215
|
-
if (access !== "write")
|
|
216
|
-
|
|
217
|
-
return signal ? [{ path: "(root)", message: WRITE_NEEDS_LABEL(signal[0], signal[1]) }] : [];
|
|
218
|
-
}
|
|
189
|
+
if (access !== "write")
|
|
190
|
+
return [];
|
|
219
191
|
if (typeof plan !== "object" || plan === null)
|
|
220
192
|
return [];
|
|
221
193
|
const after = steps(plan.after);
|
|
@@ -232,7 +204,8 @@ function accessIssues(plan, access) {
|
|
|
232
204
|
}
|
|
233
205
|
// `provisioning`: the plan is an account's own signup or sign-in plan, the one place
|
|
234
206
|
// {{mailbox_address}} is legal (mirrors the server's validate_editable_plan flag).
|
|
235
|
-
// `access`: the label the plan will be banked with
|
|
207
|
+
// `access`: the label the plan will be banked with. It is the author's call (Beryl no
|
|
208
|
+
// longer infers it from the plan); given, only a `write` is checked, for its closing proof.
|
|
236
209
|
export function lintPlan(plan, opts = {}) {
|
|
237
210
|
const issues = validate(plan, ACTION_PLAN_SCHEMA, ACTION_PLAN_SCHEMA, "");
|
|
238
211
|
// The identity rules read a well-formed plan; a shape error is reported alone.
|
package/dist/schema.generated.js
CHANGED
|
@@ -15,6 +15,8 @@ export const ACTION_PLAN_SCHEMA = {
|
|
|
15
15
|
"wait_for",
|
|
16
16
|
"expect",
|
|
17
17
|
"capture_count",
|
|
18
|
+
"capture_text",
|
|
19
|
+
"capture_value",
|
|
18
20
|
"await_email",
|
|
19
21
|
"upload",
|
|
20
22
|
"dialog",
|
|
@@ -325,6 +327,62 @@ export const ACTION_PLAN_SCHEMA = {
|
|
|
325
327
|
]
|
|
326
328
|
}
|
|
327
329
|
},
|
|
330
|
+
{
|
|
331
|
+
"if": {
|
|
332
|
+
"properties": {
|
|
333
|
+
"action": {
|
|
334
|
+
"const": "capture_text"
|
|
335
|
+
}
|
|
336
|
+
},
|
|
337
|
+
"required": [
|
|
338
|
+
"action"
|
|
339
|
+
]
|
|
340
|
+
},
|
|
341
|
+
"then": {
|
|
342
|
+
"properties": {
|
|
343
|
+
"capture_as": {
|
|
344
|
+
"minLength": 1,
|
|
345
|
+
"type": "string"
|
|
346
|
+
},
|
|
347
|
+
"selector": {
|
|
348
|
+
"minLength": 1,
|
|
349
|
+
"type": "string"
|
|
350
|
+
}
|
|
351
|
+
},
|
|
352
|
+
"required": [
|
|
353
|
+
"capture_as",
|
|
354
|
+
"selector"
|
|
355
|
+
]
|
|
356
|
+
}
|
|
357
|
+
},
|
|
358
|
+
{
|
|
359
|
+
"if": {
|
|
360
|
+
"properties": {
|
|
361
|
+
"action": {
|
|
362
|
+
"const": "capture_value"
|
|
363
|
+
}
|
|
364
|
+
},
|
|
365
|
+
"required": [
|
|
366
|
+
"action"
|
|
367
|
+
]
|
|
368
|
+
},
|
|
369
|
+
"then": {
|
|
370
|
+
"properties": {
|
|
371
|
+
"capture_as": {
|
|
372
|
+
"minLength": 1,
|
|
373
|
+
"type": "string"
|
|
374
|
+
},
|
|
375
|
+
"selector": {
|
|
376
|
+
"minLength": 1,
|
|
377
|
+
"type": "string"
|
|
378
|
+
}
|
|
379
|
+
},
|
|
380
|
+
"required": [
|
|
381
|
+
"capture_as",
|
|
382
|
+
"selector"
|
|
383
|
+
]
|
|
384
|
+
}
|
|
385
|
+
},
|
|
328
386
|
{
|
|
329
387
|
"if": {
|
|
330
388
|
"properties": {
|
package/dist/skill-tables.js
CHANGED
|
@@ -27,6 +27,8 @@ export const ACTION_DOCS = {
|
|
|
27
27
|
wait_for: "Wait for the `selector` element to appear",
|
|
28
28
|
expect: "Assert — kinds in the expect table below",
|
|
29
29
|
capture_count: "Bank the live count of `selector` matches under `capture_as` for a later `count_delta`",
|
|
30
|
+
capture_text: "Bank the `selector` element's visible text under `capture_as`, to restore it later via `{{handle}}`",
|
|
31
|
+
capture_value: "Bank the `selector` input's value under `capture_as`, to restore it later via `{{handle}}`",
|
|
30
32
|
await_email: "Await mail in the run inbox, bank the extracted value under `capture_as` (§5)",
|
|
31
33
|
upload: "Set a file input: `value` names a project config file — see below",
|
|
32
34
|
dialog: "Arm a one-shot accept/dismiss handler for the NEXT step's dialog — see below",
|
package/package.json
CHANGED