@eidra-umain/greenlight 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +265 -22
- package/dist/browser/browser.d.ts.map +1 -1
- package/dist/browser/browser.js +50 -14
- package/dist/browser/browser.js.map +1 -1
- package/dist/cli/index.js +2 -2
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/run.d.ts.map +1 -1
- package/dist/cli/run.js +151 -38
- package/dist/cli/run.js.map +1 -1
- package/dist/parser/loader.d.ts +12 -3
- package/dist/parser/loader.d.ts.map +1 -1
- package/dist/parser/loader.js +53 -5
- package/dist/parser/loader.js.map +1 -1
- package/dist/parser/schema.d.ts +24 -3
- package/dist/parser/schema.d.ts.map +1 -1
- package/dist/parser/schema.js +12 -2
- package/dist/parser/schema.js.map +1 -1
- package/dist/pilot/a11y-parser.d.ts +6 -1
- package/dist/pilot/a11y-parser.d.ts.map +1 -1
- package/dist/pilot/a11y-parser.js +19 -1
- package/dist/pilot/a11y-parser.js.map +1 -1
- package/dist/pilot/assertions.d.ts.map +1 -1
- package/dist/pilot/assertions.js +15 -0
- package/dist/pilot/assertions.js.map +1 -1
- package/dist/pilot/conditions.d.ts +15 -0
- package/dist/pilot/conditions.d.ts.map +1 -0
- package/dist/pilot/conditions.js +77 -0
- package/dist/pilot/conditions.js.map +1 -0
- package/dist/pilot/datepick.d.ts +21 -0
- package/dist/pilot/datepick.d.ts.map +1 -0
- package/dist/pilot/datepick.js +187 -0
- package/dist/pilot/datepick.js.map +1 -0
- package/dist/pilot/executor.d.ts.map +1 -1
- package/dist/pilot/executor.js +108 -48
- package/dist/pilot/executor.js.map +1 -1
- package/dist/pilot/llm.d.ts +2 -0
- package/dist/pilot/llm.d.ts.map +1 -1
- package/dist/pilot/llm.js +59 -3
- package/dist/pilot/llm.js.map +1 -1
- package/dist/pilot/locator.d.ts.map +1 -1
- package/dist/pilot/locator.js +43 -0
- package/dist/pilot/locator.js.map +1 -1
- package/dist/pilot/message-builder.d.ts +7 -1
- package/dist/pilot/message-builder.d.ts.map +1 -1
- package/dist/pilot/message-builder.js +18 -5
- package/dist/pilot/message-builder.js.map +1 -1
- package/dist/pilot/pilot.d.ts +2 -0
- package/dist/pilot/pilot.d.ts.map +1 -1
- package/dist/pilot/pilot.js +167 -8
- package/dist/pilot/pilot.js.map +1 -1
- package/dist/pilot/prompts.d.ts +3 -2
- package/dist/pilot/prompts.d.ts.map +1 -1
- package/dist/pilot/prompts.js +156 -27
- package/dist/pilot/prompts.js.map +1 -1
- package/dist/pilot/random.d.ts +37 -0
- package/dist/pilot/random.d.ts.map +1 -0
- package/dist/pilot/random.js +55 -0
- package/dist/pilot/random.js.map +1 -0
- package/dist/pilot/response-parser.d.ts +19 -1
- package/dist/pilot/response-parser.d.ts.map +1 -1
- package/dist/pilot/response-parser.js +148 -62
- package/dist/pilot/response-parser.js.map +1 -1
- package/dist/pilot/state.d.ts +1 -1
- package/dist/pilot/state.d.ts.map +1 -1
- package/dist/pilot/state.js +170 -38
- package/dist/pilot/state.js.map +1 -1
- package/dist/planner/plan-generator.d.ts +6 -0
- package/dist/planner/plan-generator.d.ts.map +1 -1
- package/dist/planner/plan-generator.js +26 -0
- package/dist/planner/plan-generator.js.map +1 -1
- package/dist/planner/plan-runner.d.ts +1 -0
- package/dist/planner/plan-runner.d.ts.map +1 -1
- package/dist/planner/plan-runner.js +104 -4
- package/dist/planner/plan-runner.js.map +1 -1
- package/dist/planner/plan-store.d.ts +6 -0
- package/dist/planner/plan-store.d.ts.map +1 -1
- package/dist/planner/plan-store.js +27 -0
- package/dist/planner/plan-store.js.map +1 -1
- package/dist/planner/plan-types.d.ts +17 -2
- package/dist/planner/plan-types.d.ts.map +1 -1
- package/dist/reporter/types.d.ts +15 -2
- package/dist/reporter/types.d.ts.map +1 -1
- package/dist/types.d.ts +3 -3
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +4 -1
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@ No selectors. No XPaths. No test IDs, drivers or glue code. Just describe what a
|
|
|
10
10
|
|
|
11
11
|
---
|
|
12
12
|
|
|
13
|
-
**[How it works](#how-it-works)** | **[Quick start](#quick-start)** | **[Project configuration](#project-configuration)** | **[CLI](#cli)** | **[Test syntax](#test-syntax)** | **[Cached plans](#cached-plans)** | **[LLM setup](#llm-setup)** | **[Architecture](#architecture)** | **[Avoiding side effects](#avoiding-side-effects-in-your-app)** | **[CI/CD](#cicd)**
|
|
13
|
+
**[How it works](#how-it-works)** | **[Quick start](#quick-start)** | **[Project configuration](#project-configuration)** | **[CLI](#cli)** | **[Test syntax](#test-syntax)** | **[Writing test steps](#writing-test-steps)** | **[Cached plans](#cached-plans)** | **[LLM setup](#llm-setup)** | **[Architecture](#architecture)** | **[Avoiding side effects](#avoiding-side-effects-in-your-app)** | **[CI/CD](#cicd)**
|
|
14
14
|
|
|
15
15
|
---
|
|
16
16
|
|
|
@@ -18,7 +18,6 @@ No selectors. No XPaths. No test IDs, drivers or glue code. Just describe what a
|
|
|
18
18
|
|
|
19
19
|
```yaml
|
|
20
20
|
suite: "Product Search"
|
|
21
|
-
base_url: "https://staging.example.com"
|
|
22
21
|
|
|
23
22
|
tests:
|
|
24
23
|
- name: "Filtering reduces results"
|
|
@@ -34,7 +33,7 @@ tests:
|
|
|
34
33
|
- check that you see "Thanks for your inquiry"
|
|
35
34
|
```
|
|
36
35
|
|
|
37
|
-
GreenLight understands form wizards, custom dropdowns, autocomplete fields, checkbox consent flows, and interactive maps. It fills in forms with realistic test data, handles before/after value comparisons, and works with any UI framework.
|
|
36
|
+
GreenLight understands form wizards, custom dropdowns, autocomplete fields, checkbox consent flows, complex date pickers and interactive maps. It fills in forms with realistic test data, handles before/after value comparisons, and works with any UI framework.
|
|
38
37
|
|
|
39
38
|
The first run uses an LLM to discover the right actions (the **discovery run**). After that, GreenLight caches a concrete action plan and replays it without LLM calls — making subsequent runs fast and deterministic.
|
|
40
39
|
|
|
@@ -119,7 +118,7 @@ If there are multiple deployments and no `default_deployment` is set, the `--dep
|
|
|
119
118
|
| `suites` | string[] | Paths or globs to suite YAML files (required) |
|
|
120
119
|
| `deployments` | map | Named deployment targets |
|
|
121
120
|
| `default_deployment` | string | Which deployment to use by default |
|
|
122
|
-
| `base_url` | string | Base URL for the site under test |
|
|
121
|
+
| `base_url` | string | Base URL for the site under test (in deployments or top-level config) |
|
|
123
122
|
| `model` | string | LLM model identifier |
|
|
124
123
|
| `llm_base_url` | string | Base URL for the OpenAI-compatible API |
|
|
125
124
|
| `timeout` | number | Per-step timeout in milliseconds |
|
|
@@ -147,9 +146,9 @@ greenlight run --model openai/gpt-4o # override LLM model
|
|
|
147
146
|
greenlight run --llm-base-url <url> # use a different OpenAI-compatible API
|
|
148
147
|
greenlight run --debug # verbose output (actions, LLM modes, timings)
|
|
149
148
|
greenlight run --trace # timestamped browser events for perf analysis
|
|
150
|
-
greenlight run --
|
|
149
|
+
greenlight run --pilot # force pilot (LLM) run, ignore cached plans
|
|
151
150
|
greenlight run --plan-status # show cache status for all tests
|
|
152
|
-
greenlight run --on-drift rerun # re-
|
|
151
|
+
greenlight run --on-drift rerun # re-run with pilot on cached plan drift (default: fail)
|
|
153
152
|
```
|
|
154
153
|
|
|
155
154
|
## GreenLight philosophy compared to Gherkin/Cucumber
|
|
@@ -171,7 +170,7 @@ GreenLight takes a different approach:
|
|
|
171
170
|
|
|
172
171
|
## Test syntax
|
|
173
172
|
|
|
174
|
-
Tests are plain English. The Pilot interprets intent, so phrasing is flexible.
|
|
173
|
+
Tests are plain English. The Pilot interprets intent, so phrasing is flexible. Quick reference:
|
|
175
174
|
|
|
176
175
|
| Action | Example |
|
|
177
176
|
|--------|---------|
|
|
@@ -182,32 +181,268 @@ Tests are plain English. The Pilot interprets intent, so phrasing is flexible. C
|
|
|
182
181
|
| Form fill | `fill in the contact form with email "a@b.com" and some test data` |
|
|
183
182
|
| Autocomplete | `type "Stock" into the city field and select the first suggestion` |
|
|
184
183
|
| Check | `check the "I agree to terms" checkbox` |
|
|
184
|
+
| Random data | `name the booking a random string` |
|
|
185
|
+
| Date/time | `set the start time to 10 minutes from now` |
|
|
185
186
|
| Remember | `remember the number of search results` |
|
|
186
187
|
| Compare | `check that the number of results is less than before` |
|
|
188
|
+
| Assert remembered | `check that the booking we just created is visible` |
|
|
187
189
|
| Assert | `check that page contains "Order Confirmed"` |
|
|
190
|
+
| Conditional | `if "Accept cookies" is visible, click it` |
|
|
188
191
|
| Map assert | `check that the map shows "Stockholm"` or `check that zoom level is at least 10` |
|
|
189
192
|
| Multi-step | `Select Red - Green - Blue in the color picker` (auto-split into 3 clicks) |
|
|
190
193
|
|
|
194
|
+
See **[Writing test steps](#writing-test-steps)** below for detailed guidance on each action type, including form filling, value comparisons, conditionals, map testing, and reusable steps.
|
|
195
|
+
|
|
196
|
+
## Writing test steps
|
|
197
|
+
|
|
198
|
+
This section covers how GreenLight interprets your plain-English steps, what happens behind the scenes, and how to write steps that work reliably.
|
|
199
|
+
|
|
200
|
+
### How steps are processed
|
|
201
|
+
|
|
202
|
+
When you write a step like `click the "Add to Cart" button`, GreenLight's **planner** (an LLM) converts it into a structured action before any browser interaction happens. Understanding this pipeline helps you write clearer steps:
|
|
203
|
+
|
|
204
|
+
1. **Planning** — the LLM reads all your steps and classifies each one into an action type
|
|
205
|
+
2. **Execution** — the pilot executes each action against the live browser page
|
|
206
|
+
3. **Caching** — successful runs are saved so future runs skip the LLM entirely
|
|
207
|
+
|
|
208
|
+
### Navigation
|
|
209
|
+
|
|
210
|
+
Navigate to pages by URL or by describing what to click:
|
|
211
|
+
|
|
212
|
+
```yaml
|
|
213
|
+
steps:
|
|
214
|
+
- go to "/products" # direct URL navigation
|
|
215
|
+
- navigate to "https://example.com/about" # full URL
|
|
216
|
+
- navigate to About from the main menu # click a link (uses the live page)
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
**Tip:** Use quoted paths starting with `/` or `http` for direct navigation. For anything that requires finding and clicking a link, describe it naturally — GreenLight will resolve it against the live page.
|
|
220
|
+
|
|
221
|
+
### Clicking and interacting
|
|
222
|
+
|
|
223
|
+
```yaml
|
|
224
|
+
steps:
|
|
225
|
+
- click "Add to Cart" # click a button or link by text
|
|
226
|
+
- click the Submit button # natural description
|
|
227
|
+
- press Enter # keyboard key
|
|
228
|
+
- scroll down # page scrolling
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
When you put text in quotes, GreenLight looks for an element with that exact text. Without quotes, it interprets the description more loosely against the accessibility tree.
|
|
232
|
+
|
|
233
|
+
### Typing and form fields
|
|
234
|
+
|
|
235
|
+
```yaml
|
|
236
|
+
steps:
|
|
237
|
+
- enter "jane@example.com" into "Email" # type into a specific field
|
|
238
|
+
- type "Stockholm" into the search field # natural field description
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
GreenLight types character-by-character to trigger JavaScript event handlers (autocomplete, validation, etc.), just like a real user.
|
|
242
|
+
|
|
243
|
+
### Dropdowns and selects
|
|
244
|
+
|
|
245
|
+
```yaml
|
|
246
|
+
steps:
|
|
247
|
+
- select "Canada" from "Country" # works with native <select> and custom dropdowns
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
A single `select` step handles both opening the dropdown and choosing the option — don't split it into two steps.
|
|
251
|
+
|
|
252
|
+
### Autocomplete fields
|
|
253
|
+
|
|
254
|
+
```yaml
|
|
255
|
+
steps:
|
|
256
|
+
- type "Stock" into the city field and select the first suggestion
|
|
257
|
+
- type "New" into the city field and select "New York"
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
GreenLight detects autocomplete fields and handles the type-wait-select flow automatically. Specify which suggestion to pick, or default to the first.
|
|
261
|
+
|
|
262
|
+
### Checkboxes
|
|
263
|
+
|
|
264
|
+
```yaml
|
|
265
|
+
steps:
|
|
266
|
+
- check the "I agree to terms" checkbox
|
|
267
|
+
- uncheck the "Newsletter" checkbox
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
Use `check`/`uncheck` instead of `click` for checkboxes — this ensures correct toggle behavior for both native and custom checkbox implementations.
|
|
271
|
+
|
|
191
272
|
### Form filling
|
|
192
273
|
|
|
193
|
-
|
|
274
|
+
For entire forms, describe what to fill and GreenLight inspects the actual form fields to generate appropriate test data:
|
|
275
|
+
|
|
276
|
+
```yaml
|
|
277
|
+
steps:
|
|
278
|
+
- fill in the contact form with email "test@example.com" and some test data
|
|
279
|
+
- fill in the registration form and submit it
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
At runtime, GreenLight reads the form's field labels, placeholders, input types, and required status, then generates realistic data for each field. Explicitly mentioned values (like the email above) are used exactly. Autocomplete fields are detected and handled with type-wait-select flows. Consent checkboxes are automatically checked.
|
|
283
|
+
|
|
284
|
+
### Multi-step interactions
|
|
285
|
+
|
|
286
|
+
When a step describes multiple sequential actions, separate them with dashes:
|
|
287
|
+
|
|
288
|
+
```yaml
|
|
289
|
+
steps:
|
|
290
|
+
- Select Red - Green - Blue in the color picker # three sequential clicks
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
Each value becomes a separate click action. Use this for tabs, radio cards, multi-select UIs — anything where you're clicking multiple items in sequence.
|
|
294
|
+
|
|
295
|
+
### Assertions
|
|
296
|
+
|
|
297
|
+
Any step starting with "check that" or "verify" is treated as an assertion — it validates page state without interacting:
|
|
298
|
+
|
|
299
|
+
```yaml
|
|
300
|
+
steps:
|
|
301
|
+
# Text presence (most common)
|
|
302
|
+
- check that the page contains "Order Confirmed"
|
|
303
|
+
- check that the page does not contain "Error"
|
|
304
|
+
|
|
305
|
+
# URL checks
|
|
306
|
+
- check that the URL contains "/success"
|
|
307
|
+
|
|
308
|
+
# Element visibility
|
|
309
|
+
- check that "Welcome back" is visible
|
|
310
|
+
- check that the error message is not visible
|
|
311
|
+
|
|
312
|
+
# Element state
|
|
313
|
+
- verify that the "Submit" button is disabled
|
|
314
|
+
- verify that the "Submit" button is enabled
|
|
315
|
+
|
|
316
|
+
# Numeric comparisons
|
|
317
|
+
- check that the count of products is greater than 0
|
|
318
|
+
- verify there are at least 5 results
|
|
319
|
+
```
|
|
320
|
+
|
|
321
|
+
**Important:** Assertions with quoted text (like `"Order Confirmed"`) are resolved at plan time — no LLM call needed at runtime. Assertions without quotes (like `check that the form is present`) require the live page to resolve.
|
|
322
|
+
|
|
323
|
+
### Random test data
|
|
324
|
+
|
|
325
|
+
When a step asks to type "a random string", "some test data", or similar, the pilot generates a realistic random value and types it. This is useful for creating unique records in tests:
|
|
326
|
+
|
|
327
|
+
```yaml
|
|
328
|
+
steps:
|
|
329
|
+
- name the booking a random string
|
|
330
|
+
- enter a random email into the "Email" field
|
|
331
|
+
- fill the "Description" field with some test data
|
|
332
|
+
```
|
|
333
|
+
|
|
334
|
+
For full forms with multiple fields, use the form filling syntax (`fill in the form with some test data`) — GreenLight inspects each field's label, type, and placeholder to generate appropriate data.
|
|
194
335
|
|
|
195
|
-
###
|
|
336
|
+
### Date and time pickers
|
|
337
|
+
|
|
338
|
+
GreenLight has built-in support for date/time pickers — including native HTML5 inputs and component-library widgets like MUI DateTimePicker with sectioned spinbuttons. No special syntax is needed; just describe what time to set using natural language:
|
|
339
|
+
|
|
340
|
+
```yaml
|
|
341
|
+
steps:
|
|
342
|
+
- set the start time to 10 minutes from now
|
|
343
|
+
- set the end time to 1 hour from now
|
|
344
|
+
- set the deadline to tomorrow at 3pm
|
|
345
|
+
- set the check-in date to next Monday
|
|
346
|
+
- enter 2026-06-15 in the date field
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
**How it works:**
|
|
350
|
+
|
|
351
|
+
1. The planner identifies date/time steps and extracts the time expression (e.g., "10 minutes from now")
|
|
352
|
+
2. [chrono-node](https://github.com/wanasit/chrono) parses the expression into an exact timestamp — no LLM needed for time math
|
|
353
|
+
3. GreenLight inspects the page's accessibility tree to find the picker elements
|
|
354
|
+
4. Values are filled into the correct fields automatically
|
|
355
|
+
|
|
356
|
+
**Supported picker types:**
|
|
357
|
+
|
|
358
|
+
- **Native HTML5** (`<input type="date">`, `datetime-local`, `time`) — filled with `YYYY-MM-DD`, `YYYY-MM-DDTHH:mm`, or `HH:mm`
|
|
359
|
+
- **Sectioned pickers** (MUI v7, etc.) — individual spinbutton sections (Day, Month, Year, Hours, Minutes) are each filled with the correct value
|
|
360
|
+
- **Multiple pickers on one page** — GreenLight matches "start" / "end" in the step text against picker group names to target the right one
|
|
361
|
+
|
|
362
|
+
**Supported time expressions:**
|
|
363
|
+
|
|
364
|
+
| Expression | Example result |
|
|
365
|
+
|-----------|---------------|
|
|
366
|
+
| `10 minutes from now` | Current time + 10 minutes |
|
|
367
|
+
| `1 hour from now` | Current time + 1 hour |
|
|
368
|
+
| `tomorrow` | Tomorrow, same time |
|
|
369
|
+
| `tomorrow at 3pm` | Tomorrow at 15:00 |
|
|
370
|
+
| `next Monday` | The upcoming Monday |
|
|
371
|
+
| `2026-06-15` | Explicit date |
|
|
372
|
+
| `2026-06-15 14:30` | Explicit date and time |
|
|
373
|
+
|
|
374
|
+
Date picker steps are always computed fresh — even on cached plan replay, the timestamp is recalculated from the current time. This means tests with relative times like "10 minutes from now" never fail due to stale cached dates.
|
|
375
|
+
|
|
376
|
+
### Value comparisons (remember/compare)
|
|
196
377
|
|
|
197
378
|
Remember a value before an action, then compare after:
|
|
198
379
|
|
|
199
380
|
```yaml
|
|
200
381
|
steps:
|
|
201
|
-
- remember the
|
|
382
|
+
- remember the number of search results
|
|
202
383
|
- apply the "In Stock" filter
|
|
203
|
-
- check that the
|
|
384
|
+
- check that the number of results has decreased
|
|
204
385
|
```
|
|
205
386
|
|
|
206
|
-
|
|
387
|
+
GreenLight captures the numeric value from the page, stores it, and compares it after the action. Supported operators: `increased`, `decreased`, `greater than`, `less than`, `equal to`, `at least`, `at most`.
|
|
207
388
|
|
|
208
|
-
|
|
389
|
+
You can also remember non-numeric values and check that they appear on the page later:
|
|
209
390
|
|
|
210
|
-
|
|
391
|
+
```yaml
|
|
392
|
+
steps:
|
|
393
|
+
- name the booking a random string
|
|
394
|
+
- remember the name of the booking
|
|
395
|
+
- click "Save"
|
|
396
|
+
- check that the booking we just created is visible in the list
|
|
397
|
+
```
|
|
398
|
+
|
|
399
|
+
This flow generates a random value, captures it, performs an action, and then verifies the value still appears on the page. Useful for testing create/edit/delete workflows where you need to verify that a specific record you just created shows up correctly.
|
|
400
|
+
|
|
401
|
+
### Conditional steps
|
|
402
|
+
|
|
403
|
+
Conditional steps let you handle pages that may look different between runs — cookie banners, feature flags, A/B tests, or optional UI elements.
|
|
404
|
+
|
|
405
|
+
#### Inline conditionals
|
|
406
|
+
|
|
407
|
+
Write the condition and action as a single step:
|
|
408
|
+
|
|
409
|
+
```yaml
|
|
410
|
+
steps:
|
|
411
|
+
- if "Accept cookies" is visible, click it
|
|
412
|
+
- click "Dismiss" if visible
|
|
413
|
+
- if the page shows "Out of stock" then click "Notify me" else click "Add to cart"
|
|
414
|
+
- if url contains "/login" then check that page contains "Sign in"
|
|
415
|
+
```
|
|
416
|
+
|
|
417
|
+
#### Block conditionals
|
|
418
|
+
|
|
419
|
+
For multi-step branches, use YAML structure:
|
|
420
|
+
|
|
421
|
+
```yaml
|
|
422
|
+
steps:
|
|
423
|
+
- navigate to /home
|
|
424
|
+
- if: cookie banner is visible
|
|
425
|
+
then:
|
|
426
|
+
- click "Accept all"
|
|
427
|
+
- wait for the banner to disappear
|
|
428
|
+
else:
|
|
429
|
+
- check that the product list is visible
|
|
430
|
+
- search for "headphones"
|
|
431
|
+
```
|
|
432
|
+
|
|
433
|
+
The `else` block is optional. Block conditionals are flattened to inline conditionals at load time — the planner and pilot handle them the same way.
|
|
434
|
+
|
|
435
|
+
#### Supported conditions
|
|
436
|
+
|
|
437
|
+
- **Visibility:** `if "text" is visible`, `if the page shows "text"`
|
|
438
|
+
- **Text presence:** `if page contains "text"`, `if the page shows "text"`
|
|
439
|
+
- **URL:** `if url contains "/path"`
|
|
440
|
+
|
|
441
|
+
If the condition is false and there's no `else`, the step is simply skipped (not failed). The test report shows which branch was taken: `[then]`, `[else]`, or `[skipped]`.
|
|
442
|
+
|
|
443
|
+
### Map testing
|
|
444
|
+
|
|
445
|
+
GreenLight has built-in support for testing interactive WebGL maps. When a step mentions maps, markers, layers, or zoom levels, GreenLight detects the map library, attaches to its instance, and queries rendered features directly — bypassing the DOM (WebGL canvas content is invisible to the accessibility tree).
|
|
211
446
|
|
|
212
447
|
```yaml
|
|
213
448
|
steps:
|
|
@@ -217,16 +452,16 @@ steps:
|
|
|
217
452
|
- check that the "hospitals" layer is visible on the map
|
|
218
453
|
```
|
|
219
454
|
|
|
220
|
-
|
|
455
|
+
Currently supported: **MapLibre GL JS**. The architecture is pluggable — Mapbox GL and Leaflet adapters can be added.
|
|
221
456
|
|
|
222
|
-
**
|
|
457
|
+
**How it works:** When a step mentions map-related content, GreenLight automatically inserts a map detection step. Map assertions query the actual rendered vector tile features (place names, road labels, etc.) and viewport state (center, zoom, bounds, layers) — not the DOM.
|
|
223
458
|
|
|
224
|
-
|
|
225
|
-
2. **Vue apps** — checks `__vue_app__` (Vue 3) and `__vue__` (Vue 2) component trees
|
|
226
|
-
3. **Global variables** — scans `window.map`, `window.mapInstance`, and similar common names
|
|
227
|
-
4. **Explicit exposure** — for maximum reliability, set `window.__greenlight_map = map` in your app
|
|
459
|
+
**Map instance detection** works automatically for most setups:
|
|
228
460
|
|
|
229
|
-
|
|
461
|
+
1. **React apps** (react-map-gl, etc.) — walks the React fiber tree
|
|
462
|
+
2. **Vue apps** — checks `__vue_app__` / `__vue__` component trees
|
|
463
|
+
3. **Global variables** — scans `window.map`, `window.mapInstance`, etc.
|
|
464
|
+
4. **Explicit exposure** — set `window.__greenlight_map = map` for maximum reliability
|
|
230
465
|
|
|
231
466
|
### Reusable steps
|
|
232
467
|
|
|
@@ -247,6 +482,14 @@ tests:
|
|
|
247
482
|
- check that page contains "Account Settings"
|
|
248
483
|
```
|
|
249
484
|
|
|
485
|
+
### Tips for reliable tests
|
|
486
|
+
|
|
487
|
+
- **Be specific with quotes** — `click "Submit"` is more reliable than `click the submit button` because it matches exact text.
|
|
488
|
+
- **One action per step** — each step should describe one thing a user does. GreenLight splits compound steps, but explicit single-action steps are more predictable.
|
|
489
|
+
- **Use assertions liberally** — after key actions, add a `check that` step to verify the expected outcome. This catches failures early and makes the test report more readable.
|
|
490
|
+
- **Don't over-specify DOM structure** — say `click "Add to Cart"` not `click the button inside the product card div`. GreenLight uses the accessibility tree, not CSS selectors.
|
|
491
|
+
- **Conditional steps for flaky UI** — if a cookie banner or popup sometimes appears, use `click "Accept" if visible` instead of making it a required step.
|
|
492
|
+
|
|
250
493
|
## Cached plans
|
|
251
494
|
|
|
252
495
|
The first run of a test uses LLM calls to discover the right browser actions (**discovery run**). After a successful run, GreenLight caches the concrete action sequence as a **heuristic plan** in `.greenlight/plans/`.
|
|
@@ -255,7 +498,7 @@ Subsequent runs replay the cached plan directly via Playwright — no LLM calls,
|
|
|
255
498
|
|
|
256
499
|
```bash
|
|
257
500
|
greenlight run # uses cached plans where available
|
|
258
|
-
greenlight run --
|
|
501
|
+
greenlight run --pilot # force pilot (LLM) run, ignore cache
|
|
259
502
|
greenlight run --plan-status # show which tests have cached plans
|
|
260
503
|
```
|
|
261
504
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../../src/browser/browser.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAY,KAAK,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK,IAAI,EAAE,MAAM,YAAY,CAAA;AACnF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAE5C,MAAM,WAAW,cAAc;IAC9B,MAAM,EAAE,OAAO,CAAA;IACf,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;CAC3C;
|
|
1
|
+
{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../../src/browser/browser.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAY,KAAK,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK,IAAI,EAAE,MAAM,YAAY,CAAA;AACnF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAE5C,MAAM,WAAW,cAAc;IAC9B,MAAM,EAAE,OAAO,CAAA;IACf,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;CAC3C;AAuBD,0CAA0C;AAC1C,wBAAsB,aAAa,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,CAK5E;AAED,mEAAmE;AACnE,wBAAsB,aAAa,CAClC,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,cAAc,GACpB,OAAO,CAAC,cAAc,CAAC,CAIzB;AAgBD;;;;GAIG;AACH,wBAAsB,+BAA+B,CACpD,MAAM,EAAE,cAAc,GACpB,OAAO,CAAC;IAAE,OAAO,EAAE,cAAc,CAAA;CAAE,CAAC,CAqCtC;AAED,8EAA8E;AAC9E,wBAAsB,UAAU,CAC/B,OAAO,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,GAC5B,OAAO,CAAC,IAAI,CAAC,CAyCf;AAED,6CAA6C;AAC7C,wBAAsB,YAAY,CAAC,gBAAgB,EAAE,OAAO,GAAG,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAwB5F;AAED,8CAA8C;AAC9C,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,SAAS,GAAG,cAAc,CAKlE"}
|
package/dist/browser/browser.js
CHANGED
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
import path from "path";
|
|
6
6
|
import { createRequire } from "module";
|
|
7
7
|
import { chromium } from "playwright";
|
|
8
|
-
/** Zoom level applied in headed mode (
|
|
9
|
-
const BROWSER_ZOOM =
|
|
8
|
+
/** Zoom level applied in headed mode (50%). */
|
|
9
|
+
const BROWSER_ZOOM = 50;
|
|
10
10
|
/**
|
|
11
11
|
* Resolve the path to the playwright-zoom extension directory.
|
|
12
12
|
* Uses createRequire because playwright-zoom is a CJS package.
|
|
@@ -16,16 +16,18 @@ function zoomExtensionPath() {
|
|
|
16
16
|
const pwZoomDir = path.dirname(require.resolve("playwright-zoom"));
|
|
17
17
|
return path.join(pwZoomDir, "lib", "zoom-extension");
|
|
18
18
|
}
|
|
19
|
+
/** Common Chromium flags shared across launch modes. */
|
|
20
|
+
const CHROMIUM_ARGS = [
|
|
21
|
+
"--enable-webgl",
|
|
22
|
+
"--enable-webgl2-compute-context",
|
|
23
|
+
"--enable-gpu-rasterization",
|
|
24
|
+
"--ignore-gpu-blocklist",
|
|
25
|
+
];
|
|
19
26
|
/** Launch a Chromium browser instance. */
|
|
20
27
|
export async function launchBrowser(config) {
|
|
21
28
|
return chromium.launch({
|
|
22
29
|
headless: !config.headed,
|
|
23
|
-
args:
|
|
24
|
-
"--enable-webgl",
|
|
25
|
-
"--enable-webgl2-compute-context",
|
|
26
|
-
"--enable-gpu-rasterization",
|
|
27
|
-
"--ignore-gpu-blocklist",
|
|
28
|
-
],
|
|
30
|
+
args: CHROMIUM_ARGS,
|
|
29
31
|
});
|
|
30
32
|
}
|
|
31
33
|
/** Create an isolated browser context with configured viewport. */
|
|
@@ -51,14 +53,24 @@ async function applyBrowserZoom(page, zoom) {
|
|
|
51
53
|
*/
|
|
52
54
|
export async function launchPersistentContextWithZoom(config) {
|
|
53
55
|
const extPath = zoomExtensionPath();
|
|
54
|
-
|
|
56
|
+
// Set the window size to match the viewport so the browser chrome
|
|
57
|
+
// doesn't add extra space around the rendered page.
|
|
58
|
+
const { width, height } = config.viewport;
|
|
59
|
+
// Create a temp user data dir with Chrome prefs that disable translate
|
|
60
|
+
const { mkdtempSync, writeFileSync, mkdirSync } = await import("node:fs");
|
|
61
|
+
const { join } = await import("node:path");
|
|
62
|
+
const { tmpdir } = await import("node:os");
|
|
63
|
+
const userDataDir = mkdtempSync(join(tmpdir(), "greenlight-chrome-"));
|
|
64
|
+
const defaultDir = join(userDataDir, "Default");
|
|
65
|
+
mkdirSync(defaultDir, { recursive: true });
|
|
66
|
+
writeFileSync(join(defaultDir, "Preferences"), JSON.stringify({ translate: { enabled: false }, translate_blocked_languages: ["*"] }));
|
|
67
|
+
const context = await chromium.launchPersistentContext(userDataDir, {
|
|
55
68
|
headless: false,
|
|
56
69
|
viewport: config.viewport,
|
|
57
70
|
args: [
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
"--
|
|
61
|
-
"--ignore-gpu-blocklist",
|
|
71
|
+
...CHROMIUM_ARGS,
|
|
72
|
+
`--window-size=${String(width)},${String(height)}`,
|
|
73
|
+
"--disable-features=Translate,TranslateUI",
|
|
62
74
|
`--disable-extensions-except=${extPath}`,
|
|
63
75
|
`--load-extension=${extPath}`,
|
|
64
76
|
],
|
|
@@ -113,7 +125,31 @@ export async function createPage(context, options) {
|
|
|
113
125
|
}
|
|
114
126
|
/** Close a browser or persistent context. */
|
|
115
127
|
export async function closeBrowser(browserOrContext) {
|
|
116
|
-
|
|
128
|
+
try {
|
|
129
|
+
if ("pages" in browserOrContext) {
|
|
130
|
+
// Persistent context (headed mode).
|
|
131
|
+
// Chrome 145 has a bug where its shutdown code crashes with SIGSEGV,
|
|
132
|
+
// triggering the macOS "quit unexpectedly" dialog. To avoid this,
|
|
133
|
+
// get the browser PID via CDP and SIGKILL it directly — the process
|
|
134
|
+
// never runs its buggy shutdown path.
|
|
135
|
+
const pages = browserOrContext.pages();
|
|
136
|
+
const page = pages[0] ?? await browserOrContext.newPage();
|
|
137
|
+
try {
|
|
138
|
+
const cdp = await browserOrContext.newCDPSession(page);
|
|
139
|
+
const info = await cdp.send("SystemInfo.getProcessInfo");
|
|
140
|
+
const browserProc = info.processInfo.find((p) => p.type === "browser");
|
|
141
|
+
if (browserProc) {
|
|
142
|
+
process.kill(browserProc.id, "SIGKILL");
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
catch { /* fall through to normal close */ }
|
|
147
|
+
}
|
|
148
|
+
await browserOrContext.close();
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
// Browser process may have already exited
|
|
152
|
+
}
|
|
117
153
|
}
|
|
118
154
|
/** Extract browser options from RunConfig. */
|
|
119
155
|
export function toBrowserOptions(config) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser.js","sourceRoot":"","sources":["../../src/browser/browser.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAgD,MAAM,YAAY,CAAA;AAQnF,+CAA+C;AAC/C,MAAM,YAAY,GAAG,EAAE,CAAA;AAEvB;;;GAGG;AACH,SAAS,iBAAiB;IACzB,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAA;IAClE,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,gBAAgB,CAAC,CAAA;AACrD,CAAC;AAED,0CAA0C;AAC1C,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAsB;IACzD,OAAO,QAAQ,CAAC,MAAM,CAAC;QACtB,QAAQ,EAAE,CAAC,MAAM,CAAC,MAAM;QACxB,IAAI,EAAE
|
|
1
|
+
{"version":3,"file":"browser.js","sourceRoot":"","sources":["../../src/browser/browser.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAgD,MAAM,YAAY,CAAA;AAQnF,+CAA+C;AAC/C,MAAM,YAAY,GAAG,EAAE,CAAA;AAEvB;;;GAGG;AACH,SAAS,iBAAiB;IACzB,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAA;IAClE,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,gBAAgB,CAAC,CAAA;AACrD,CAAC;AAED,wDAAwD;AACxD,MAAM,aAAa,GAAG;IACrB,gBAAgB;IAChB,iCAAiC;IACjC,4BAA4B;IAC5B,wBAAwB;CACxB,CAAA;AAED,0CAA0C;AAC1C,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAsB;IACzD,OAAO,QAAQ,CAAC,MAAM,CAAC;QACtB,QAAQ,EAAE,CAAC,MAAM,CAAC,MAAM;QACxB,IAAI,EAAE,aAAa;KACnB,CAAC,CAAA;AACH,CAAC;AAED,mEAAmE;AACnE,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,OAAgB,EAChB,MAAsB;IAEtB,OAAO,OAAO,CAAC,UAAU,CAAC;QACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;KACzB,CAAC,CAAA;AACH,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,gBAAgB,CAAC,IAAU,EAAE,IAAY;IACvD,MAAM,IAAI,CAAC,QAAQ,CAClB,CAAC,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,EAAE,GAAG,CAAC,EAC7E,IAAI,CACJ,CAAA;IACD,2CAA2C;IAC3C,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAA;AAC/B,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACpD,MAAsB;IAEtB,MAAM,OAAO,GAAG,iBAAiB,EAAE,CAAA;IACnC,kEAAkE;IAClE,oDAAoD;IACpD,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAA;IAEzC,uEAAuE;IACvE,MAAM,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAA;IACzE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAA;IAC1C,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAA;IAC1C,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,oBAAoB,CAAC,CAAC,CAAA;IACrE,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;IAC/C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAC1C,aAAa,CACZ,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,EAC/B,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,2BAA2B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CACrF,CAAA;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,uBAAuB,CAAC,WAAW,EAAE;QACnE,QAAQ,EAAE,KAAK;QACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,IAAI,EAAE;YACL,GAAG,aAAa;YAChB,iBAAiB,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE;YAClD,0CAA0C;YAC1C,+BAA+B,OAAO,EAAE;YACxC,oBAAoB,OAAO,EAAE;SAC7B;KACD,CAAC,CAAA;IAEF,oDAAoD;IACpD,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,EAAE,CAAA;IACrC,KAAK,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;QAC/B,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IAChC,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,CAAA;AACnB,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,UAAU,CAC/B,OAAuB,EACvB,OAA8B;IAE9B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAA;IACpC,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE;QAC7B,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE;YAC7C,KAAK,EAAE,IAAI;YACX,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;SACnB,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,wEAAwE;IACxE,sEAAsE;IACtE,gEAAgE;IAChE,kEAAkE;IAClE,+DAA+D;IAC/D,sCAAsC;IACtC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;QACxC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,EAAE,CAAA;QAC/B,IAAI,OAAO,CAAC,mBAAmB,EAAE,IAAI,OAAO,CAAC,YAAY,EAAE,KAAK,OAAO,IAAI,OAAO,CAAC,YAAY,EAAE,KAAK,KAAK,EAAE,CAAC;YAC7G,IAAI,CAAC;gBACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;gBACrC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;gBACnC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;oBACtC,MAAM,KAAK,CAAC,QAAQ,CAAC;wBACpB,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE;qBACvD,CAAC,CAAA;oBACF,OAAM;gBACP,CAAC;YACF,CAAC;YAAC,MAAM,CAAC,CAAC,kDAAkD,CAAC,CAAC;QAC/D,CAAC;QACD,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAA;IACvB,CAAC,CAAC,CAAA;IAEF,2DAA2D;IAC3D,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE;YACtB,gBAAgB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QACrD,CAAC,CAAC,CAAA;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,6CAA6C;AAC7C,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,gBAA0C;IAC5E,IAAI,CAAC;QACJ,IAAI,OAAO,IAAI,gBAAgB,EAAE,CAAC;YACjC,oCAAoC;YACpC,qEAAqE;YACrE,kEAAkE;YAClE,oEAAoE;YACpE,sCAAsC;YACtC,MAAM,KAAK,GAAG,gBAAgB,CAAC,KAAK,EAAE,CAAA;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,MAAM,gBAAgB,CAAC,OAAO,EAAE,CAAA;YACzD,IAAI,CAAC;gBACJ,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAA;gBACtD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAoD,CAAA;gBAC3G,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAA;gBACtE,IAAI,WAAW,EAAE,CAAC;oBACjB,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;oBACvC,OAAM;gBACP,CAAC;YACF,CAAC;YAAC,MAAM,CAAC,CAAC,kCAAkC,CAAC,CAAC;QAC/C,CAAC;QACD,MAAM,gBAAgB,CAAC,KAAK,EAAE,CAAA;IAC/B,CAAC;IAAC,MAAM,CAAC;QACR,0CAA0C;IAC3C,CAAC;AACF,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,gBAAgB,CAAC,MAAiB;IACjD,OAAO;QACN,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;KACzB,CAAA;AACF,CAAC"}
|
package/dist/cli/index.js
CHANGED
|
@@ -30,7 +30,7 @@ program
|
|
|
30
30
|
.option("-d, --deployment <name>", "select a named deployment from greenlight.yaml")
|
|
31
31
|
.option("--debug", "enable verbose debug output", false)
|
|
32
32
|
.option("--trace", "log timestamped browser events for performance analysis", false)
|
|
33
|
-
.option("--
|
|
33
|
+
.option("--pilot", "force pilot (LLM) run, ignore cached plans", false)
|
|
34
34
|
.option("--on-drift <mode>", 'behavior on plan drift: "fail" or "rerun" (default: fail)')
|
|
35
35
|
.option("--plan-status", "show cached plan status for all test cases and exit", false)
|
|
36
36
|
.action(async (suitesArg, opts) => {
|
|
@@ -70,7 +70,7 @@ program
|
|
|
70
70
|
projectConfig?.provider ??
|
|
71
71
|
DEFAULTS.provider),
|
|
72
72
|
llmBaseUrl: opts.llmBaseUrl ?? projectConfig?.llm_base_url,
|
|
73
|
-
|
|
73
|
+
pilot: opts.pilot,
|
|
74
74
|
onDrift: parseOnDrift(opts.onDrift ?? DEFAULTS.onDrift),
|
|
75
75
|
};
|
|
76
76
|
// Initialize global runtime state (debug, trace)
|
package/dist/cli/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":";AAEA,OAAO,eAAe,CAAA;AACtB,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,QAAQ,EAAiC,MAAM,aAAa,CAAA;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAC3C,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AAErD,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAA;AAE7B,OAAO;KACL,IAAI,CAAC,YAAY,CAAC;KAClB,WAAW,CAAC,4BAA4B,CAAC;KACzC,OAAO,CAAC,OAAO,CAAC,CAAA;AAElB,OAAO;KACL,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,+CAA+C,CAAC;KAC5D,QAAQ,CACR,aAAa,EACb,uDAAuD,CACvD;KACA,MAAM,CAAC,mBAAmB,EAAE,2CAA2C,CAAC;KACxE,MAAM,CAAC,kBAAkB,EAAE,6BAA6B,CAAC;KACzD,MAAM,CAAC,yBAAyB,EAAE,mCAAmC,CAAC;KACtE,MAAM,CAAC,qBAAqB,EAAE,wCAAwC,CAAC;KACvE,MAAM,CAAC,UAAU,EAAE,sCAAsC,CAAC;KAC1D,MAAM,CAAC,oBAAoB,EAAE,0CAA0C,CAAC;KACxE,MAAM,CAAC,gBAAgB,EAAE,kCAAkC,CAAC;KAC5D,MAAM,CACN,iBAAiB,EACjB,uDAAuD,CACvD;KACA,MAAM,CAAC,sBAAsB,EAAE,qCAAqC,CAAC;KACrE,MAAM,CACN,mBAAmB,EACnB,qDAAqD,CACrD;KACA,MAAM,CACN,yBAAyB,EACzB,gDAAgD,CAChD;KACA,MAAM,CAAC,SAAS,EAAE,6BAA6B,EAAE,KAAK,CAAC;KACvD,MAAM,CACN,SAAS,EACT,yDAAyD,EACzD,KAAK,CACL;KACA,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":";AAEA,OAAO,eAAe,CAAA;AACtB,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,QAAQ,EAAiC,MAAM,aAAa,CAAA;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAC3C,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AAErD,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAA;AAE7B,OAAO;KACL,IAAI,CAAC,YAAY,CAAC;KAClB,WAAW,CAAC,4BAA4B,CAAC;KACzC,OAAO,CAAC,OAAO,CAAC,CAAA;AAElB,OAAO;KACL,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,+CAA+C,CAAC;KAC5D,QAAQ,CACR,aAAa,EACb,uDAAuD,CACvD;KACA,MAAM,CAAC,mBAAmB,EAAE,2CAA2C,CAAC;KACxE,MAAM,CAAC,kBAAkB,EAAE,6BAA6B,CAAC;KACzD,MAAM,CAAC,yBAAyB,EAAE,mCAAmC,CAAC;KACtE,MAAM,CAAC,qBAAqB,EAAE,wCAAwC,CAAC;KACvE,MAAM,CAAC,UAAU,EAAE,sCAAsC,CAAC;KAC1D,MAAM,CAAC,oBAAoB,EAAE,0CAA0C,CAAC;KACxE,MAAM,CAAC,gBAAgB,EAAE,kCAAkC,CAAC;KAC5D,MAAM,CACN,iBAAiB,EACjB,uDAAuD,CACvD;KACA,MAAM,CAAC,sBAAsB,EAAE,qCAAqC,CAAC;KACrE,MAAM,CACN,mBAAmB,EACnB,qDAAqD,CACrD;KACA,MAAM,CACN,yBAAyB,EACzB,gDAAgD,CAChD;KACA,MAAM,CAAC,SAAS,EAAE,6BAA6B,EAAE,KAAK,CAAC;KACvD,MAAM,CACN,SAAS,EACT,yDAAyD,EACzD,KAAK,CACL;KACA,MAAM,CAAC,SAAS,EAAE,4CAA4C,EAAE,KAAK,CAAC;KACtE,MAAM,CACN,mBAAmB,EACnB,2DAA2D,CAC3D;KACA,MAAM,CACN,eAAe,EACf,qDAAqD,EACrD,KAAK,CACL;KACA,MAAM,CACN,KAAK,EACJ,SAAmB,EACnB,IAiBC,EACA,EAAE;IACH,wCAAwC;IACxC,MAAM,aAAa,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAE9D,0DAA0D;IAC1D,IAAI,UAAoB,CAAA;IACxB,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC;SAAM,IAAI,aAAa,EAAE,MAAM,EAAE,CAAC;QAClC,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,CAAC;SAAM,CAAC;QACP,OAAO,CAAC,KAAK,CACZ,4EAA4E,CAC5E,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAChB,CAAC;IAED,gEAAgE;IAChE,MAAM,MAAM,GAAc;QACzB,UAAU;QACV,UAAU,EAAE,IAAI,CAAC,IAAI;QACrB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,aAAa,EAAE,QAAQ;QAChD,QAAQ,EAAE,aAAa,CACtB,IAAI,CAAC,QAAQ,IAAI,aAAa,EAAE,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAC7D;QACD,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;QAC1D,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,aAAa,EAAE,MAAM,IAAI,QAAQ,CAAC,MAAM;QAC/D,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACtB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;YAC7B,CAAC,CAAC,CAAC,aAAa,EAAE,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC;QACjD,OAAO,EAAE,IAAI,CAAC,OAAO;YACpB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC,aAAa,EAAE,OAAO,IAAI,QAAQ,CAAC,OAAO,CAAC;QAC/C,QAAQ,EAAE,aAAa,EAAE,QAAQ;YAChC,CAAC,CAAC,EAAE,GAAG,aAAa,CAAC,QAAQ,EAAE;YAC/B,CAAC,CAAC,EAAE,GAAG,QAAQ,CAAC,QAAQ,EAAE;QAC3B,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,aAAa,EAAE,KAAK,IAAI,QAAQ,CAAC,KAAK;QAC3D,QAAQ,EAAE,aAAa,CACtB,IAAI,CAAC,QAAQ;YACZ,aAAa,EAAE,QAAQ;YACvB,QAAQ,CAAC,QAAQ,CAClB;QACD,UAAU,EACT,IAAI,CAAC,UAAU,IAAI,aAAa,EAAE,YAAY;QAC/C,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO,CAAC;KACvD,CAAA;IAED,iDAAiD;IACjD,WAAW,CAAC;QACX,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK,EAAE,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;KACpC,CAAC,CAAA;IAEF,4CAA4C;IAC5C,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAE3D,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAChB,CAAC;IAED,4CAA4C;IAC5C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACrB,MAAM,cAAc,CAAC,aAAa,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAA;QAC1D,OAAM;IACP,CAAC;IAED,MAAM,UAAU,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;AACxC,CAAC,CACD,CAAA;AAEF,SAAS,aAAa,CAAC,KAAa;IACnC,IACC,KAAK,KAAK,YAAY;QACtB,KAAK,KAAK,QAAQ;QAClB,KAAK,KAAK,QAAQ;QAClB,KAAK,KAAK,QAAQ,EACjB,CAAC;QACF,OAAO,KAAK,CAAA;IACb,CAAC;IACD,OAAO,CAAC,KAAK,CACZ,qBAAqB,KAAK,mDAAmD,CAC7E,CAAA;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAChB,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IACnC,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,MAAM,EAAE,CAAC;QAC7D,OAAO,KAAK,CAAA;IACb,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,qBAAqB,KAAK,gCAAgC,CAAC,CAAA;IACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IAClC,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QAC3C,OAAO,KAAK,CAAA;IACb,CAAC;IACD,OAAO,CAAC,KAAK,CACZ,6BAA6B,KAAK,+BAA+B,CACjE,CAAA;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAChB,CAAC;AAED,qDAAqD;AACrD,KAAK,UAAU,YAAY,CAAC,QAAkB;IAC7C,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,sDAAsD;QACtD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACnB,SAAQ;QACT,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,IAAI,OAAO,CAAW,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YACxD,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;gBAC7B,IAAI,GAAG;oBAAE,GAAG,CAAC,GAAG,CAAC,CAAA;;oBACZ,GAAG,CAAC,MAAM,CAAC,CAAA;YACjB,CAAC,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAA;IACvB,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,OAAO,CAAC,KAAK,EAAE,CAAA"}
|
package/dist/cli/run.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/cli/run.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/cli/run.ts"],"names":[],"mappings":"AAiCA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AA0C5C,6DAA6D;AAC7D,wBAAsB,cAAc,CACnC,UAAU,EAAE,MAAM,EAAE,EACpB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC,CA4Cf;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAC/B,MAAM,EAAE,SAAS,EACjB,aAAa,EAAE,MAAM,EAAE,GACrB,OAAO,CAAC,IAAI,CAAC,CAmYf"}
|