@e2edev/e2e 0.7.0 → 0.9.0

package/skills/e2e/references/agent.md

@@ -51,17 +51,20 @@ E2E_MODEL=anthropic/claude-sonnet-4.5 E2E_MODEL_API_KEY=... npx --no-install e2e
 import { credentials } from '@e2edev/e2e';
 
 await agent.act('add a todo named "Buy milk" and mark it done');
-await agent.act('invite {email} as an editor', { email: 'ada@example.test' });
+await agent.act('invite {email} as an editor', { params: { email: 'ada@example.test' } });
 
 const member = credentials.user('member');
 await agent.act('sign in with the given credentials', {
-  username: member.username,
-  password: member.password, // a Secret: the model sees its name, the runner fills the field
+  params: {
+    username: member.username,
+    password: member.password, // a Secret: the model sees its name, the runner fills the field
+  },
 });
 ```
 
-`act(instruction, params?, options?)` plans and performs a multi-action flow
-and ends in a verdict. Passed returns `{ ok: true }`. Failed or blocked
+`act(instruction, options?)` plans and performs a multi-action flow and ends
+in a verdict. Passed resolves with what the step did: `summary`, `modelCalls`,
+`actions`, and `cache` (how the trace cache took part). Failed or blocked
 throws an `AgentError` whose `code` says why: `ACTION_FAILED` for a plain
 failure, `STEP_BUDGET_EXHAUSTED` or `STEP_TIMEOUT` when the budget or the
 clock ran out, and a blocked code (`AUTH_CREDENTIAL_UNAVAILABLE`,
@@ -69,10 +72,11 @@ clock ran out, and a blocked code (`AUTH_CREDENTIAL_UNAVAILABLE`,
 `AUTOMATION_UNSUPPORTED`) when something outside the product prevented a
 verdict.
 
-Options: `timeout` (default the test timeout), `maxSteps` (default 25
-actions), `maxModelCalls` (default 25). Per-call values can only lower the
-configured limits. `schema` and `vision` on `act` are not implemented and
-reject with `UNSUPPORTED_CAPABILITY`.
+Options: `params` (the values the instruction refers to; a `Secret` is filled
+by the runner), `timeout` (default the test timeout), `maxSteps` (default 25
+actions), `maxModelCalls` (default 25). Per-call budgets can only lower the
+configured limits. `act` takes no `schema` and no `vision`: structured output
+is `extract({ schema })`, and `vision` belongs to the judgments.
 
 ## assert, waitFor, extract: one question
 
@@ -82,7 +86,7 @@ import { z } from 'zod';
 await agent.assert('the dashboard shows a trial badge'); // one look, one judgment
 
 await agent.waitFor('the export finished and a download link appeared', { // polls
-  intervalMs: 500,
+  interval: 500,
   timeout: 120_000,
 });
 
@@ -95,7 +99,7 @@ expect(data.titles).toContain('Buy milk');
 - `assert` does not poll. A false judgment is `ASSERTION_FAILED` with the
   model's explanation and a screenshot in the report. Malformed output gets
   one repair round, then `MODEL_OUTPUT_INVALID`.
-- `waitFor` observes every `intervalMs` (default 3 s) and spends a judgment
+- `waitFor` observes every `interval` (default 3 s) and spends a judgment
   only when the screen changed; `STEP_TIMEOUT` after `timeout` (default
   30 s).
 - `extract` accepts any Standard Schema validator (zod works). Invalid output
@@ -136,7 +140,11 @@ the end state may not), and it is what lets the trace cache record the step.
 ## What the model sees
 
 A redacted snapshot of the screen (roles, names, text, states), a summary of
-prior steps, and your context. Never raw HTML, cookies, headers, environment
+prior steps, and your context. The first screen of a step arrives whole;
+every action result after it reports what changed, keyed by node ids that
+stay stable while an element exists, or the whole screen again when most of
+it changed, and is read after the action's effect landed. Never raw HTML,
+cookies, headers, environment
 values, or a `Secret`'s value; password fields arrive masked. Pixels only
 with `vision`, and only while no secret has been filled. Nothing the model
 returns runs as code or selectors: the runner validates and authorizes every

package/skills/e2e/references/debugging.md

@@ -16,7 +16,8 @@ jq '.run.results[] | select(.status != "passed") | .attempts[-1]
 
 3. Artifacts named there live under `.e2e/artifacts/`: screenshots, a
    Playwright `trace.zip` per attempt (`npx playwright show-trace <file>`),
-   downloads, and with `--debug` the transcript of every agent step.
+   downloads, with `--video` a `video/video.webm` per attempt, and with
+   `--debug` the transcript of every agent step.
 
 ## Error codes and what to do
 
@@ -54,6 +55,7 @@ jq '.run.results[] | select(.status != "passed") | .attempts[-1]
 | `--no-cache` | Rule out a stale `agent.act` replay |
 | `--debug` | Read each agent step's duration, model calls, cost, and transcript |
 | `--ai-trace`, then `npx unbox-ai runs .e2e/ai-trace.json` | See exactly what the model was shown and called |
+| `--video` | Watch the failed attempt; `step.startedAt` minus the video artifact's `startedAt` is the step's offset into it |
 | `command.log: '.e2e/logs/app.log'` | Read the app's own output when it never becomes ready or errors mid-test |
 | `await app.screenshot('before-submit')` | Attach evidence at a chosen point |
 | `CI=1 npx --no-install e2e run` | Reproduce CI-only behaviour: `ONLY_IN_CI`, read-only cache, `reuseExisting` ignored |

package/skills/e2e/references/running.md

@@ -27,6 +27,7 @@ npx --no-install e2e telemetry [disable|enable] # anonymous usage telemetry: sta
 | `--pass-with-no-tests` | Exit 0 when nothing matches instead of `NO_TESTS`. |
 | `--debug` | Phase timings and an agent step table on stderr; step transcripts saved as artifacts. |
 | `--ai-trace` | Record every model call to `.e2e/ai-trace.json`. |
+| `--video` | Record every attempt (WebM on a browser engine, MP4 on a device engine) under its artifact directory; the failure recap names the file. Fails with `UNSUPPORTED_ARTIFACT` when the engine cannot record. |
 
 ```bash
 npx --no-install e2e run tests/signup.e2e.ts
@@ -77,9 +78,9 @@ recorded flow is stale — `--no-cache` only skips the cache for one run.
 - `junit`: `.e2e/junit.xml` beside the report, for CI test summaries.
   Combine it with the terminal output: `--reporter list,junit`.
 - `json`: the report document on stdout.
-- Artifacts (screenshots, Playwright traces, `--debug` transcripts,
-  downloads) live under `.e2e/artifacts/`; every path is recorded in the
-  report.
+- Artifacts (screenshots, Playwright traces, `--video` recordings, `--debug`
+  transcripts, downloads) live under `.e2e/artifacts/`; every path is
+  recorded in the report.
 
 ## Exit codes
 

package/skills/e2e/references/setup.md

@@ -62,8 +62,6 @@ export default {
   tests: 'tests/**/*.e2e.ts',
   targets: [
     {
-      name: 'web',
-      platform: 'web',
       engine: playwright({
         url: 'http://127.0.0.1:3000',
         command: { executable: 'pnpm', args: ['dev'], log: '.e2e/logs/app.log' },
@@ -80,19 +78,19 @@ export default {
 
 | Key | Default | Notes |
 | --- | --- | --- |
-| `targets` | required | Non-empty. Each target: `name` (used by `--target` and in reports), `platform` (`web`, `ios`, `android`, or any label), `engine`. |
+| `targets` | required | Non-empty. Each target: `platform` (`web`, `ios`, `android`, or any label), `engine`, and an optional `name` (defaults to the platform; used by `--target` and in reports). |
 | `tests` | `'tests/**/*.e2e.ts'` | A glob or an array of globs, `/` separators. |
 | `timeout` | `120000` | Per test attempt, in ms. Also the default `agent.act` deadline. |
 | `actionTimeout` | `30000` | Every locator action and engine operation, including each observation inside an agent step. Raise it for slow model providers. |
 | `assertionTimeout` | `5000` | `expect` polling window. |
 | `retries` | `0`, `1` in CI | 0 to 10. |
-| `workers` | half the cores, `1` in CI | Test files run in parallel across workers. Use `1` for device targets. |
-| `reporters` | `['list']` | `list`, `json`, `junit`. `json` excludes `list`. |
+| `workers` | half the cores, `1` in CI | Test files run in parallel across workers, at most the `workers` the engine declares per target (a device target: one per device). |
+| `reporters` | `['list']` | `list`, `json`, `junit`, and reporter objects (`{ name, onEvent?, onRunFinished? }`) that receive the finished run. `json` excludes `list`; `--reporter` keeps the objects. |
 | `cache` | `'read-write'`, `'read-only'` in CI | The trace cache for `agent.act`; `'off'` disables it. |
 | `agent` | unset | `createAgent(...)`, an options block `{ model, context, visionModel, maxSteps, maxModelCalls, vision, providerOptions }`, or a custom `StepExecutor`. Omitted, the built-in agent runs with `E2E_MODEL`. A model passed to `createAgent({ model })` is the one model for `act` and the judgments, over `E2E_MODEL`. |
 | `credentials` | `{}` | Named `{ username, password, allowedOrigins? }` entries; `password` may be a function returning the value. |
 | `screen.testIdAttribute` | `'data-testid'` | Attribute read by `getByTestId`. |
-| `artifacts` | `['screenshot', 'trace']` | Kinds to keep, or `{ kinds, store }`. |
+| `artifacts` | `['screenshot', 'trace']` | Kinds to keep (`screenshot`, `trace`, and the opt-in `video`), or `{ kinds, store, video }`; `video: { retain: 'on-failure' }` keeps only the recordings of attempts that did not pass. |
 | `projectId` | the package name | Report and cache identity. |
 
 ## The app under test
@@ -119,8 +117,8 @@ Two browsers are two targets sharing one app declaration:
 const app = { url: 'http://127.0.0.1:3000' };
 export default {
   targets: [
-    { name: 'chromium', platform: 'web', engine: playwright(app) },
-    { name: 'mobile-webkit', platform: 'web', engine: playwright({ ...app, browser: 'webkit', viewport: { width: 390, height: 844 } }) },
+    { name: 'chromium', engine: playwright(app) },
+    { name: 'mobile-webkit', engine: playwright({ ...app, browser: 'webkit', viewport: { width: 390, height: 844 } }) },
   ],
 } satisfies E2EConfig;
 ```
@@ -209,7 +207,7 @@ import { agentDeviceTools } from '@e2edev/agent-device/tools';
 const iphone = agentDevice({ platform: 'ios', app: 'com.example.app' });
 
 export default {
-  targets: [{ name: 'ios', platform: 'ios', engine: iphone }],
+  targets: [{ engine: iphone }],
   workers: 1,
   agent: createAgent({ tools: agentDeviceTools(iphone) }),
 } satisfies E2EConfig;
@@ -218,7 +216,12 @@ export default {
 - `app` is a bundle id, package name, or display name opened fresh per
   attempt. `appPath` installs a `.app` or `.apk` once per worker; without
   `app`, the installed bundle is the one opened.
-- `workers: 1`: workers share one simulator.
+- One worker per device. With no `device`, every booted simulator or
+  emulator of the platform is the pool, as many as `workers` allows, so
+  booting four simulators runs the files four at a time with no config. A
+  single `device` runs one worker whatever `workers` says; a list,
+  `device: ['iPhone 17', 'iPhone 17 Pro']`, is an explicit pool. Devices boot
+  in `prepare`, before the run's clock starts.
 - Cancelled device commands keep running; the next attempt waits for them.
   Raw screenshot files are removed when capture finishes, including after
   cancellation.

package/skills/e2e/references/writing-tests.md

@@ -93,7 +93,6 @@ Fixtures are lazy; destructure them in the callback.
 | `back()` | One history step back. |
 | `restart()` | Recreates the context and keeps persisted state, including a restored session. |
 | `clearState()` | Clears cookies and storage, then relaunches. Not inside a serial group. |
-| `deepLink(url)` | Opens an allowed deep or universal link. |
 | `screenshot(label?)` | Saves a redacted screenshot as an artifact and returns its path. |
 
 ## Locators
@@ -104,7 +103,7 @@ subtree.
 
 | Query | Matches |
 | --- | --- |
-| `getByRole(role, { name?, exact?, checked?, disabled?, selected?, expanded?, hidden?, visible? })` | Semantic role, optionally by accessible name and state. First choice. |
+| `getByRole(role, { name?, exact?, checked?, disabled?, selected?, expanded?, visible? })` | Semantic role, optionally by accessible name and state. First choice. |
 | `getByLabel(text, { exact?, visible? })` | Form controls by label. |
 | `getByPlaceholder(text)` | Inputs by placeholder. |
 | `getByText(text, { exact?, visible? })` | Visible text. |
@@ -118,7 +117,9 @@ Roles: `button`, `link`, `textbox`, `searchbox`, `combobox`, `listbox`,
 `contentinfo`, `complementary`, `region`. The union is closed; anything else
 is a type error.
 
-Text matching is exact by default after whitespace normalization.
+Text matching is exact by default after whitespace normalization, and
+`getByText` returns the innermost match: a container that echoes its child's
+text (an iOS Text host view around its StaticText) does not count twice.
 `exact: false` is a case-insensitive substring match; a `RegExp` matches as
 written.
 
@@ -146,9 +147,9 @@ await screen.scrollUntilVisible(screen.getByRole('button', { name: 'Accept' }));
 Each action resolves one node, waits for it to be actionable within
 `config.actionTimeout` (30 s, or `{ timeout }`), and performs one operation.
 
-`tap()` (alias `click()`), `doubleTap()`, `longPress({ durationMs? })`,
+`tap()` (alias `click()`), `doubleTap()`, `longPress({ duration? })`,
 `fill(value | Secret)`, `clear()`, `press(key)`, `check()`, `uncheck()`,
-`selectOption(label | { label } | { index })`, `focus()`, `hover()`,
+`selectOption(label | { label } | { value } | { index })`, `focus()`, `hover()`,
 `setInputFiles(paths)` (relative to the project root), `dragTo(locator)`,
 `scrollIntoView()`, `swipe({ direction, momentum? })`.
 

package/dist/internal/trace.d.ts

@@ -1,18 +0,0 @@
-/**
- * Live agent tracing for iterating on agentic tests: `E2E_DEBUG=agent` streams
- * every invocation phase, model selection, and locate-candidate outcome to
- * stderr as it happens, from whichever process runs the attempt. This is a
- * debugging aid, not telemetry: the report stays the canonical record.
- */
-/**
- * Writes one bounded, sanitized trace line when agent tracing is on. Takes a
- * thunk so call sites in polling loops pay nothing while tracing is off.
- */
-export declare function agentTrace(message: () => string): void;
-/**
- * Dumps one full observation as the model will receive it. The text is
- * already redacted and size-bounded by the observation pipeline, so the dump
- * needs no further truncation to stay safe.
- */
-export declare function observationTrace(header: () => string, text: string): void;
-//# sourceMappingURL=trace.d.ts.map

package/dist/internal/trace.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"trace.d.ts","sourceRoot":"","sources":["../../src/internal/trace.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA6BH;;;GAGG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,MAAM,GAAG,IAAI,CAGtD;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAGzE"}

package/dist/internal/trace.js

@@ -1,46 +0,0 @@
-/**
- * Live agent tracing for iterating on agentic tests: `E2E_DEBUG=agent` streams
- * every invocation phase, model selection, and locate-candidate outcome to
- * stderr as it happens, from whichever process runs the attempt. This is a
- * debugging aid, not telemetry: the report stays the canonical record.
- */
-import { sanitizeText, truncateUtf8 } from './errors.js';
-/**
- * Parsed on first use, not at import, so tests and embedders can set
- * `E2E_DEBUG` after this module loads but before the first trace call.
- */
-let cachedFlags;
-function flags() {
-    cachedFlags ??= new Set((process.env['E2E_DEBUG'] ?? '')
-        .split(',')
-        .map((flag) => flag.trim())
-        .filter((flag) => flag !== ''));
-    return cachedFlags;
-}
-function agentTraceEnabled() {
-    return flags().has('agent') || flags().has('all');
-}
-/** `E2E_DEBUG=observations` additionally dumps every observation the model sees. */
-function observationTraceEnabled() {
-    return flags().has('observations') || flags().has('all');
-}
-/**
- * Writes one bounded, sanitized trace line when agent tracing is on. Takes a
- * thunk so call sites in polling loops pay nothing while tracing is off.
- */
-export function agentTrace(message) {
-    if (!agentTraceEnabled())
-        return;
-    process.stderr.write(`[e2e agent] ${truncateUtf8(sanitizeText(message()), 2_000)}\n`);
-}
-/**
- * Dumps one full observation as the model will receive it. The text is
- * already redacted and size-bounded by the observation pipeline, so the dump
- * needs no further truncation to stay safe.
- */
-export function observationTrace(header, text) {
-    if (!observationTraceEnabled())
-        return;
-    process.stderr.write(`[e2e observation] ${header()}\n${text}\n[e2e observation] end\n`);
-}
-//# sourceMappingURL=trace.js.map

package/dist/internal/trace.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"trace.js","sourceRoot":"","sources":["../../src/internal/trace.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEzD;;;GAGG;AACH,IAAI,WAA4C,CAAC;AAEjD,SAAS,KAAK;IACZ,WAAW,KAAK,IAAI,GAAG,CACrB,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;SAC7B,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CACjC,CAAC;IACF,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,iBAAiB;IACxB,OAAO,KAAK,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AACpD,CAAC;AAED,oFAAoF;AACpF,SAAS,uBAAuB;IAC9B,OAAO,KAAK,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,KAAK,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAC3D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,OAAqB;IAC9C,IAAI,CAAC,iBAAiB,EAAE;QAAE,OAAO;IACjC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,YAAY,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;AACxF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAoB,EAAE,IAAY;IACjE,IAAI,CAAC,uBAAuB,EAAE;QAAE,OAAO;IACvC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,MAAM,EAAE,KAAK,IAAI,2BAA2B,CAAC,CAAC;AAC1F,CAAC"}

package/dist/run/public.d.ts

@@ -1,19 +0,0 @@
-/**
- * Programmatic host surface (`@e2edev/e2e/run`): run the runner in-process and stream
- * structured events, without shelling out to the CLI. This is how an
- * embedding host — a hosted platform, a CI wrapper, an IDE — drives runs.
- *
- * The CLI is a thin consumer of exactly this surface; nothing here is
- * CLI-only. The surface is host tooling, not part of sdk-0.1: it can grow in
- * minors, and `report.json` (report-1) stays the canonical record of a run.
- */
-export { run } from './runner.ts';
-export type { RunOptions, RunOutcome } from './runner.ts';
-export type { RunEvent, RunEventFact, RunEventHeader, RunEventOf, RunEventResult, RunEventSink, RunExitCode, RunStatus, SetupStep, } from './events.ts';
-export type { ArtifactProducer, ArtifactRecord, AttemptRecord, ResultRecord, ResultStatus, RunError, SerialAttemptRecord, SerialGroupRecord, SerialMemberRecord, } from './records.ts';
-export type { StepAgentDetails, StepCacheInfo, StepEvent, StepKind, StepMetrics, StepModelInfo, StepProgress, StepRecord, } from './steps.ts';
-export type { Report1Document } from '../report/build.ts';
-export type { ErrorCategory, ErrorPhase, SerializedError } from '../internal/errors.ts';
-export type { TestIdentity } from '../collect/collect.ts';
-export type { SkipInfo } from '../collect/select.ts';
-//# sourceMappingURL=public.d.ts.map

package/dist/run/public.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"public.d.ts","sourceRoot":"","sources":["../../src/run/public.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAClC,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC1D,YAAY,EACV,QAAQ,EACR,YAAY,EACZ,cAAc,EACd,UAAU,EACV,cAAc,EACd,YAAY,EACZ,WAAW,EACX,SAAS,EACT,SAAS,GACV,MAAM,aAAa,CAAC;AACrB,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,QAAQ,EACR,mBAAmB,EACnB,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AACtB,YAAY,EACV,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,QAAQ,EACR,WAAW,EACX,aAAa,EACb,YAAY,EACZ,UAAU,GACX,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAI1D,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxF,YAAY,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAC1D,YAAY,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC"}

package/dist/run/public.js

@@ -1,11 +0,0 @@
-/**
- * Programmatic host surface (`@e2edev/e2e/run`): run the runner in-process and stream
- * structured events, without shelling out to the CLI. This is how an
- * embedding host — a hosted platform, a CI wrapper, an IDE — drives runs.
- *
- * The CLI is a thin consumer of exactly this surface; nothing here is
- * CLI-only. The surface is host tooling, not part of sdk-0.1: it can grow in
- * minors, and `report.json` (report-1) stays the canonical record of a run.
- */
-export { run } from './runner.js';
-//# sourceMappingURL=public.js.map

package/dist/run/public.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"public.js","sourceRoot":"","sources":["../../src/run/public.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC"}