@e2edev/e2e 0.13.0 → 0.14.0

package/skills/e2e/references/setup.md

@@ -14,6 +14,10 @@
   (npm's `ERESOLVE`); upgrade `playwright` within the range.
   Missing browsers download when the engine first boots. In CI install them
   up front: `npx playwright install chromium --with-deps`.
+- For mobile tests, `@e2edev/agent-device` plus `agent-device` (`0.21.x`),
+  likewise a peer dependency: a project that already drives devices with the
+  agent-device CLI keeps its version. The range pins the tested minor because
+  agent-device minors break; it moves with each engine release.
 
 ## Scaffold
 
@@ -73,7 +77,7 @@ export default {
       }),
     },
   ],
-  // Only for agent.* steps. The model is an AI SDK instance; gateway() from 'ai' reads AI_GATEWAY_API_KEY.
+  // The model behind every agent.* step: an AI SDK instance; gateway() from 'ai' reads AI_GATEWAY_API_KEY.
   agents: {
     default: createAgent({
       model: gateway('openai/gpt-5.6-luna'),
@@ -95,10 +99,11 @@ export default {
 | `assertionTimeout` | `5000` | `expect` polling window. |
 | `retries` | `0`, `1` in CI | 0 to 10. |
 | `workers` | half the cores, `1` in CI | Test files run in parallel across workers, at most the `workers` the engine declares per target (a device target: one per device). |
-| `reporters` | `['list']` | `list`, `json`, `junit`, and reporter objects (`{ name, onEvent?, onRunFinished? }`) that receive the finished run. `json` excludes `list`; `--reporter` keeps the objects. |
+| `reporters` | `['list']` | `list`, `json`, `junit`, `markdown`, and reporter objects (`{ name, onEvent?, onRunFinished? }`) that receive the finished run. `json` excludes `list`; `--reporter` keeps the objects. |
 | `cache` | `'read-write'`, `'read-only'` in CI | The trace cache for `agent.act`; `'off'` disables it. |
 | `agents` | `{ default: built-in }` | Agents by name. `default` is what tests run with; `e2e run --agent <name>` runs with another. Each entry is `createAgent(...)`, an options block `{ model, context, maxSteps, maxModelCalls, vision, providerOptions }`, or a custom `StepExecutor`. `model` is an AI SDK instance; without one anywhere, acquiring `agent` is `MODEL_UNAVAILABLE`. |
 | `credentials` | `{}` | Named `{ username, password, allowedOrigins? }` entries; `password` may be a function returning the value. |
+| `secrets` | `{}` | Named values the model never sees (API keys, tokens): a string, a function returning the value, or `{ value, allowedOrigins? }`. A name cannot also be a credential. |
 | `screen.testIdAttribute` | `'data-testid'` | Attribute read by `getByTestId`. |
 | `artifacts` | `['screenshot', 'trace']` | Kinds to keep (`screenshot`, `trace`, and the opt-in `video`), or `{ kinds, store, video }`; `video: { retain: 'on-failure' }` keeps only the recordings of attempts that did not pass. |
 | `projectId` | the package name | Report and cache identity. |
@@ -206,6 +211,7 @@ or read the address from the environment:
 | --- | --- |
 | `AI_GATEWAY_API_KEY`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, ... | Read by the provider package the config constructs the model with (`gateway()` from `ai`, `openrouter()`, `openai()`), not by the runner. The runner reads no model variable; the model is always an AI SDK instance in the config. |
 | `E2E_USER_<NAME>_USERNAME`, `E2E_USER_<NAME>_PASSWORD` | Override `credentials.<name>`. `<NAME>` is the credential name uppercased, other characters as `_`. |
+| `E2E_SECRET_<NAME>` | Overrides the value of `secrets.<name>`, same uppercasing rule. |
 | `CI` | Turns on CI defaults: `retries: 1`, `workers: 1`, `test.only` rejected, cache read-only, `reuseExisting` ignored. |
 | `E2E_TELEMETRY_DISABLED`, `DO_NOT_TRACK` | Turn anonymous usage telemetry off; `e2e telemetry disable` saves the same choice. `E2E_TELEMETRY_DEBUG=1` prints each event instead of sending it. |
 
@@ -240,6 +246,11 @@ export default {
   single `device` runs one worker whatever `workers` says; a list,
   `device: ['iPhone 17', 'iPhone 17 Pro']`, is an explicit pool. Devices boot
   in `prepare`, before the run's clock starts.
+- A test's steps never wait for the screen to settle; `expect` verifies the
+  outcome. Only a control that appeared or moved with the previous action
+  waits out the `transition` budget (default 500 ms) before it is acted on.
+  Agent actions settle for `settle` ms (default 150) before the agent
+  observes again; `settle: false` skips that wait.
 - Cancelled device commands keep running; the next attempt waits for them.
   Raw screenshot files are removed when capture finishes, including after
   cancellation.
@@ -260,5 +271,7 @@ export default {
 - `npx e2e run tests/example.e2e.ts` passes against the app.
 - `package.json` has a script such as `"test:e2e": "e2e run"`.
 - `.gitignore` lists the `.e2e/` outputs (init adds them). Committing
-  `.e2e/cache/` is opt-in: remove that line to share `agent.act` replays.
-- CI runs the deterministic suite on pull requests; see `running`.
+  `.e2e/cache/` is opt-in: remove that line so CI and teammates replay
+  `agent.act` steps instead of re-running the model.
+- CI runs the whole suite on pull requests, agent steps included; see
+  `running`.

package/skills/e2e/references/writing-tests.md

@@ -12,28 +12,31 @@ test.describe('todos', { tags: ['todos'] }, () => {
     await app.open('/todos');
   });
 
-  test('adds and completes a todo', async ({ screen, web }) => {
-    await screen.getByLabel('New todo').fill('Write the release notes');
-    await screen.getByRole('button', { name: 'Add' }).tap();
-
+  test('adds and completes a todo', async ({ agent, screen, web }) => {
+    await agent.act('add a todo named {title}', { params: { title: 'Write the release notes' } });
     await expect(screen.getByRole('listitem')).toHaveCount(1);
     await expect(screen.getByRole('status', { name: 'Remaining' })).toHaveText('1 remaining');
 
-    await screen.getByRole('checkbox', { name: 'Write the release notes' }).check();
+    await agent.act('mark the todo as done');
     await expect(screen.getByRole('status', { name: 'Remaining' })).toHaveText('0 remaining');
     await expect(web).toHaveURL('/todos');
   });
 
   test('ignores an empty submission', async ({ screen }) => {
+    // An exact interaction: the empty submit is the point of the test.
     await screen.getByRole('button', { name: 'Add' }).tap();
     await expect(screen.getByRole('listitem')).toHaveCount(0);
   });
 });
 ```
 
-Files match the config `tests` glob, default `tests/**/*.e2e.ts`. Every test
-starts from clean state: a fresh browser context and no page open, so a test
-calls `app.open()` first (here in `beforeEach`).
+The agent does the flow; `expect` pins what must be true after each goal,
+and that check is what lets the trace cache replay the step on later runs.
+`screen` actions are for exact interactions and values, like the empty
+submit above or a sign-in form. Files match the config `tests` glob, default
+`tests/**/*.e2e.ts`. Every test starts from clean state: a fresh browser
+context and no page open, so a test calls `app.open()` first (here in
+`beforeEach`).
 
 ## Registration
 
@@ -237,7 +240,12 @@ credentials: {
 
 `E2E_USER_ADMIN_USERNAME` and `E2E_USER_ADMIN_PASSWORD` override either
 field per run. `credentials.user('admin').password` is a `Secret` with no
-plaintext accessor; only `fill()` and `agent.act` params accept it. Once a
+plaintext accessor; only `fill()` and `agent.act` params accept it. Any other
+sensitive value (an API key, a token) is a `secrets` entry,
+`secrets: { 'stripe-key': process.env.STRIPE_KEY ?? '' }`, overridable with
+`E2E_SECRET_STRIPE_KEY`; `secrets.get('stripe-key')` is the same kind of
+handle and fills any editable input, with the value redacted by name
+everywhere the runner writes. Once a
 secret is filled, model pixels and assertion screenshots are withheld for
 the rest of that session, including later tests sharing a serial session.
 `app.screenshot()` fails with `POLICY_DENIED` before capture. Sign in inside
@@ -282,8 +290,9 @@ expect(response.status).toBe(201);
   Otherwise tests are independent and may run on different workers.
 - Tag by area and by cost (`smoke`, `billing`, `agent`) and run subsets with
   `--tag`.
-- Mix agent steps in where the path varies and pin the outcome with
-  `expect`; see the `agent` topic.
+- Drive flows with `agent.act` and pin each outcome with `expect`. The
+  calls on this page are for exact values and exact checks; the `agent`
+  topic covers the steps that do the work.
 
 ## Mistakes to avoid
 

package/dist/credentials.d.ts

@@ -1,13 +0,0 @@
-/** Opaque credential handles. */
-import type { ResolvedCredential } from './config/resolve.ts';
-import type { Credentials } from './types.ts';
-/** Installed by the runner once config resolution completes. */
-export declare function setCredentialRegistry(map: ReadonlyMap<string, ResolvedCredential> | undefined): void;
-/**
- * Removes `map` from the slot if it is still the installed registry. A host
- * that closes one attempt while opening the next must not wipe what the new
- * attempt just installed.
- */
-export declare function releaseCredentialRegistry(map: ReadonlyMap<string, ResolvedCredential>): void;
-export declare const credentials: Credentials;
-//# sourceMappingURL=credentials.d.ts.map

package/dist/credentials.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"credentials.d.ts","sourceRoot":"","sources":["../src/credentials.ts"],"names":[],"mappings":"AAAA,iCAAiC;AAKjC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,KAAK,EAAc,WAAW,EAAU,MAAM,YAAY,CAAC;AAKlE,gEAAgE;AAChE,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,GAAG,SAAS,GAAG,IAAI,CAGpG;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,GAAG,EAAE,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,GAAG,IAAI,CAE5F;AAcD,eAAO,MAAM,WAAW,EAAE,WAuBzB,CAAC"}

package/dist/credentials.js

@@ -1,51 +0,0 @@
-/** Opaque credential handles. */
-import { credentialBrand, secretBrand } from './internal/brands.js';
-import { ConfigurationError } from './internal/errors.js';
-import { realmSlot } from './internal/realm-slot.js';
-/** Global slot so test modules in an isolated realm reach the runner's registry. */
-const credentialsSlot = realmSlot('e2e.credentials.v1');
-/** Installed by the runner once config resolution completes. */
-export function setCredentialRegistry(map) {
-    if (map === undefined)
-        credentialsSlot.delete(globalThis);
-    else
-        credentialsSlot.set(globalThis, map);
-}
-/**
- * Removes `map` from the slot if it is still the installed registry. A host
- * that closes one attempt while opening the next must not wipe what the new
- * attempt just installed.
- */
-export function releaseCredentialRegistry(map) {
-    if (credentialsSlot.get(globalThis) === map)
-        credentialsSlot.delete(globalThis);
-}
-function getRegistry() {
-    return credentialsSlot.get(globalThis);
-}
-function makeSecret(name) {
-    return Object.freeze({
-        name,
-        purpose: 'password',
-        [secretBrand]: true,
-    });
-}
-export const credentials = {
-    user(name) {
-        const registry = getRegistry();
-        if (registry === undefined) {
-            throw new ConfigurationError('AUTH_CREDENTIAL_UNAVAILABLE', 'credentials.user() is only available while the e2e runner is active');
-        }
-        const resolved = registry.get(name);
-        if (resolved === undefined) {
-            throw new ConfigurationError('AUTH_CREDENTIAL_UNAVAILABLE', `credential "${name}" is not configured; add it to config.credentials or E2E_USER_* variables`);
-        }
-        return Object.freeze({
-            name,
-            username: resolved.username,
-            password: makeSecret(name),
-            [credentialBrand]: true,
-        });
-    },
-};
-//# sourceMappingURL=credentials.js.map

package/dist/credentials.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"credentials.js","sourceRoot":"","sources":["../src/credentials.ts"],"names":[],"mappings":"AAAA,iCAAiC;AAEjC,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAIrD,oFAAoF;AACpF,MAAM,eAAe,GAAG,SAAS,CAA0C,oBAAoB,CAAC,CAAC;AAEjG,gEAAgE;AAChE,MAAM,UAAU,qBAAqB,CAAC,GAAwD;IAC5F,IAAI,GAAG,KAAK,SAAS;QAAE,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;;QACrD,eAAe,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;AAC5C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,GAA4C;IACpF,IAAI,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,GAAG;QAAE,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;AAClF,CAAC;AAED,SAAS,WAAW;IAClB,OAAO,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,IAAI;QACJ,OAAO,EAAE,UAAmB;QAC5B,CAAC,WAAW,CAAC,EAAE,IAAa;KAC7B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAgB;IACtC,IAAI,CAAC,IAAY;QACf,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;QAC/B,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,IAAI,kBAAkB,CAC1B,6BAA6B,EAC7B,qEAAqE,CACtE,CAAC;QACJ,CAAC;QACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,IAAI,kBAAkB,CAC1B,6BAA6B,EAC7B,eAAe,IAAI,2EAA2E,CAC/F,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC;YACnB,IAAI;YACJ,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC;YAC1B,CAAC,eAAe,CAAC,EAAE,IAAa;SACjC,CAAC,CAAC;IACL,CAAC;CACF,CAAC"}