@e2edev/e2e 0.9.0 → 0.11.0

package/skills/e2e/references/setup.md

@@ -3,11 +3,10 @@
 ## Requirements
 
 - Node.js 22.12 or newer.
-- ES modules. The `package.json` nearest to the config and the tests must set
-  `"type": "module"`. An `e2e.config.mts` is ESM on its own, but `.ts` tests
-  still follow their package. In a CommonJS package either run
-  `npm pkg set type=module` (this also changes how its `.js` files run) or put
-  the tests in their own ESM package directory.
+- ES modules. e2e loads `.ts` config, tests, and helpers as ES modules
+  whatever the nearest `package.json` `type` says, so a CommonJS package (a
+  Next.js app, for instance) needs no change. Write them with `import`, never
+  `require` or `module.exports`.
 - For browser tests, `@e2edev/playwright`. Missing browsers download when the
   engine first boots. In CI install them up front:
   `npx playwright install chromium --with-deps`.
@@ -57,6 +56,7 @@ unknown keys are `INVALID_CONFIG`.
 import type { E2EConfig } from '@e2edev/e2e';
 import { createAgent } from '@e2edev/e2e/agent';
 import { playwright } from '@e2edev/playwright';
+import { gateway } from 'ai';
 
 export default {
   tests: 'tests/**/*.e2e.ts',
@@ -68,8 +68,13 @@ export default {
       }),
     },
   ],
-  // Only for agent.* steps. The model comes from E2E_MODEL unless set here.
-  agent: createAgent({ system: 'You are a thorough QA agent. Verify every outcome on screen.' }),
+  // Only for agent.* steps. The model is an AI SDK instance; gateway() from 'ai' reads AI_GATEWAY_API_KEY.
+  agents: {
+    default: createAgent({
+      model: gateway('openai/gpt-5.4-mini'),
+      system: 'You are a thorough QA agent. Verify every outcome on screen.',
+    }),
+  },
   credentials: {
     admin: { username: 'admin@example.test', password: process.env.ADMIN_PASSWORD ?? '' },
   },
@@ -87,7 +92,7 @@ export default {
 | `workers` | half the cores, `1` in CI | Test files run in parallel across workers, at most the `workers` the engine declares per target (a device target: one per device). |
 | `reporters` | `['list']` | `list`, `json`, `junit`, and reporter objects (`{ name, onEvent?, onRunFinished? }`) that receive the finished run. `json` excludes `list`; `--reporter` keeps the objects. |
 | `cache` | `'read-write'`, `'read-only'` in CI | The trace cache for `agent.act`; `'off'` disables it. |
-| `agent` | unset | `createAgent(...)`, an options block `{ model, context, visionModel, maxSteps, maxModelCalls, vision, providerOptions }`, or a custom `StepExecutor`. Omitted, the built-in agent runs with `E2E_MODEL`. A model passed to `createAgent({ model })` is the one model for `act` and the judgments, over `E2E_MODEL`. |
+| `agents` | `{ default: built-in }` | Agents by name. `default` is what tests run with; `e2e run --agent <name>` runs with another. Each entry is `createAgent(...)`, an options block `{ model, context, visionModel, maxSteps, maxModelCalls, vision, providerOptions }`, or a custom `StepExecutor`. `model` is an AI SDK instance; without one anywhere, acquiring `agent` is `MODEL_UNAVAILABLE`. |
 | `credentials` | `{}` | Named `{ username, password, allowedOrigins? }` entries; `password` may be a function returning the value. |
 | `screen.testIdAttribute` | `'data-testid'` | Attribute read by `getByTestId`. |
 | `artifacts` | `['screenshot', 'trace']` | Kinds to keep (`screenshot`, `trace`, and the opt-in `video`), or `{ kinds, store, video }`; `video: { retain: 'on-failure' }` keeps only the recordings of attempts that did not pass. |
@@ -110,6 +115,8 @@ process, and identity. `playwright()` accepts:
 | `browser` | `'chromium'` (default), `'firefox'`, `'webkit'`. |
 | `viewport` | `{ width, height }`, default 1280x720. |
 | `connect` | `{ cdpEndpoint }` to attach to a remote Chromium over CDP instead of launching. |
+| `headers` | Request headers sent to allowed origins only (a Vercel `x-vercel-protection-bypass`, ngrok's `ngrok-skip-browser-warning`). Reaches every path onto the page, `agent.act` included; turns the browser HTTP cache off and blocks service workers. |
+| `basicAuth` | `{ username, password }` answering a `401` challenge from an allowed origin; never sent to any other. |
 
 Two browsers are two targets sharing one app declaration:
 
@@ -183,10 +190,7 @@ or read the address from the environment:
 
 | Variable | Effect |
 | --- | --- |
-| `E2E_MODEL` | `provider/model-id` for `agent.*` steps when the config sets no model. Routed through the Vercel AI Gateway. |
-| `E2E_MODEL_API_KEY` | Credential for that model. `AI_GATEWAY_API_KEY` is the fallback. |
-| `E2E_MODEL_ENDPOINT` | Any OpenAI-compatible endpoint URL for that model; the gateway is the default. HTTPS unless loopback. `agent.model.endpoint` wins. |
-| `E2E_VISION_MODEL` | Model for calls with `vision`; falls back to the main model. |
+| `AI_GATEWAY_API_KEY`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, ... | Read by the provider package the config constructs the model with (`gateway()` from `ai`, `openrouter()`, `openai()`), not by the runner. The runner reads no model variable; the model is always an AI SDK instance in the config. |
 | `E2E_USER_<NAME>_USERNAME`, `E2E_USER_<NAME>_PASSWORD` | Override `credentials.<name>`. `<NAME>` is the credential name uppercased, other characters as `_`. |
 | `CI` | Turns on CI defaults: `retries: 1`, `workers: 1`, `test.only` rejected, cache read-only, `reuseExisting` ignored. |
 | `E2E_TELEMETRY_DISABLED`, `DO_NOT_TRACK` | Turn anonymous usage telemetry off; `e2e telemetry disable` saves the same choice. `E2E_TELEMETRY_DEBUG=1` prints each event instead of sending it. |
@@ -209,7 +213,7 @@ const iphone = agentDevice({ platform: 'ios', app: 'com.example.app' });
 export default {
   targets: [{ engine: iphone }],
   workers: 1,
-  agent: createAgent({ tools: agentDeviceTools(iphone) }),
+  agents: { default: createAgent({ tools: agentDeviceTools(iphone) }) },
 } satisfies E2EConfig;
 ```
 

package/skills/e2e/references/writing-tests.md

@@ -65,6 +65,7 @@ test.setup('sign in', { sessions: ['admin'] }, async ({ app, screen, session })
 | `requires` | `[]` | Capabilities the engine must contribute, e.g. `['web']`. Otherwise the test is skipped at selection instead of failing with `UNSUPPORTED_CAPABILITY`. |
 | `session` | unset | Restore state saved by a setup test. |
 | `agentContext` | unset | Extra context for `agent.*` calls in this test or group. |
+| `agent` | the run's agent | Pin the test or group to a configured agent (`agents.<name>`). Innermost wins; `agent.act(..., { agent })` can name another for one call. |
 | `serial` | `false` | Groups only. Members share one app state, run in order on one worker, and retry as a whole. Inside, per-member `retries`, `session`, `platforms`, `requires`, and `skip` are errors. |
 
 Hook order follows nesting, not position: outer `beforeEach` first, inner

package/dist/config/esm.d.ts

@@ -1,4 +0,0 @@
-/** Package-scope diagnostics for TypeScript config and test entry points. */
-/** Explains how to opt a `.ts` entry point's package into ESM when required. */
-export declare function esmPackageHint(modulePath: string): string | undefined;
-//# sourceMappingURL=esm.d.ts.map

package/dist/config/esm.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"esm.d.ts","sourceRoot":"","sources":["../../src/config/esm.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAK7E,gFAAgF;AAChF,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CA2BrE"}

package/dist/config/esm.js

@@ -1,35 +0,0 @@
-/** Package-scope diagnostics for TypeScript config and test entry points. */
-import { existsSync, readFileSync, realpathSync } from 'node:fs';
-import path from 'node:path';
-/** Explains how to opt a `.ts` entry point's package into ESM when required. */
-export function esmPackageHint(modulePath) {
-    const filename = existsSync(modulePath) ? realpathSync(modulePath) : modulePath;
-    if (path.extname(filename) !== '.ts')
-        return undefined;
-    let dir = path.dirname(filename);
-    for (;;) {
-        const manifestPath = path.join(dir, 'package.json');
-        if (existsSync(manifestPath)) {
-            let manifest;
-            try {
-                manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
-            }
-            catch {
-                // Leave malformed or unreadable manifests to the loader's own diagnostic.
-                return undefined;
-            }
-            if (typeof manifest !== 'object' || manifest === null || Array.isArray(manifest)) {
-                return undefined;
-            }
-            if ('type' in manifest && manifest.type === 'module')
-                return undefined;
-            return `e2e requires ES modules for .ts config and test files, but ${manifestPath} does not set "type": "module". To opt this package into ESM, run npm pkg set type=module from ${dir}. This also changes how existing .js files in the package are interpreted.`;
-        }
-        const parent = path.dirname(dir);
-        if (parent === dir || path.basename(parent) === 'node_modules')
-            break;
-        dir = parent;
-    }
-    return `e2e requires ES modules for .ts config and test files. Create a package.json with "type": "module" in ${path.dirname(filename)}, or run e2e init in the project directory.`;
-}
-//# sourceMappingURL=esm.js.map

package/dist/config/esm.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"esm.js","sourceRoot":"","sources":["../../src/config/esm.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAE7E,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACjE,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,gFAAgF;AAChF,MAAM,UAAU,cAAc,CAAC,UAAkB;IAC/C,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAChF,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,KAAK;QAAE,OAAO,SAAS,CAAC;IAEvD,IAAI,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjC,SAAS,CAAC;QACR,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;QACpD,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC7B,IAAI,QAAiB,CAAC;YACtB,IAAI,CAAC;gBACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;YAC5D,CAAC;YAAC,MAAM,CAAC;gBACP,0EAA0E;gBAC1E,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjF,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,IAAI,MAAM,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ;gBAAE,OAAO,SAAS,CAAC;YACvE,OAAO,8DAA8D,YAAY,kGAAkG,GAAG,4EAA4E,CAAC;QACrQ,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,cAAc;YAAE,MAAM;QACtE,GAAG,GAAG,MAAM,CAAC;IACf,CAAC;IAED,OAAO,yGAAyG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,6CAA6C,CAAC;AACtL,CAAC"}