@e2edev/e2e 0.9.0 → 0.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@e2edev/e2e",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "SDK, runner, and CLI for agentic end-to-end testing",
   "keywords": [
     "e2e",
@@ -73,13 +73,13 @@
     "ajv-formats": "3.0.1",
     "typescript": "7.0.2",
     "vitest": "5.0.0",
-    "@e2edev/playwright": "0.7.1"
+    "@e2edev/playwright": "0.8.0"
   },
   "engines": {
     "node": ">=22.12.0"
   },
   "scripts": {
-    "build": "node -e \"const fs=require('node:fs');for(const dir of ['dist','skills'])fs.rmSync(dir,{recursive:true,force:true});fs.cpSync('../../skills/e2e','skills/e2e',{recursive:true})\" && tsc --project tsconfig.build.json",
+    "build": "node -e \"const fs=require('node:fs');for(const dir of ['dist','skills'])fs.rmSync(dir,{recursive:true,force:true});fs.cpSync('../../skills/e2e','skills/e2e',{recursive:true})\" && tsc --project tsconfig.build.json && node -e \"const fs=require('node:fs');const versions=Object.fromEntries(['playwright','agent-device'].map((dir)=>['@e2edev/'+dir,JSON.parse(fs.readFileSync('../'+dir+'/package.json','utf8')).version]));fs.writeFileSync('dist/cli/init/engine-versions.json',JSON.stringify(versions)+'\\n')\"",
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
     "test:watch": "vitest",

package/skills/e2e/SKILL.md

@@ -29,7 +29,7 @@ export default {
     },
   ],
   // Only needed for agent.* steps; the model comes from E2E_MODEL.
-  agent: createAgent({ system: 'You are a thorough QA agent. Verify every outcome on screen.' }),
+  agents: { default: createAgent({ system: 'You are a thorough QA agent. Verify every outcome on screen.' }) },
 } satisfies E2EConfig;
 ```
 
@@ -90,7 +90,7 @@ one. Without them, the installed CLI prints the same text:
 - Import `test`, `expect`, and `credentials` from `@e2edev/e2e`. A test that
   uses the `web` fixture imports `test` from `@e2edev/playwright` instead: the
   same runtime `test`, typed with `web`.
-- Config and tests are ES modules. `package.json` needs `"type": "module"`.
+- Config and tests are ES modules and load as such whatever `package.json` sets as `type`.
 - Nothing waits explicitly: queries poll, actions wait, `expect` retries.
   Reads such as `textContent()` and `count()` do not retry, so assert with a
   matcher when a value has to settle.

package/skills/e2e/references/agent.md

@@ -6,7 +6,7 @@ transcript between calls. Deterministic tests never load a model.
 
 ## Configure a model
 
-There is no default model. Pick one of three shapes:
+Agents live under `agents` by name; `default` is the one tests use, `e2e run --agent <name>` re-points that default, a test or describe pins one with `{ agent: 'name' }`, and any `agent.*` call can name one with `{ agent: 'name' }` (innermost wins). There is no default model. Pick one of three shapes for an entry:
 
 ```ts
 // e2e.config.ts
@@ -14,13 +14,13 @@ import { createAgent } from '@e2edev/e2e/agent';
 
 export default {
   // 1. The built-in agent; the model comes from E2E_MODEL at run time.
-  agent: createAgent({ system: 'You are a thorough QA agent. Verify every outcome on screen.' }),
+  agents: { default: createAgent({ system: 'You are a thorough QA agent. Verify every outcome on screen.' }) },
 
   // 2. An options block: a gateway model string plus project vocabulary.
-  // agent: { model: 'anthropic/claude-sonnet-4.5', context: 'A billing dashboard. Plans are Free, Team, and Pro.' },
+  // agents: { default: { model: 'anthropic/claude-sonnet-4.5', context: 'A billing dashboard. Plans are Free, Team, and Pro.' } },
 
   // 3. A live AI SDK model instance for a provider called directly.
-  // agent: createAgent({ model: openai('gpt-5.4-mini') }),
+  // agents: { default: createAgent({ model: openai('gpt-5.4-mini') }) },
 } satisfies E2EConfig;
 ```
 
@@ -30,7 +30,7 @@ E2E_MODEL=anthropic/claude-sonnet-4.5 E2E_MODEL_API_KEY=... npx --no-install e2e
 
 - A `provider/model-id` string is routed through the Vercel AI Gateway; one
   `E2E_MODEL_API_KEY` (or `AI_GATEWAY_API_KEY`) reaches every provider.
-- With no `agent` key at all, the built-in agent still runs and takes its
+- With no `agents` key at all, the built-in agent still runs as `default` and takes its
   model from `E2E_MODEL`. Use `createAgent` for a `system` prompt, tools, or
   a pinned model.
 - `ai@^7` must be installed for any `agent.*` step; the runner loads it
@@ -40,7 +40,7 @@ E2E_MODEL=anthropic/claude-sonnet-4.5 E2E_MODEL_API_KEY=... npx --no-install e2e
 - `visionModel` (or `E2E_VISION_MODEL`) serves the calls that send pixels.
 - The model passed to `createAgent({ model })` is the one model for every
   `agent.*` call, `act` and the judgments alike, and outranks `E2E_MODEL`.
-  An `agent.model` naming a different model is `INVALID_CONFIG`.
+  An agent `model` naming a different model is `INVALID_CONFIG`.
 - Missing model or key: checked once per run when the first test acquires
   the `agent` fixture. One run-level `MODEL_UNAVAILABLE` (exit 2) stops the
   run; the remaining tests are skipped, not failed one by one.

package/skills/e2e/references/debugging.md

@@ -23,7 +23,7 @@ jq '.run.results[] | select(.status != "passed") | .attempts[-1]
 
 | Code | Usual cause | Fix |
 | --- | --- | --- |
-| `CONFIG_LOAD_FAILED` mentioning ES modules | `package.json` lacks `"type": "module"` | `npm pkg set type=module`, or rename the config to `.mts` and keep the tests in an ESM package |
+| `CONFIG_LOAD_FAILED` | The config throws while loading, or imports a package that is not installed or a subpath that does not exist | The message names the cause: install the dependency, or fix the import it quotes |
 | `INVALID_CONFIG` | Unknown key or a stale shape: a top-level `app`, `defineConfig`, a `backend` key, `json` combined with `list` reporters | Move app options into `playwright({ ... })`; use `satisfies E2EConfig`; the message names the key |
 | `CONFIG_NOT_FOUND`, `CONFIG_AMBIGUOUS` | Wrong `--config` path; both `.ts` and `.mts` present | Fix the path; keep one config file |
 | `NO_TESTS` | The glob or a positional matched nothing | The message names each positional that matched nothing. Check `tests` in the config and the `.e2e.ts` suffix |

package/skills/e2e/references/running.md

@@ -20,6 +20,7 @@ npx --no-install e2e telemetry [disable|enable] # anonymous usage telemetry: sta
 | `--target <ids>` | Comma-separated target names. Only selected targets start app commands and services; unknown names fail before startup. |
 | `--tag <tag>` | Repeatable tag filter; `--tag-mode all` requires every tag. |
 | `--headed` | Visible browser or simulator when the engine supports it. |
+| `--agent <name>` | Run with another configured agent (`agents.<name>`); default is `agents.default`. |
 | `--workers <n>`, `--retries <n>` | Override the resolved values. |
 | `--reporter <ids>` | `list`, `json`, `junit`, comma-separated. `json` cannot combine with `list`. |
 | `--artifacts <dir>` | Artifact root, default `.e2e/artifacts`. |
@@ -78,6 +79,9 @@ recorded flow is stale — `--no-cache` only skips the cache for one run.
 - `junit`: `.e2e/junit.xml` beside the report, for CI test summaries.
   Combine it with the terminal output: `--reporter list,junit`.
 - `json`: the report document on stdout.
+- `github()` from `@e2edev/github`: on GitHub Actions, one pull request comment per
+  run (edited on rerun) and the job summary; needs `pull-requests: write` and
+  `GITHUB_TOKEN` in the step's env.
 - Artifacts (screenshots, Playwright traces, `--video` recordings, `--debug`
   transcripts, downloads) live under `.e2e/artifacts/`; every path is
   recorded in the report.

package/skills/e2e/references/setup.md

@@ -3,11 +3,10 @@
 ## Requirements
 
 - Node.js 22.12 or newer.
-- ES modules. The `package.json` nearest to the config and the tests must set
-  `"type": "module"`. An `e2e.config.mts` is ESM on its own, but `.ts` tests
-  still follow their package. In a CommonJS package either run
-  `npm pkg set type=module` (this also changes how its `.js` files run) or put
-  the tests in their own ESM package directory.
+- ES modules. e2e loads `.ts` config, tests, and helpers as ES modules
+  whatever the nearest `package.json` `type` says, so a CommonJS package (a
+  Next.js app, for instance) needs no change. Write them with `import`, never
+  `require` or `module.exports`.
 - For browser tests, `@e2edev/playwright`. Missing browsers download when the
   engine first boots. In CI install them up front:
   `npx playwright install chromium --with-deps`.
@@ -69,7 +68,7 @@ export default {
     },
   ],
   // Only for agent.* steps. The model comes from E2E_MODEL unless set here.
-  agent: createAgent({ system: 'You are a thorough QA agent. Verify every outcome on screen.' }),
+  agents: { default: createAgent({ system: 'You are a thorough QA agent. Verify every outcome on screen.' }) },
   credentials: {
     admin: { username: 'admin@example.test', password: process.env.ADMIN_PASSWORD ?? '' },
   },
@@ -87,7 +86,7 @@ export default {
 | `workers` | half the cores, `1` in CI | Test files run in parallel across workers, at most the `workers` the engine declares per target (a device target: one per device). |
 | `reporters` | `['list']` | `list`, `json`, `junit`, and reporter objects (`{ name, onEvent?, onRunFinished? }`) that receive the finished run. `json` excludes `list`; `--reporter` keeps the objects. |
 | `cache` | `'read-write'`, `'read-only'` in CI | The trace cache for `agent.act`; `'off'` disables it. |
-| `agent` | unset | `createAgent(...)`, an options block `{ model, context, visionModel, maxSteps, maxModelCalls, vision, providerOptions }`, or a custom `StepExecutor`. Omitted, the built-in agent runs with `E2E_MODEL`. A model passed to `createAgent({ model })` is the one model for `act` and the judgments, over `E2E_MODEL`. |
+| `agents` | `{ default: built-in }` | Agents by name. `default` is what tests run with; `e2e run --agent <name>` runs with another. Each entry is `createAgent(...)`, an options block `{ model, context, visionModel, maxSteps, maxModelCalls, vision, providerOptions }`, or a custom `StepExecutor`. Omitted, the built-in agent runs with `E2E_MODEL`. |
 | `credentials` | `{}` | Named `{ username, password, allowedOrigins? }` entries; `password` may be a function returning the value. |
 | `screen.testIdAttribute` | `'data-testid'` | Attribute read by `getByTestId`. |
 | `artifacts` | `['screenshot', 'trace']` | Kinds to keep (`screenshot`, `trace`, and the opt-in `video`), or `{ kinds, store, video }`; `video: { retain: 'on-failure' }` keeps only the recordings of attempts that did not pass. |
@@ -110,6 +109,8 @@ process, and identity. `playwright()` accepts:
 | `browser` | `'chromium'` (default), `'firefox'`, `'webkit'`. |
 | `viewport` | `{ width, height }`, default 1280x720. |
 | `connect` | `{ cdpEndpoint }` to attach to a remote Chromium over CDP instead of launching. |
+| `headers` | Request headers sent to allowed origins only (a Vercel `x-vercel-protection-bypass`, ngrok's `ngrok-skip-browser-warning`). Reaches every path onto the page, `agent.act` included; turns the browser HTTP cache off and blocks service workers. |
+| `basicAuth` | `{ username, password }` answering a `401` challenge from an allowed origin; never sent to any other. |
 
 Two browsers are two targets sharing one app declaration:
 
@@ -185,7 +186,7 @@ or read the address from the environment:
 | --- | --- |
 | `E2E_MODEL` | `provider/model-id` for `agent.*` steps when the config sets no model. Routed through the Vercel AI Gateway. |
 | `E2E_MODEL_API_KEY` | Credential for that model. `AI_GATEWAY_API_KEY` is the fallback. |
-| `E2E_MODEL_ENDPOINT` | Any OpenAI-compatible endpoint URL for that model; the gateway is the default. HTTPS unless loopback. `agent.model.endpoint` wins. |
+| `E2E_MODEL_ENDPOINT` | Any OpenAI-compatible endpoint URL for that model; the gateway is the default. HTTPS unless loopback. The agent's `model.endpoint` wins. |
 | `E2E_VISION_MODEL` | Model for calls with `vision`; falls back to the main model. |
 | `E2E_USER_<NAME>_USERNAME`, `E2E_USER_<NAME>_PASSWORD` | Override `credentials.<name>`. `<NAME>` is the credential name uppercased, other characters as `_`. |
 | `CI` | Turns on CI defaults: `retries: 1`, `workers: 1`, `test.only` rejected, cache read-only, `reuseExisting` ignored. |
@@ -209,7 +210,7 @@ const iphone = agentDevice({ platform: 'ios', app: 'com.example.app' });
 export default {
   targets: [{ engine: iphone }],
   workers: 1,
-  agent: createAgent({ tools: agentDeviceTools(iphone) }),
+  agents: { default: createAgent({ tools: agentDeviceTools(iphone) }) },
 } satisfies E2EConfig;
 ```
 

package/skills/e2e/references/writing-tests.md

@@ -65,6 +65,7 @@ test.setup('sign in', { sessions: ['admin'] }, async ({ app, screen, session })
 | `requires` | `[]` | Capabilities the engine must contribute, e.g. `['web']`. Otherwise the test is skipped at selection instead of failing with `UNSUPPORTED_CAPABILITY`. |
 | `session` | unset | Restore state saved by a setup test. |
 | `agentContext` | unset | Extra context for `agent.*` calls in this test or group. |
+| `agent` | the run's agent | Pin the test or group to a configured agent (`agents.<name>`). Innermost wins; `agent.act(..., { agent })` can name another for one call. |
 | `serial` | `false` | Groups only. Members share one app state, run in order on one worker, and retry as a whole. Inside, per-member `retries`, `session`, `platforms`, `requires`, and `skip` are errors. |
 
 Hook order follows nesting, not position: outer `beforeEach` first, inner

package/dist/config/esm.d.ts

@@ -1,4 +0,0 @@
-/** Package-scope diagnostics for TypeScript config and test entry points. */
-/** Explains how to opt a `.ts` entry point's package into ESM when required. */
-export declare function esmPackageHint(modulePath: string): string | undefined;
-//# sourceMappingURL=esm.d.ts.map

package/dist/config/esm.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"esm.d.ts","sourceRoot":"","sources":["../../src/config/esm.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAK7E,gFAAgF;AAChF,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CA2BrE"}

package/dist/config/esm.js

@@ -1,35 +0,0 @@
-/** Package-scope diagnostics for TypeScript config and test entry points. */
-import { existsSync, readFileSync, realpathSync } from 'node:fs';
-import path from 'node:path';
-/** Explains how to opt a `.ts` entry point's package into ESM when required. */
-export function esmPackageHint(modulePath) {
-    const filename = existsSync(modulePath) ? realpathSync(modulePath) : modulePath;
-    if (path.extname(filename) !== '.ts')
-        return undefined;
-    let dir = path.dirname(filename);
-    for (;;) {
-        const manifestPath = path.join(dir, 'package.json');
-        if (existsSync(manifestPath)) {
-            let manifest;
-            try {
-                manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
-            }
-            catch {
-                // Leave malformed or unreadable manifests to the loader's own diagnostic.
-                return undefined;
-            }
-            if (typeof manifest !== 'object' || manifest === null || Array.isArray(manifest)) {
-                return undefined;
-            }
-            if ('type' in manifest && manifest.type === 'module')
-                return undefined;
-            return `e2e requires ES modules for .ts config and test files, but ${manifestPath} does not set "type": "module". To opt this package into ESM, run npm pkg set type=module from ${dir}. This also changes how existing .js files in the package are interpreted.`;
-        }
-        const parent = path.dirname(dir);
-        if (parent === dir || path.basename(parent) === 'node_modules')
-            break;
-        dir = parent;
-    }
-    return `e2e requires ES modules for .ts config and test files. Create a package.json with "type": "module" in ${path.dirname(filename)}, or run e2e init in the project directory.`;
-}
-//# sourceMappingURL=esm.js.map

package/dist/config/esm.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"esm.js","sourceRoot":"","sources":["../../src/config/esm.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAE7E,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACjE,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,gFAAgF;AAChF,MAAM,UAAU,cAAc,CAAC,UAAkB;IAC/C,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAChF,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,KAAK;QAAE,OAAO,SAAS,CAAC;IAEvD,IAAI,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjC,SAAS,CAAC;QACR,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;QACpD,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC7B,IAAI,QAAiB,CAAC;YACtB,IAAI,CAAC;gBACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;YAC5D,CAAC;YAAC,MAAM,CAAC;gBACP,0EAA0E;gBAC1E,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjF,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,IAAI,MAAM,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ;gBAAE,OAAO,SAAS,CAAC;YACvE,OAAO,8DAA8D,YAAY,kGAAkG,GAAG,4EAA4E,CAAC;QACrQ,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,cAAc;YAAE,MAAM;QACtE,GAAG,GAAG,MAAM,CAAC;IACf,CAAC;IAED,OAAO,yGAAyG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,6CAA6C,CAAC;AACtL,CAAC"}