@andwhy/e2e-pipeline 0.11.0 → 0.12.0

package/README.md

@@ -48,8 +48,9 @@ Re-running `init --force` in an existing project is the way to pick up a new ver
 - `.github/workflows/checkly-heal.yml` — a failing deployed monitor triggers a triage-and-heal agent: production regressions escalate to Slack immediately; stale tests get fixed and flaky tests hardened — including behavior-neutral test hooks (`data-testid`) in app source when needed — and opened as a human-review PR, whose own preview run validates the app-side hooks. Inert until a Checkly webhook alert channel dispatches to it — the skill walks through that setup.
 
 The workflows need repository secrets: `CHECKLY_API_KEY`, `CHECKLY_ACCOUNT_ID`,
-`VERCEL_AUTOMATION_BYPASS_SECRET`, and for auto-heal `ANTHROPIC_API_KEY` and
-`SLACK_WEBHOOK_URL` (values in 1Password).
+`VERCEL_AUTOMATION_BYPASS_SECRET`, and for auto-heal `SLACK_WEBHOOK_URL`
+(values in 1Password). `ANTHROPIC_API_KEY` is an organization secret inherited
+by all repos — nothing to add.
 
 E2E specs live under **`tests/e2e/`** so a repo can keep unit tests elsewhere under
 `tests/` without Playwright trying to run them. Seeds sit in the same tree and are

package/dist/bin/init.js

@@ -79,6 +79,11 @@ const run = async () => {
             console.log(`  ${path.relative(projectRoot, f)}`);
         }
     }
+    // Dependencies install before the formatting pass: prettier is resolved
+    // from the project's node_modules, which a fresh clone doesn't have yet.
+    // Formatting after the install means scaffolded files pick up the project's
+    // style even on a first run in a bare checkout.
+    installDeps(projectRoot, fileFormat);
     const formatted = await formatFiles(projectRoot, [
         ...result.created,
         ...result.regenerated,
@@ -110,13 +115,12 @@ const run = async () => {
             "  CHECKLY_API_KEY, CHECKLY_ACCOUNT_ID      — deploy monitors, auto-heal",
             "  VERCEL_AUTOMATION_BYPASS_SECRET          — PR tests through Vercel",
             "                                             deployment protection",
-            "  ANTHROPIC_API_KEY                        — Checkly auto-heal agent",
             "  SLACK_WEBHOOK_URL                        — auto-heal notifications",
-            "The auto-heal workflow stays inert until a Checkly webhook alert",
-            "channel dispatches to it — the skill walks through that setup.",
+            "(ANTHROPIC_API_KEY is an org-level secret inherited by all repos —",
+            "nothing to add.) The auto-heal workflow stays inert until a Checkly",
+            "webhook alert channel dispatches to it — the skill walks through that.",
         ].join("\n"));
     }
-    installDeps(projectRoot, fileFormat);
     console.log("Next steps:");
     let step = 1;
     const ext = fileFormat === "mjs" ? "mjs" : "ts";

package/dist/utils/install.js

@@ -47,8 +47,20 @@ export const installDeps = (projectRoot, fileFormat = "ts") => {
     console.log(`  ${cmd}\n`);
     execSync(cmd, { cwd: projectRoot, stdio: "inherit" });
     console.log("\nInstalling Chromium browser…\n");
-    execSync("npx playwright install chromium", {
-        cwd: projectRoot,
-        stdio: "inherit",
-    });
+    // Non-fatal: restricted networks (CI sandboxes, proxied agents) block the
+    // browser CDN, and a failed download must not abort the install — the
+    // scaffolded files are already written and CI installs its own browsers.
+    try {
+        execSync("npx playwright install chromium", {
+            cwd: projectRoot,
+            stdio: "inherit",
+        });
+    }
+    catch {
+        console.warn([
+            "\nChromium download failed (restricted network?). The scaffold is",
+            "complete — run `npx playwright install chromium` yourself before",
+            "local test runs.",
+        ].join("\n"));
+    }
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andwhy/e2e-pipeline",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "Scaffolds the full e2e testing pipeline: agent-assisted Playwright tests, CI runs against Vercel previews, and Checkly monitors that follow production deploys",
   "type": "module",
   "license": "MIT",

package/templates/claude/partials/_robustness-core.md

@@ -59,11 +59,32 @@
   await expect(page).not.toHaveURL(initialUrl, { timeout: 10_000 });
   ```
 
-- **Skip unsupported domains in `beforeEach`.** If a domain has no entry in a spec's locale
-  config, skip all tests up front rather than letting them fail:
+- **Skip unsupported domains in `beforeEach` — but never previews.** If a domain has no
+  entry in a spec's locale config, skip all tests up front rather than letting them fail:
 
   ```ts
   test.beforeEach(async ({ page, baseURL }) => {
     test.skip(!getLocaleConfig(baseURL!), "No config for this domain");
   });
   ```
+
+  A preview deployment's hostname (`*.vercel.app`) is never in the locale map, so a plain
+  exact-match lookup silently skips the entire suite on every PR run and in
+  `checkly pw-test` — green checks that tested nothing. The lookup must resolve preview
+  hosts to the config of the production domain the preview serves (ask the user which one
+  if it isn't obvious; `PREVIEW_LOCALE_HOST` overrides it at runtime):
+
+  ```ts
+  const getLocaleConfig = (baseURL: string) => {
+    const hostname = new URL(baseURL).hostname.replace(/^www\./, "");
+    const config = LOCALE_CONFIG[hostname];
+    if (config) return config;
+    // Previews serve <primary domain>'s content; without this every preview run skips.
+    if (hostname.endsWith(".vercel.app"))
+      return LOCALE_CONFIG[process.env.PREVIEW_LOCALE_HOST ?? "<primary domain>"];
+    return undefined;
+  };
+  ```
+
+  Only a genuinely unknown non-preview host skips. Single-domain suites without a locale
+  map are unaffected — they have no lookup to fall through.

package/templates/claude/rules/playwright-workflow.md

@@ -36,7 +36,10 @@ These rules apply to **all** agents (generator, healer, and orchestrator). Inclu
    project consent logic; do not reinvent it. The helper file extension must match rule 7.
 
 4. Never hardcode a single locale's URL pattern. Use a locale config map keyed
-   by bare hostname and resolve at runtime.
+   by bare hostname and resolve at runtime. Preview hosts (`*.vercel.app`) are
+   never in the map and must fall back to the primary production domain's
+   config (overridable via `PREVIEW_LOCALE_HOST`) — an exact-match-only lookup
+   skips the whole suite on every PR run.
 
 5. After all tests pass locally, verify with
    `BASE_URL=<dev-url> pnpm playwright test` via the terminal tool. If failures

package/templates/claude/skills/playwright-test-pipeline/SKILL.md

@@ -88,12 +88,12 @@ The scaffold writes templates, not a working setup. Walk these with the user:
    - `VERCEL_AUTOMATION_BYPASS_SECRET` — Vercel → project → Settings →
      Deployment Protection → Protection Bypass for Automation. Per project —
      never copied from another repo.
-   - `ANTHROPIC_API_KEY` — 1Password ("Anthropic"); powers the Checkly
-     auto-heal agent. Optional until Phase 5 wires the alert webhook, but
-     cheapest to add alongside the others.
-   - `SLACK_WEBHOOK_URL` — a Slack incoming webhook for the project's
-     notification channel; the auto-heal workflow posts its alerts there.
-     Ask which channel before creating one.
+   - `SLACK_WEBHOOK_URL` — the **org-shared** e2e-channel webhook: paste the
+     value from 1Password ("e2e Slack webhook"). Same value in every repo;
+     do not walk the user through creating a Slack app or webhook — routing
+     to the project channel happens via a Slack rule in Phase 5.
+   - `ANTHROPIC_API_KEY` needs **no action**: it is an organization-level
+     secret inherited by every repo. Do not tell the user to add it.
 5. **Existing tests under `tests/`** (unit, a11y — the scaffold lists them as
    "outside tests/e2e"): leave `testDir` at `./tests/e2e` and give that suite
    its own Playwright project with its own `testDir` if it should run under
@@ -180,8 +180,9 @@ overwritten):
 **Before the first merge, the repo needs secrets** (values in 1Password):
 `CHECKLY_API_KEY` and `CHECKLY_ACCOUNT_ID` for the deploy and auto-heal,
 `VERCEL_AUTOMATION_BYPASS_SECRET` for the PR run through deployment
-protection, `ANTHROPIC_API_KEY` and `SLACK_WEBHOOK_URL` for auto-heal. If
-the scaffold just created the workflows, say so explicitly and list the
+protection, `SLACK_WEBHOOK_URL` for auto-heal notifications
+(`ANTHROPIC_API_KEY` is inherited from the organization — nothing to add).
+If the scaffold just created the workflows, say so explicitly and list the
 secrets — a merge without them fails CI and looks like a pipeline bug.
 
 After merge, monitors update once the production deployment goes live — the
@@ -198,14 +199,20 @@ team's Slack message is written by the pipeline *after* triage (regression
 vs. stale test), so it carries a diagnosis and, for stale tests, the fix PR.
 Set up once per project:
 
-1. Confirm the `ANTHROPIC_API_KEY` and `SLACK_WEBHOOK_URL` repository
-   secrets from Phase 0 exist — the workflow escalates nothing without them.
-2. A GitHub token for Checkly to call `repository_dispatch`: fine-grained
-   PAT, scoped to this repo only, **Contents: read and write** permission,
-   the longest expiry policy allows — note the renewal date wherever the team
-   tracks credentials, an expired token silences the trigger. Store it in
-   Checkly as an environment variable (Checkly → Environment variables),
-   e.g. `GH_DISPATCH_TOKEN` — never in code or `checkly.config`.
+1. Confirm the `SLACK_WEBHOOK_URL` repository secret from Phase 0 exists —
+   the workflow escalates nothing without it. (`ANTHROPIC_API_KEY` comes
+   from the organization automatically.)
+2. The shared `GH_DISPATCH_TOKEN` secret exists in Checkly and covers this
+   repo. This is **one-time organization setup, not yours to perform**: the
+   token (`checkly-heal-dispatch`, a fine-grained PAT) is created once and
+   maintained by its owner, named in 1Password — see "One-time organization
+   setup" in the usage guide
+   (https://app.notion.com/p/32df837b627080748719c0f94a3aeaaf). Your only
+   per-project concern: if this repo is not yet on the token's repository
+   list, the webhook channel's test (step 3) fails with a 404 — then ask
+   the token owner to widen it (a one-minute edit on their side; the token
+   value doesn't change). Do not walk the user through creating a PAT or a
+   Checkly secret.
 3. Checkly → Alert channels → "Add more channels" → **Webhook**:
    - Method `POST`, URL
      `https://api.github.com/repos/<org>/<repo>/dispatches`
@@ -236,10 +243,21 @@ Set up once per project:
 5. **Do not also subscribe a Slack alert channel to these suites.** The
    pipeline is the single alert path; a parallel raw Checkly alert double-pings
    the channel and trains people to ignore one of them.
-6. Dry run: GitHub → Actions → "Checkly auto-heal" → Run workflow, with the
-   suite's check name. Confirm the run completes and the Slack message
-   arrives (a passing check should end as "not reproducible" — summary only,
-   no ping — so a real dry run is easiest right after any genuine failure).
+6. **One Slack Workflow Builder rule** routes this project's messages from
+   the shared e2e channel to the project channel. All notifications post to
+   the e2e channel tagged `[e2e:<repo-name>]`; the rule matches the **full
+   bracketed tag** — never a short name like `gbk`, which collides with
+   other text — and sends the message to the project channel. The sent
+   message's template is just the message-text variable (plus the
+   link-to-original variable when available, so threads happen on the
+   original) — no static prefix text. Have the user add a **second
+   manager** to the rule and record it in the rule registry in the e2e
+   channel's canvas.
+7. Dry run: GitHub → Actions → "Checkly auto-heal" → Run workflow, with the
+   suite's check name. Confirm the run completes and the message shows up in
+   the e2e channel and, via the rule, in the project channel (a passing
+   check should end as "not reproducible" — summary only, no message — so a
+   real dry run is easiest right after any genuine failure).
 
 What the user should expect afterwards: a failing monitor produces either a
 `checkly-heal` PR plus a Slack note (stale or flaky test — **review

package/templates/github/workflows/checkly-heal.yml

@@ -9,9 +9,12 @@ name: Checkly auto-heal
 # validates those; a diff guard aborts changes to protected paths
 # (.github, .claude, configs, lockfiles). The agent never pushes: branch,
 # commit, PR, and Slack are workflow steps.
-# Needs repo secrets: ANTHROPIC_API_KEY, CHECKLY_API_KEY, CHECKLY_ACCOUNT_ID,
-# SLACK_WEBHOOK_URL (values in 1Password). Setup docs: the
-# playwright-test-pipeline skill walks through the Checkly webhook channel.
+# Needs repo secrets CHECKLY_API_KEY, CHECKLY_ACCOUNT_ID, SLACK_WEBHOOK_URL
+# (the org-shared e2e-channel webhook; values in 1Password); ANTHROPIC_API_KEY
+# is inherited from the organization. Slack messages carry an [e2e:<repo>]
+# tag so Slack Workflow Builder rules route them from the shared channel to
+# project channels. Setup docs: the playwright-test-pipeline skill walks
+# through the Checkly webhook channel and the Slack rule.
 on:
   repository_dispatch:
     types: [checkly-alert]
@@ -294,6 +297,10 @@ jobs:
             *)
               text=":rotating_light: *Checkly: \`$CHECK_NAME\` failed and the auto-heal run did not finish cleanly.* ${SUMMARY:-Check the run logs.} This needs a developer." ;;
           esac
+          # Routing tag: Slack Workflow Builder rules match the full bracketed
+          # tag ([e2e:<repo>]) to fan messages out from the shared e2e channel
+          # to project channels without substring collisions.
+          text="[e2e:${GITHUB_REPOSITORY##*/}] $text"
           [ "$APP_CHANGE" = "true" ] && text="$text\nIncludes app-side test hooks — the PR's preview run validates them."
           [ -n "$RESULT_LINK" ] && text="$text\nCheckly result: $RESULT_LINK"
           text="$text\nRun: $RUN_URL"

package/templates/github/workflows/pr-playwright-tests.yml

@@ -92,6 +92,9 @@ jobs:
         env:
           BASE_URL: ${{ github.event.deployment_status.target_url }}
           VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}
+          # Which production domain's locale config preview hosts resolve to
+          # (repo variable, optional — specs default to the primary domain).
+          PREVIEW_LOCALE_HOST: ${{ vars.PREVIEW_LOCALE_HOST }}
           PLAYWRIGHT_JSON_OUTPUT_NAME: playwright-results.json
         run: |
           pnpm playwright test \