@andwhy/e2e-pipeline 0.10.1 → 0.11.1

package/README.md

@@ -35,6 +35,7 @@ Re-running `init --force` in an existing project is the way to pick up a new ver
 - `.claude/skills/playwright-test-pipeline/` — guided walkthrough skill; triggers when someone asks to create or fix tests and escorts them through setup check, Plan → Generate → Heal, preview validation, and deployment
 - `.mcp.json` — Playwright MCP server config (project root)
 - `playwright.mcp.config.{ts,mjs}` — config the MCP server loads: extends `playwright.config` and clears `testIgnore`, so the planner/generator can execute seeds that normal runs ignore
+- `.github/prompts/checkly-heal.md` — instructions for the auto-heal agent the `checkly-heal` workflow runs
 
 **Seeded once** — starting points your project then owns. Written only when absent and never overwritten, `--force` included:
 
@@ -43,10 +44,13 @@ Re-running `init --force` in an existing project is the way to pick up a new ver
 - `tests/e2e/seed.spec.{ts,js}` — seed file for Playwright MCP `setup_page`
 - `tests/e2e/helpers/preview-consent-setup.{ts,js}` — `prepareVercelPreviewPage` (Vercel deployment-protection bypass); projects extend it with consent logic
 - `.github/workflows/pr-playwright-tests.yml` — runs the suite against the PR's Vercel preview and comments the result (plus `.github/scripts/playwright-pr-comment.mjs`)
-- `.github/workflows/checkly-deploy.yml` — deploys monitors on merge to `main`
+- `.github/workflows/checkly-deploy.yml` — deploys monitors when a production deployment succeeds
+- `.github/workflows/checkly-heal.yml` — a failing deployed monitor triggers a triage-and-heal agent: production regressions escalate to Slack immediately; stale tests get fixed and flaky tests hardened — including behavior-neutral test hooks (`data-testid`) in app source when needed — and opened as a human-review PR, whose own preview run validates the app-side hooks. Inert until a Checkly webhook alert channel dispatches to it — the skill walks through that setup.
 
 The workflows need repository secrets: `CHECKLY_API_KEY`, `CHECKLY_ACCOUNT_ID`,
-and `VERCEL_AUTOMATION_BYPASS_SECRET` (values in 1Password).
+`VERCEL_AUTOMATION_BYPASS_SECRET`, and for auto-heal `SLACK_WEBHOOK_URL`
+(values in 1Password). `ANTHROPIC_API_KEY` is an organization secret inherited
+by all repos — nothing to add.
 
 E2E specs live under **`tests/e2e/`** so a repo can keep unit tests elsewhere under
 `tests/` without Playwright trying to run them. Seeds sit in the same tree and are

package/dist/bin/init.js

@@ -79,6 +79,11 @@ const run = async () => {
             console.log(`  ${path.relative(projectRoot, f)}`);
         }
     }
+    // Dependencies install before the formatting pass: prettier is resolved
+    // from the project's node_modules, which a fresh clone doesn't have yet.
+    // Formatting after the install means scaffolded files pick up the project's
+    // style even on a first run in a bare checkout.
+    installDeps(projectRoot, fileFormat);
     const formatted = await formatFiles(projectRoot, [
         ...result.created,
         ...result.regenerated,
@@ -107,12 +112,15 @@ const run = async () => {
         console.log([
             "\nGitHub workflows created. They need repository secrets before they run",
             "(values in 1Password):",
-            "  CHECKLY_API_KEY, CHECKLY_ACCOUNT_ID      — deploy monitors on merge",
+            "  CHECKLY_API_KEY, CHECKLY_ACCOUNT_ID      — deploy monitors, auto-heal",
             "  VERCEL_AUTOMATION_BYPASS_SECRET          — PR tests through Vercel",
             "                                             deployment protection",
+            "  SLACK_WEBHOOK_URL                        — auto-heal notifications",
+            "(ANTHROPIC_API_KEY is an org-level secret inherited by all repos —",
+            "nothing to add.) The auto-heal workflow stays inert until a Checkly",
+            "webhook alert channel dispatches to it — the skill walks through that.",
         ].join("\n"));
     }
-    installDeps(projectRoot, fileFormat);
     console.log("Next steps:");
     let step = 1;
     const ext = fileFormat === "mjs" ? "mjs" : "ts";

package/dist/utils/copy.js

@@ -77,6 +77,9 @@ const GITIGNORE_LINES = [
     // Session artifacts (traces, snapshots) the Playwright MCP server writes
     // while the planner/generator drive the browser.
     ".playwright-mcp/",
+    // Working directory of the Checkly auto-heal agent (context, outcome,
+    // downloaded results) — CI artifacts, never committed.
+    ".checkly-heal/",
 ];
 // Lines earlier scaffold versions added and this one must take back: seeds
 // were once regenerable root-level scaffolding, but since 0.5.0 the planner
@@ -228,6 +231,10 @@ export const scaffold = async (templatesRoot, projectRoot, { force = false, file
             path.join("workflows", "pr-playwright-tests.yml"),
             path.join(".github", "workflows", "pr-playwright-tests.yml"),
         ],
+        [
+            path.join("workflows", "checkly-heal.yml"),
+            path.join(".github", "workflows", "checkly-heal.yml"),
+        ],
         [
             path.join("scripts", "playwright-pr-comment.mjs"),
             path.join(".github", "scripts", "playwright-pr-comment.mjs"),
@@ -244,6 +251,25 @@ export const scaffold = async (templatesRoot, projectRoot, { force = false, file
             result.created.push(destPath);
         }
     }
+    // The heal prompt is package-owned like the agent definitions: it carries
+    // agent behavior, not project config, so --force updates it.
+    const githubOwnedMappings = [
+        [
+            path.join("prompts", "checkly-heal.md"),
+            path.join(".github", "prompts", "checkly-heal.md"),
+        ],
+    ];
+    for (const [src, dest] of githubOwnedMappings) {
+        const destPath = path.join(projectRoot, dest);
+        if (!force && (await fs.pathExists(destPath))) {
+            result.skipped.push(destPath);
+        }
+        else {
+            await fs.ensureDir(path.dirname(destPath));
+            await fs.copy(path.join(githubSrc, src), destPath);
+            result.created.push(destPath);
+        }
+    }
     await appendGitignore(projectRoot, result);
     return result;
 };

package/dist/utils/install.js

@@ -47,8 +47,20 @@ export const installDeps = (projectRoot, fileFormat = "ts") => {
     console.log(`  ${cmd}\n`);
     execSync(cmd, { cwd: projectRoot, stdio: "inherit" });
     console.log("\nInstalling Chromium browser…\n");
-    execSync("npx playwright install chromium", {
-        cwd: projectRoot,
-        stdio: "inherit",
-    });
+    // Non-fatal: restricted networks (CI sandboxes, proxied agents) block the
+    // browser CDN, and a failed download must not abort the install — the
+    // scaffolded files are already written and CI installs its own browsers.
+    try {
+        execSync("npx playwright install chromium", {
+            cwd: projectRoot,
+            stdio: "inherit",
+        });
+    }
+    catch {
+        console.warn([
+            "\nChromium download failed (restricted network?). The scaffold is",
+            "complete — run `npx playwright install chromium` yourself before",
+            "local test runs.",
+        ].join("\n"));
+    }
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andwhy/e2e-pipeline",
-  "version": "0.10.1",
+  "version": "0.11.1",
   "description": "Scaffolds the full e2e testing pipeline: agent-assisted Playwright tests, CI runs against Vercel previews, and Checkly monitors that follow production deploys",
   "type": "module",
   "license": "MIT",

package/templates/claude/partials/_robustness-core.md

@@ -59,11 +59,32 @@
   await expect(page).not.toHaveURL(initialUrl, { timeout: 10_000 });
   ```
 
-- **Skip unsupported domains in `beforeEach`.** If a domain has no entry in a spec's locale
-  config, skip all tests up front rather than letting them fail:
+- **Skip unsupported domains in `beforeEach` — but never previews.** If a domain has no
+  entry in a spec's locale config, skip all tests up front rather than letting them fail:
 
   ```ts
   test.beforeEach(async ({ page, baseURL }) => {
     test.skip(!getLocaleConfig(baseURL!), "No config for this domain");
   });
   ```
+
+  A preview deployment's hostname (`*.vercel.app`) is never in the locale map, so a plain
+  exact-match lookup silently skips the entire suite on every PR run and in
+  `checkly pw-test` — green checks that tested nothing. The lookup must resolve preview
+  hosts to the config of the production domain the preview serves (ask the user which one
+  if it isn't obvious; `PREVIEW_LOCALE_HOST` overrides it at runtime):
+
+  ```ts
+  const getLocaleConfig = (baseURL: string) => {
+    const hostname = new URL(baseURL).hostname.replace(/^www\./, "");
+    const config = LOCALE_CONFIG[hostname];
+    if (config) return config;
+    // Previews serve <primary domain>'s content; without this every preview run skips.
+    if (hostname.endsWith(".vercel.app"))
+      return LOCALE_CONFIG[process.env.PREVIEW_LOCALE_HOST ?? "<primary domain>"];
+    return undefined;
+  };
+  ```
+
+  Only a genuinely unknown non-preview host skips. Single-domain suites without a locale
+  map are unaffected — they have no lookup to fall through.

package/templates/claude/rules/playwright-workflow.md

@@ -36,7 +36,10 @@ These rules apply to **all** agents (generator, healer, and orchestrator). Inclu
    project consent logic; do not reinvent it. The helper file extension must match rule 7.
 
 4. Never hardcode a single locale's URL pattern. Use a locale config map keyed
-   by bare hostname and resolve at runtime.
+   by bare hostname and resolve at runtime. Preview hosts (`*.vercel.app`) are
+   never in the map and must fall back to the primary production domain's
+   config (overridable via `PREVIEW_LOCALE_HOST`) — an exact-match-only lookup
+   skips the whole suite on every PR run.
 
 5. After all tests pass locally, verify with
    `BASE_URL=<dev-url> pnpm playwright test` via the terminal tool. If failures
@@ -201,3 +204,16 @@ If any tests fail:
 - **Plan files** are saved at `tests/e2e/<feature>/<name>.plan.md`.
 - **Test files** follow **Project test rules** §7–8 — typically `tests/e2e/<feature>/<group>.spec.js`
   or `.spec.ts` with one `describe` and multiple tests.
+
+## Checkly auto-heal (CI, not this session)
+
+Failing deployed monitors are handled by `.github/workflows/checkly-heal.yml`:
+it triages first (production regression → Slack alert; stale test → fix; flaky
+test → harden), may add behavior-neutral test hooks (`data-testid`) to app
+source when the robust fix needs them, and opens a `checkly-heal` PR a human
+must review — app-side hooks are validated by the PR's own preview run. Its
+agent instructions live in `.github/prompts/checkly-heal.md` (package-owned —
+update via `init --force`, don't hand-edit). If a user asks about a
+`checkly-heal` PR, that workflow authored it; reviewing one means verifying the
+updated test still describes correct production behavior, not that it merely
+passes.

package/templates/claude/skills/playwright-test-pipeline/SKILL.md

@@ -76,7 +76,7 @@ The scaffold writes templates, not a working setup. Walk these with the user:
    page, and pointing it at an empty or unreachable app produces a plan that
    asserts nothing useful.
 4. **Repository secrets for the workflows**: if the scaffold just created
-   `.github/workflows/`, tell the user now — not at merge time — to add three
+   `.github/workflows/`, tell the user now — not at merge time — to add the
    repository secrets (GitHub → Settings → Secrets and variables → Actions).
    You cannot add them; a person must. Without them the first PR push and the
    first merge fail CI in ways that look like pipeline bugs.
@@ -88,6 +88,11 @@ The scaffold writes templates, not a working setup. Walk these with the user:
    - `VERCEL_AUTOMATION_BYPASS_SECRET` — Vercel → project → Settings →
      Deployment Protection → Protection Bypass for Automation. Per project —
      never copied from another repo.
+   - `SLACK_WEBHOOK_URL` — a Slack incoming webhook for the project's
+     notification channel; the auto-heal workflow posts its alerts there.
+     Ask which channel before creating one.
+   - `ANTHROPIC_API_KEY` needs **no action**: it is an organization-level
+     secret inherited by every repo. Do not tell the user to add it.
 5. **Existing tests under `tests/`** (unit, a11y — the scaffold lists them as
    "outside tests/e2e"): leave `testDir` at `./tests/e2e` and give that suite
    its own Playwright project with its own `testDir` if it should run under
@@ -148,7 +153,7 @@ until after merge.
 
 ## Phase 5 — Merge and deployment
 
-The scaffold ships two workflows (seeded once — existing ones are never
+The scaffold ships three workflows (seeded once — existing ones are never
 overwritten):
 
 - `.github/workflows/pr-playwright-tests.yml` — runs the suite against the
@@ -161,13 +166,23 @@ overwritten):
   merge that adds tests for new `data-testid`s would go red until the release
   carrying them is live — with gated releases that gap can be days. A
   `workflow_dispatch` fallback allows manual runs.
+- `.github/workflows/checkly-heal.yml` — a failing deployed monitor
+  dispatches here (via the webhook alert channel below). An agent triages
+  first: a production regression or anything needing a behavioral app change
+  alerts Slack immediately; a stale test gets fixed and a flaky test gets
+  hardened — including behavior-neutral test hooks (`data-testid`) in app
+  source when the robust fix needs them — and opened as a PR that a human
+  must review. Test-only fixes are verified against production and on
+  Checkly's infrastructure; app-side hooks are validated by the PR's own
+  preview run (pr-playwright-tests). Inert until the webhook exists.
 
 **Before the first merge, the repo needs secrets** (values in 1Password):
-`CHECKLY_API_KEY` and `CHECKLY_ACCOUNT_ID` for the deploy,
+`CHECKLY_API_KEY` and `CHECKLY_ACCOUNT_ID` for the deploy and auto-heal,
 `VERCEL_AUTOMATION_BYPASS_SECRET` for the PR run through deployment
-protection. If the scaffold just created the workflows, say so explicitly and
-list the three secrets — a merge without them fails CI and looks like a
-pipeline bug.
+protection, `SLACK_WEBHOOK_URL` for auto-heal notifications
+(`ANTHROPIC_API_KEY` is inherited from the organization — nothing to add).
+If the scaffold just created the workflows, say so explicitly and list the
+secrets — a merge without them fails CI and looks like a pipeline bug.
 
 After merge, monitors update once the production deployment goes live — the
 user verifies in the Checkly dashboard (https://app.checklyhq.com/); nothing
@@ -175,20 +190,69 @@ to run by hand. Manual fallback if a repo deleted the deploy workflow:
 `pnpm checkly test --record`, then `pnpm checkly deploy` (requires the
 Phase 4 login).
 
-**Alert notifications are not automatic — walk the user through it.** A
-failing monitor only shows in the Checkly dashboard until an alert channel
-exists, so nobody gets notified. In Checkly:
-
-1. Alert channels → "Add more channels" → select **Slack**.
-2. Channel configuration: the project's Slack notification channel
-   (`#…`). A dedicated test-notification channel may exist by now — ask
-   which channel to use rather than assuming.
-3. Verify the subscriptions: **only the current check** subscribed, and
-   **uncheck** "Automatically subscribe newly created checks to this alert
-   channel".
-4. Test the alert channel (button) and confirm the notification arrived in
-   Slack.
-5. "Create alert Slack channel".
+**Alert notifications are not automatic — walk the user through the
+auto-heal webhook.** A failing monitor only shows in the Checkly dashboard
+until an alert channel exists. The alert channel for these suites is a
+**webhook into the auto-heal workflow**, not a direct Slack channel: the
+team's Slack message is written by the pipeline *after* triage (regression
+vs. stale test), so it carries a diagnosis and, for stale tests, the fix PR.
+Set up once per project:
+
+1. Confirm the `SLACK_WEBHOOK_URL` repository secret from Phase 0 exists —
+   the workflow escalates nothing without it. (`ANTHROPIC_API_KEY` comes
+   from the organization automatically.)
+2. The shared `GH_DISPATCH_TOKEN` secret exists in Checkly and covers this
+   repo. This is **one-time organization setup, not yours to perform**: the
+   token (`checkly-heal-dispatch`, a fine-grained PAT) is created once and
+   maintained by its owner, named in 1Password — see "One-time organization
+   setup" in the usage guide
+   (https://app.notion.com/p/32df837b627080748719c0f94a3aeaaf). Your only
+   per-project concern: if this repo is not yet on the token's repository
+   list, the webhook channel's test (step 3) fails with a 404 — then ask
+   the token owner to widen it (a one-minute edit on their side; the token
+   value doesn't change). Do not walk the user through creating a PAT or a
+   Checkly secret.
+3. Checkly → Alert channels → "Add more channels" → **Webhook**:
+   - Method `POST`, URL
+     `https://api.github.com/repos/<org>/<repo>/dispatches`
+   - Headers: `Authorization: Bearer {{GH_DISPATCH_TOKEN}}` and
+     `Accept: application/vnd.github+json`
+   - Body (Checkly's editor lists the available variables — confirm the
+     names there):
+
+     ```json
+     {
+       "event_type": "checkly-alert",
+       "client_payload": {
+         "check_name": "{{CHECK_NAME}}",
+         "check_id": "{{CHECK_ID}}",
+         "result_id": "{{CHECK_RESULT_ID}}",
+         "result_link": "{{RESULT_LINK}}",
+         "alert_type": "{{ALERT_TYPE}}"
+       }
+     }
+     ```
+
+   - Notify on **failure**; enable **reminders** while a check stays failing
+     — a reminder re-fires the webhook, the workflow dedupes repeats, and a
+     lost first delivery gets retried by the next one. Recovery/degraded
+     notifications are ignored by the workflow either way.
+4. Subscriptions: **only the current check**, and **uncheck** "Automatically
+   subscribe newly created checks to this alert channel".
+5. **Do not also subscribe a Slack alert channel to these suites.** The
+   pipeline is the single alert path; a parallel raw Checkly alert double-pings
+   the channel and trains people to ignore one of them.
+6. Dry run: GitHub → Actions → "Checkly auto-heal" → Run workflow, with the
+   suite's check name. Confirm the run completes and the Slack message
+   arrives (a passing check should end as "not reproducible" — summary only,
+   no ping — so a real dry run is easiest right after any genuine failure).
+
+What the user should expect afterwards: a failing monitor produces either a
+`checkly-heal` PR plus a Slack note (stale or flaky test — **review
+required, the PR is never merged automatically**; PRs carrying app-side test
+hooks say so and point at their preview run for validation), or a Slack
+escalation (regression, fix needs a behavioral app change, agent failed). A
+transient blip that doesn't reproduce stays silent by design.
 
 ## Conventions to uphold
 

package/templates/github/prompts/checkly-heal.md

@@ -0,0 +1,141 @@
+# Checkly monitor auto-heal
+
+You are running unattended in CI because a deployed Checkly monitor failed
+against production. Your job is to triage the failure first, and only then —
+when the test is stale or flaky rather than the site broken — fix it. A
+workflow wraps this run: it commits, opens the pull request, and notifies the
+team. You never push, never open PRs, never send notifications.
+
+## Inputs
+
+- `.checkly-heal/context.json` — the failing check's name, the alert type, a
+  link to the Checkly result, and the paths of any downloaded artifacts.
+- `.checkly-heal/result.json` — the raw Checkly check result when the API call
+  succeeded: error message, failing test and step, stack trace. May be absent;
+  proceed without it if so.
+- `.claude/agents/playwright-test-healer.md` — debugging practice and
+  robustness rules. Follow its methodology. The overrides below take
+  precedence over anything it says.
+
+## Overrides for this run
+
+These invert or forbid parts of the normal healer behavior. They are not
+negotiable:
+
+1. **`BASE_URL` pins to production, not localhost.** The monitor asserts
+   against production, so production is the reproduction and verification
+   target. Resolve the URL from `checkly.config.{ts,mjs}`: find the
+   `playwrightChecks` entry whose `name` (or `logicalId`) matches the failing
+   check name from the context, and use its `BASE_URL` environment variable.
+2. **Never `test.fixme()`, never skip, disable, or delete a test.** A skipped
+   test is a silently disabled production monitor. If you cannot fix it,
+   classify accordingly and stop.
+3. **App source changes are limited to behavior-neutral test hooks.** You may
+   add or restore attributes that exist for tests — `data-testid`, `aria-*`
+   where genuinely correct — on components the failing test targets. You may
+   not change logic, markup structure, styling, content, dependencies, or
+   configuration. If only a behavioral app change can fix the failure, that is
+   the `needs-app-change` classification, not something you attempt. Never
+   touch `.github/`, `.claude/`, `package.json`, lockfiles, or any
+   `*.config.*` file.
+4. **Never run git commit, push, or any GitHub CLI/API command.**
+
+## Procedure
+
+### 1. Orient
+
+Read the context and result files. Identify the failing spec file, test, and
+step. Read the spec. Resolve the production `BASE_URL` (override 1).
+
+### 2. Reproduce
+
+Run only the failing spec against production:
+
+```bash
+BASE_URL=<production-url> pnpm playwright test <failing-spec> --project=Chrome
+```
+
+- **Fails** → continue to triage.
+- **Passes** → run it again with `--repeat-each=3`. All green means the
+  failure was transient on Checkly's side: classification `not-reproducible`,
+  stop, change nothing. Intermittent failures mean the test is **flaky** —
+  continue to triage with that hypothesis.
+
+### 3. Triage — before any edit
+
+Decide which of these is true, using the failure output, a fresh trace, and
+the live page:
+
+- **The site is broken.** The user-visible outcome the test asserts genuinely
+  does not work on production — a purchase flow erroring, a page failing to
+  render, content missing that should exist. Classification `regression`.
+  Stop immediately. Do not touch the test: adjusting a test to accept broken
+  behavior hides an outage.
+- **The test is flaky.** The outcome works, but the test fails
+  intermittently — races, missing waits for dynamic content, animation
+  timing, order-dependent assertions. Classification `flaky` — proceed to
+  step 4 and fix the root cause per the robustness rules; blanket timeout
+  increases are not a fix.
+- **The test is stale.** Production changed legitimately (new markup, changed
+  copy, moved element) and the test's selectors or assertions no longer
+  describe it, while the actual user outcome still works. Classification
+  `healed` — proceed to step 4.
+- **The fix needs a behavioral app change.** The failure is rooted in app
+  code beyond a test hook — or in a pending release that will resolve it.
+  Classification `needs-app-change`. Stop and say precisely what a developer
+  must change, or which release resolves it.
+
+When genuinely uncertain between regression and any fixable case, choose
+`regression`. A false alarm costs a developer minutes; a masked outage costs
+much more.
+
+### 4. Heal (stale or flaky)
+
+Fix the test following the healer methodology and robustness rules: prefer
+resilient locators, fix root causes over timeouts, keep edits minimal.
+
+When the robust fix needs a stable hook the app no longer (or never) provides,
+add the minimal behavior-neutral attribute to the component (override 3) and
+target it from the test — do not fall back to a brittle text or CSS selector
+just to avoid touching app source. Keep such changes to the fewest attributes
+that make the test robust.
+
+### 5. Verify
+
+- **Test-only fixes:** re-run the spec against production until green — for a
+  flaky fix, prove stability with `--repeat-each=5`, not a single pass. Then
+  run the full e2e suite once the same way to catch collateral damage, and
+  validate on Checkly's infrastructure:
+
+  ```bash
+  pnpm checkly pw-test --env BASE_URL="<production-url>" -- <failing-spec>
+  ```
+
+  Credentials are in the environment. If cloud verification cannot run, say
+  so in the outcome details rather than claiming it.
+- **Fixes that add app-side test hooks:** production does not carry your new
+  attributes yet, so a full production run cannot pass — that is expected.
+  Verify everything verifiable (the spec compiles, unrelated tests still pass
+  against production, the selector logic matches the component you edited)
+  and state exactly what remains for the preview run: the pull request the
+  workflow opens gets its own Vercel preview deployment, and the repo's
+  `pr-playwright-tests` workflow runs the full suite against it — that run is
+  the authoritative validation for app-side hooks. Say this in the details so
+  the reviewer knows to check it.
+
+### 6. Write the outcome — required on every path
+
+Write `.checkly-heal/outcome.json` before finishing, whatever happened:
+
+```json
+{
+  "classification": "healed | flaky | regression | needs-app-change | not-reproducible",
+  "summary": "One sentence for the Slack notification.",
+  "details": "Markdown for the PR body or alert: what failed, the root cause, what changed — or exactly what a developer needs to do. For app-side hooks: which components gained which attributes, and that the PR's preview run validates them."
+}
+```
+
+For `healed` and `flaky`, the details become the PR description a human
+reviews — include what broke, why, what you changed, and how it was verified.
+If you finish without writing this file, the workflow treats the run as failed
+and escalates to a developer, so an honest partial outcome always beats none.

package/templates/github/workflows/checkly-heal.yml

@@ -0,0 +1,318 @@
+name: Checkly auto-heal
+
+# A failing Checkly monitor dispatches here (webhook alert channel →
+# repository_dispatch). The agent triages first: a stale or flaky test gets
+# fixed and opened as a human-review PR; a real production regression, a fix
+# needing behavioral app code, or any agent failure escalates to Slack
+# instead. Fixes may include behavior-neutral test hooks in app source
+# (data-testid) — the PR's own preview run (pr-playwright-tests.yml)
+# validates those; a diff guard aborts changes to protected paths
+# (.github, .claude, configs, lockfiles). The agent never pushes: branch,
+# commit, PR, and Slack are workflow steps.
+# Needs repo secrets CHECKLY_API_KEY, CHECKLY_ACCOUNT_ID, SLACK_WEBHOOK_URL
+# (values in 1Password); ANTHROPIC_API_KEY is inherited from the organization.
+# Setup docs: the playwright-test-pipeline skill walks through the Checkly
+# webhook channel.
+on:
+  repository_dispatch:
+    types: [checkly-alert]
+  workflow_dispatch:
+    inputs:
+      check_name:
+        description: "Failing check name (as in checkly.config)"
+        required: true
+      check_id:
+        description: "Checkly check ID (optional, enables result download)"
+        required: false
+      result_id:
+        description: "Checkly check result ID (optional)"
+        required: false
+
+run-name: "Checkly auto-heal · ${{ github.event.client_payload.check_name || inputs.check_name }}"
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
+
+# One heal at a time; queued alerts (e.g. an outage failing several checks)
+# run serially and the dedup below makes repeats cheap no-ops.
+concurrency:
+  group: checkly-heal
+  cancel-in-progress: false
+
+env:
+  # The e2e test root. Changes outside it (behavior-neutral test hooks in app
+  # source) are allowed but flagged on the PR and in Slack.
+  # Match your playwright.config testDir.
+  E2E_ROOT: tests/e2e
+  # Paths the agent must never change — anything here aborts and escalates.
+  # Extended regex against the changed-file list.
+  PROTECTED_PATHS: '^(\.github/|\.claude/|\.mcp\.json$|.*\.config\.(ts|mjs|js|cjs)$|package\.json$|pnpm-lock\.yaml$|package-lock\.json$|yarn\.lock$|\.env)'
+  # How long a merged heal PR blocks a second silent attempt on the same check.
+  REPEAT_WINDOW_DAYS: 7
+
+jobs:
+  heal:
+    # Recovery/degraded notifications are not work; only failures are.
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      contains(github.event.client_payload.alert_type || 'ALERT_FAILURE', 'FAILURE')
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+
+    steps:
+      - name: Check out the default branch
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      # Dedup and repeat-offender guard. An open heal PR for this check means
+      # the alert is already handled. A heal PR merged recently means the fix
+      # did not hold — a second silent attempt would mask instability, so that
+      # escalates to a human instead.
+      - name: Guard against duplicate and repeated heals
+        id: guard
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CHECK_NAME: ${{ github.event.client_payload.check_name || inputs.check_name }}
+        run: |
+          slug=$(printf '%s' "$CHECK_NAME" | tr -c 'a-zA-Z0-9' '-' | tr -s '-' | sed 's/^-//;s/-$//' | cut -c1-40 | tr '[:upper:]' '[:lower:]')
+          echo "slug=${slug:-check}" >> "$GITHUB_OUTPUT"
+
+          open=$(gh pr list --state open --label checkly-heal --json headRefName \
+            --jq "[.[] | select(.headRefName | startswith(\"checkly-heal/$slug\"))] | length")
+          if [ "$open" -gt 0 ]; then
+            echo "mode=skip" >> "$GITHUB_OUTPUT"
+            echo "A heal PR for this check is already open — nothing to do." >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          cutoff=$(date -u -d "$REPEAT_WINDOW_DAYS days ago" +%Y-%m-%dT%H:%M:%SZ)
+          recent=$(gh pr list --state merged --label checkly-heal --json headRefName,mergedAt \
+            --jq "[.[] | select((.headRefName | startswith(\"checkly-heal/$slug\")) and .mergedAt > \"$cutoff\")] | length")
+          if [ "$recent" -gt 0 ]; then
+            echo "mode=repeat" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          echo "mode=heal" >> "$GITHUB_OUTPUT"
+
+      - name: Set up pnpm
+        if: steps.guard.outputs.mode == 'heal'
+        uses: pnpm/action-setup@v4
+
+      - name: Set up Node
+        if: steps.guard.outputs.mode == 'heal'
+        uses: actions/setup-node@v5
+        with:
+          node-version: 22
+          cache: pnpm
+
+      # --ignore-scripts: only the tests run here; project postinstall hooks
+      # are not needed and may want CMS tokens CI does not have.
+      - name: Install dependencies
+        if: steps.guard.outputs.mode == 'heal'
+        run: pnpm install --frozen-lockfile --ignore-scripts
+
+      - name: Install Playwright browsers
+        if: steps.guard.outputs.mode == 'heal'
+        run: pnpm playwright install --with-deps chromium
+
+      # Checkly already diagnosed the failure — error, failing step, trace.
+      # Best effort: the agent reproduces from scratch when this is missing.
+      - name: Fetch the Checkly result and write agent context
+        if: steps.guard.outputs.mode == 'heal'
+        env:
+          CHECKLY_API_KEY: ${{ secrets.CHECKLY_API_KEY }}
+          CHECKLY_ACCOUNT_ID: ${{ secrets.CHECKLY_ACCOUNT_ID }}
+          CHECK_NAME: ${{ github.event.client_payload.check_name || inputs.check_name }}
+          CHECK_ID: ${{ github.event.client_payload.check_id || inputs.check_id }}
+          RESULT_ID: ${{ github.event.client_payload.result_id || inputs.result_id }}
+          RESULT_LINK: ${{ github.event.client_payload.result_link }}
+          ALERT_TYPE: ${{ github.event.client_payload.alert_type }}
+        run: |
+          mkdir -p .checkly-heal
+          if [ -n "$CHECK_ID" ] && [ -n "$RESULT_ID" ]; then
+            curl -sf --max-time 30 \
+              -H "Authorization: Bearer $CHECKLY_API_KEY" \
+              -H "X-Checkly-Account: $CHECKLY_ACCOUNT_ID" \
+              "https://api.checklyhq.com/v1/check-results/$CHECK_ID/$RESULT_ID" \
+              -o .checkly-heal/result.json \
+              || echo "Checkly result fetch failed — agent reproduces from scratch."
+          fi
+          jq -n \
+            --arg check_name "$CHECK_NAME" \
+            --arg alert_type "${ALERT_TYPE:-ALERT_FAILURE}" \
+            --arg result_link "${RESULT_LINK:-}" \
+            --arg result_file "$([ -f .checkly-heal/result.json ] && echo .checkly-heal/result.json)" \
+            '{check_name: $check_name, alert_type: $alert_type, result_link: $result_link, result_file: $result_file}' \
+            > .checkly-heal/context.json
+
+      # The agent gets test-fixing tools and Checkly credentials (for
+      # pw-test verification) — deliberately no GitHub token: it cannot push,
+      # comment, or open PRs even if it tries.
+      - name: Run the triage-and-heal agent
+        if: steps.guard.outputs.mode == 'heal'
+        id: agent
+        timeout-minutes: 35
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          CHECKLY_API_KEY: ${{ secrets.CHECKLY_API_KEY }}
+          CHECKLY_ACCOUNT_ID: ${{ secrets.CHECKLY_ACCOUNT_ID }}
+        run: |
+          npm install -g @anthropic-ai/claude-code@2
+          claude -p "$(cat .github/prompts/checkly-heal.md)" \
+            --model claude-opus-5 \
+            --allowedTools "Bash,Read,Edit,Write,Glob,Grep" \
+            --permission-mode dontAsk \
+            --max-turns 150 \
+            --output-format json \
+            > .checkly-heal/agent-output.json || true
+
+      # The guard is deterministic on purpose: the PR path requires an outcome
+      # file claiming a fix AND a diff clear of protected paths. App-source
+      # changes (test hooks) are allowed but derived from the diff, not the
+      # agent's claim, and flagged downstream. Everything else — protected
+      # paths, missing outcome, other classifications — escalates.
+      - name: Read the outcome and guard the diff
+        if: steps.guard.outputs.mode == 'heal'
+        id: outcome
+        run: |
+          classification="agent-failed"
+          summary="The heal agent did not produce an outcome."
+          if [ -f .checkly-heal/outcome.json ] && jq -e . .checkly-heal/outcome.json > /dev/null 2>&1; then
+            classification=$(jq -r '.classification // "agent-failed"' .checkly-heal/outcome.json)
+            summary=$(jq -r '.summary // ""' .checkly-heal/outcome.json)
+          fi
+
+          files=$(git status --porcelain | awk '{print $NF}' | grep -v "^\.checkly-heal/" || true)
+          protected=$(printf '%s' "$files" | grep -Ec "$PROTECTED_PATHS" || true)
+          changed=$(printf '%s' "$files" | grep -c . || true)
+          app_change=$(printf '%s' "$files" | grep -cv "^$E2E_ROOT/" || true)
+
+          if [ "$classification" = "healed" ] || [ "$classification" = "flaky" ]; then
+            if [ "$protected" -gt 0 ]; then
+              classification="agent-failed"
+              summary="Diff guard tripped: protected files (.github, .claude, configs, lockfiles) were changed. Nothing was pushed."
+            elif [ "$changed" -eq 0 ]; then
+              classification="agent-failed"
+              summary="The agent reported a fix but changed no files."
+            fi
+          fi
+
+          echo "app_change=$([ "$app_change" -gt 0 ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          echo "classification=$classification" >> "$GITHUB_OUTPUT"
+          {
+            echo "summary<<CHECKLY_HEAL_EOF"
+            echo "$summary"
+            echo "CHECKLY_HEAL_EOF"
+          } >> "$GITHUB_OUTPUT"
+          echo "## Checkly auto-heal: $classification" >> "$GITHUB_STEP_SUMMARY"
+          echo "$summary" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Open the heal PR for human review
+        if: >-
+          steps.guard.outputs.mode == 'heal' &&
+          (steps.outcome.outputs.classification == 'healed' ||
+           steps.outcome.outputs.classification == 'flaky')
+        id: pr
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CHECK_NAME: ${{ github.event.client_payload.check_name || inputs.check_name }}
+          RESULT_LINK: ${{ github.event.client_payload.result_link }}
+          SLUG: ${{ steps.guard.outputs.slug }}
+          CLASSIFICATION: ${{ steps.outcome.outputs.classification }}
+          APP_CHANGE: ${{ steps.outcome.outputs.app_change }}
+        run: |
+          branch="checkly-heal/$SLUG-${{ github.run_id }}"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -b "$branch"
+          git add -A -- ':(exclude).checkly-heal'
+          git commit -m "Heal Checkly check: $CHECK_NAME"
+          git push -u origin "$branch"
+
+          gh label create checkly-heal --force \
+            --description "Automated fix for a failing Checkly monitor" \
+            --color BFD4F2 || true
+
+          case "$CLASSIFICATION" in
+            flaky) kind="a flaky test (hardened, stability proven with repeated runs)" ;;
+            *)     kind="a stale test" ;;
+          esac
+          {
+            echo "A Checkly monitor failed and the auto-heal agent classified it as $kind. **Human review required — do not merge unread.**"
+            echo ""
+            echo "- Check: \`$CHECK_NAME\`"
+            [ -n "$RESULT_LINK" ] && echo "- Failing result: $RESULT_LINK"
+            echo "- Heal run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            if [ "$APP_CHANGE" = "true" ]; then
+              echo ""
+              echo "> [!IMPORTANT]"
+              echo "> This fix includes app-source changes (behavior-neutral test hooks such as \`data-testid\`). Production does not carry them yet, so the authoritative validation is **this PR's own preview run** (pr-playwright-tests) — check it before merging."
+            fi
+            echo ""
+            jq -r '.details // ""' .checkly-heal/outcome.json
+          } > pr-body.md
+
+          url=$(gh pr create \
+            --title "Heal Checkly check: $CHECK_NAME" \
+            --body-file pr-body.md \
+            --label checkly-heal)
+          echo "url=$url" >> "$GITHUB_OUTPUT"
+
+      # The single alert path: every outcome that needs eyes lands in Slack,
+      # written after triage. Runs on step failure too (always() + the guard
+      # conditions), so a crashed agent still alerts.
+      - name: Notify Slack
+        if: always() && steps.guard.outputs.mode != 'skip'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          CHECK_NAME: ${{ github.event.client_payload.check_name || inputs.check_name }}
+          RESULT_LINK: ${{ github.event.client_payload.result_link }}
+          MODE: ${{ steps.guard.outputs.mode }}
+          CLASSIFICATION: ${{ steps.outcome.outputs.classification }}
+          SUMMARY: ${{ steps.outcome.outputs.summary }}
+          APP_CHANGE: ${{ steps.outcome.outputs.app_change }}
+          PR_URL: ${{ steps.pr.outputs.url }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          case "${MODE}:${CLASSIFICATION}" in
+            repeat:*)
+              text=":rotating_light: *Checkly: \`$CHECK_NAME\` is failing again* — a heal PR for this check merged within the last $REPEAT_WINDOW_DAYS days, so no second automatic fix was attempted. This needs a developer." ;;
+            *:healed)
+              text=":wrench: *Checkly: \`$CHECK_NAME\` failed — fix ready for review.* $SUMMARY\nPR: $PR_URL" ;;
+            *:flaky)
+              text=":wrench: *Checkly: \`$CHECK_NAME\` was flaky — test hardened, fix ready for review.* $SUMMARY\nPR: $PR_URL" ;;
+            *:regression)
+              text=":rotating_light: *Checkly: \`$CHECK_NAME\` failed — production regression.* The test is correct; the site is broken. $SUMMARY" ;;
+            *:needs-app-change)
+              text=":warning: *Checkly: \`$CHECK_NAME\` failed — fix needs an app code change.* $SUMMARY" ;;
+            *:not-reproducible)
+              # Transient blip, nothing to act on — summary only, no ping.
+              echo "Not reproducible — no Slack notification." >> "$GITHUB_STEP_SUMMARY"
+              exit 0 ;;
+            *)
+              text=":rotating_light: *Checkly: \`$CHECK_NAME\` failed and the auto-heal run did not finish cleanly.* ${SUMMARY:-Check the run logs.} This needs a developer." ;;
+          esac
+          [ "$APP_CHANGE" = "true" ] && text="$text\nIncludes app-side test hooks — the PR's preview run validates them."
+          [ -n "$RESULT_LINK" ] && text="$text\nCheckly result: $RESULT_LINK"
+          text="$text\nRun: $RUN_URL"
+
+          if [ -z "$SLACK_WEBHOOK_URL" ]; then
+            echo "SLACK_WEBHOOK_URL is not set — notification skipped:" >> "$GITHUB_STEP_SUMMARY"
+            printf '%b\n' "$text" >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+          jq -n --arg text "$(printf '%b' "$text")" '{text: $text}' \
+            | curl -sf --max-time 15 -X POST -H "Content-Type: application/json" -d @- "$SLACK_WEBHOOK_URL" \
+            || echo "Slack notification failed — see summary." >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload agent artifacts
+        if: always() && steps.guard.outputs.mode == 'heal'
+        uses: actions/upload-artifact@v4
+        with:
+          name: checkly-heal-${{ github.run_id }}
+          path: .checkly-heal/
+          if-no-files-found: ignore
+          retention-days: 14

package/templates/github/workflows/pr-playwright-tests.yml

@@ -92,6 +92,9 @@ jobs:
         env:
           BASE_URL: ${{ github.event.deployment_status.target_url }}
           VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}
+          # Which production domain's locale config preview hosts resolve to
+          # (repo variable, optional — specs default to the primary domain).
+          PREVIEW_LOCALE_HOST: ${{ vars.PREVIEW_LOCALE_HOST }}
           PLAYWRIGHT_JSON_OUTPUT_NAME: playwright-results.json
         run: |
           pnpm playwright test \