@andwhy/e2e-pipeline 0.12.1 → 0.13.0

package/dist/bin/init.js

@@ -117,8 +117,10 @@ const run = async () => {
             "                                             deployment protection",
             "  SLACK_WEBHOOK_URL                        — auto-heal notifications",
             "(ANTHROPIC_API_KEY is an org-level secret inherited by all repos —",
-            "nothing to add.) The auto-heal workflow stays inert until a Checkly",
-            "webhook alert channel dispatches to it — the skill walks through that.",
+            "nothing to add; if the heal agent fails to authenticate, a repo",
+            "secret of the same name overrides it.) The auto-heal workflow stays",
+            "inert until a Checkly webhook alert channel dispatches to it — the",
+            "skill walks through that.",
         ].join("\n"));
     }
     console.log("Next steps:");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andwhy/e2e-pipeline",
-  "version": "0.12.1",
+  "version": "0.13.0",
   "description": "Scaffolds the full e2e testing pipeline: agent-assisted Playwright tests, CI runs against Vercel previews, and Checkly monitors that follow production deploys",
   "type": "module",
   "license": "MIT",

package/templates/claude/skills/playwright-test-pipeline/SKILL.md

@@ -92,8 +92,13 @@ The scaffold writes templates, not a working setup. Walk these with the user:
      value from 1Password ("e2e Slack webhook"). Same value in every repo;
      do not walk the user through creating a Slack app or webhook — routing
      to the project channel happens via a Slack rule in Phase 5.
-   - `ANTHROPIC_API_KEY` needs **no action**: it is an organization-level
-     secret inherited by every repo. Do not tell the user to add it.
+   - `ANTHROPIC_API_KEY` usually needs **no action**: it is an
+     organization-level secret inherited by every repo — do not walk the
+     user through adding it up front. But it can fail to reach a repo
+     (org-secret scoping, or a stale stored value): if a heal run's agent
+     step fails to authenticate, have the user add a repository-level
+     `ANTHROPIC_API_KEY` — a repo secret with the same name overrides the
+     organization one.
 5. **Existing tests under `tests/`** (unit, a11y — the scaffold lists them as
    "outside tests/e2e"): leave `testDir` at `./tests/e2e` and give that suite
    its own Playwright project with its own `testDir` if it should run under
@@ -181,7 +186,9 @@ overwritten):
 `CHECKLY_API_KEY` and `CHECKLY_ACCOUNT_ID` for the deploy and auto-heal,
 `VERCEL_AUTOMATION_BYPASS_SECRET` for the PR run through deployment
 protection, `SLACK_WEBHOOK_URL` for auto-heal notifications
-(`ANTHROPIC_API_KEY` is inherited from the organization — nothing to add).
+(`ANTHROPIC_API_KEY` is inherited from the organization — nothing to add
+unless a heal run fails to authenticate; then a repo-level secret of the
+same name overrides it, see Phase 0).
 If the scaffold just created the workflows, say so explicitly and list the
 secrets — a merge without them fails CI and looks like a pipeline bug.
 
@@ -201,7 +208,9 @@ Set up once per project:
 
 1. Confirm the `SLACK_WEBHOOK_URL` repository secret from Phase 0 exists —
    the workflow escalates nothing without it. (`ANTHROPIC_API_KEY` comes
-   from the organization automatically.)
+   from the organization automatically — but if the dry run in step 6 ends
+   agent-failed with an authentication error, override it with a repo-level
+   secret of the same name, per Phase 0.)
 2. The shared `GH_DISPATCH_TOKEN` secret exists in Checkly and covers this
    repo. This is **one-time organization setup, not yours to perform**: the
    token (`checkly-heal-dispatch`, a fine-grained PAT) is created once and

package/templates/github/prompts/checkly-heal.md

@@ -43,6 +43,12 @@ negotiable:
    workflow reads your outcome from it after you finish, and destroying it
    turns your entire run into an unexplained failure. No `rm -rf`, no
    `git clean`. Only add files to it.
+6. **Runtime is metered — stop the moment the classification is certain.**
+   You run under a hard 35-minute step timeout, and every minute is paid CI
+   and API time. Run the narrowest thing that answers the current question,
+   never the full suite, and never re-run a test whose answer you already
+   have. If the deadline nears, write an honest partial outcome instead of
+   pushing on — a run killed by the timeout reports nothing.
 
 ## Procedure
 
@@ -60,17 +66,20 @@ run "wasn't real".
 
 ### 2. Reproduce
 
-Run only the failing spec against production:
+Identify the narrowest reproduction: the failing spec and test named by
+`result.json` when present; without a result file, the spec files belonging
+to the failing check. Run that once against production, stopping at the
+first failure:
 
 ```bash
-BASE_URL=<production-url> pnpm playwright test <failing-spec> --project=Chrome
+BASE_URL=<production-url> pnpm playwright test <failing-spec> --project=Chrome --max-failures=1
 ```
 
 - **Fails** → continue to triage.
-- **Passes** → run it again with `--repeat-each=3`. All green means the
-  failure was transient on Checkly's side: classification `not-reproducible`,
-  stop, change nothing. Intermittent failures mean the test is **flaky** —
-  continue to triage with that hypothesis.
+- **Passes** → classification `not-reproducible`. Write the outcome and stop
+  immediately — no repeat runs, no wider investigation. Checkly saw it fail
+  and it passes now, so the blip was transient; a genuinely unstable test
+  will alert again, and that repetition is the signal a human acts on.
 
 ### 3. Triage — before any edit
 
@@ -84,9 +93,10 @@ the live page:
   behavior hides an outage.
 - **The test is flaky.** The outcome works, but the test fails
   intermittently — races, missing waits for dynamic content, animation
-  timing, order-dependent assertions. Classification `flaky` — proceed to
-  step 4 and fix the root cause per the robustness rules; blanket timeout
-  increases are not a fix.
+  timing, order-dependent assertions. When the trace alone doesn't settle
+  it, repeating the one failing test is the only permitted repeat run.
+  Classification `flaky` — proceed to step 4 and fix the root cause per the
+  robustness rules; blanket timeout increases are not a fix.
 - **The test is stale.** Production changed legitimately (new markup, changed
   copy, moved element) and the test's selectors or assertions no longer
   describe it, while the actual user outcome still works. Classification
@@ -113,10 +123,11 @@ that make the test robust.
 
 ### 5. Verify
 
-- **Test-only fixes:** re-run the spec against production until green — for a
-  flaky fix, prove stability with `--repeat-each=5`, not a single pass. Then
-  run the full e2e suite once the same way to catch collateral damage, and
-  validate on Checkly's infrastructure:
+- **Test-only fixes:** re-run the fixed spec against production until green —
+  for a flaky fix, prove stability with `--repeat-each=5` on that one spec,
+  not a single pass. Do not run the full suite (override 6): the heal PR gets
+  its own `pr-playwright-tests` preview run, and that is where collateral
+  damage surfaces. Validate the fixed spec on Checkly's infrastructure:
 
   ```bash
   pnpm checkly pw-test --env BASE_URL="<production-url>" -- <failing-spec>

package/templates/github/workflows/checkly-heal.yml

@@ -11,7 +11,9 @@ name: Checkly auto-heal
 # commit, PR, and Slack are workflow steps.
 # Needs repo secrets CHECKLY_API_KEY, CHECKLY_ACCOUNT_ID, SLACK_WEBHOOK_URL
 # (the org-shared e2e-channel webhook; values in 1Password); ANTHROPIC_API_KEY
-# is inherited from the organization. Slack messages carry an [e2e:<repo>]
+# is inherited from the organization (a repo secret of the same name
+# overrides it if the org value fails to authenticate). Slack messages
+# carry an [e2e:<repo>]
 # tag so Slack Workflow Builder rules route them from the shared channel to
 # project channels. Setup docs: the playwright-test-pipeline skill walks
 # through the Checkly webhook channel and the Slack rule.
@@ -165,36 +167,55 @@ jobs:
           npm install -g @anthropic-ai/claude-code@2
           # Transcript lives OUTSIDE .checkly-heal so it survives whatever the
           # agent does to its working directory, and outside the repo tree so
-          # the diff guard never counts it.
+          # the diff guard never counts it. stream-json writes each event as it
+          # happens — a run killed by the timeout still leaves the full history
+          # (plain json buffers everything and a kill leaves an empty file).
           claude -p "$(cat .github/prompts/checkly-heal.md)" \
             --model claude-opus-5 \
             --allowedTools "Bash,Read,Edit,Write,Glob,Grep" \
             --permission-mode dontAsk \
             --max-turns 150 \
-            --output-format json \
-            > /tmp/checkly-heal-agent-output.json || echo "claude exited non-zero: $?" >> "$GITHUB_STEP_SUMMARY"
+            --output-format stream-json --verbose \
+            > /tmp/checkly-heal-agent-output.json || echo "claude exited non-zero: $?" | tee -a "$GITHUB_STEP_SUMMARY"
+          # Diagnostics go to the job log AND the summary: the log is readable
+          # over the API, the summary only in the browser.
           {
             echo "### Agent step diagnostics"
             echo '```'
             ls -la .checkly-heal 2>&1 || echo ".checkly-heal missing"
             tail -c 2000 /tmp/checkly-heal-agent-output.json 2>/dev/null || echo "no agent output"
             echo '```'
-          } >> "$GITHUB_STEP_SUMMARY"
+          } | tee -a "$GITHUB_STEP_SUMMARY"
 
       # The guard is deterministic on purpose: the PR path requires an outcome
       # file claiming a fix AND a diff clear of protected paths. App-source
       # changes (test hooks) are allowed but derived from the diff, not the
       # agent's claim, and flagged downstream. Everything else — protected
       # paths, missing outcome, other classifications — escalates.
+      # always(): a killed agent step (timeout, cancel) must still produce a
+      # classification, or Slack reports an unexplained failure.
       - name: Read the outcome and guard the diff
-        if: steps.guard.outputs.mode == 'heal'
+        if: always() && steps.guard.outputs.mode == 'heal'
         id: outcome
+        env:
+          AGENT_CONCLUSION: ${{ steps.agent.conclusion }}
         run: |
           classification="agent-failed"
           summary="The heal agent did not produce an outcome."
-          if [ -f .checkly-heal/outcome.json ] && jq -e . .checkly-heal/outcome.json > /dev/null 2>&1; then
+          if [ "$AGENT_CONCLUSION" = "success" ] && [ -f .checkly-heal/outcome.json ] && jq -e . .checkly-heal/outcome.json > /dev/null 2>&1; then
             classification=$(jq -r '.classification // "agent-failed"' .checkly-heal/outcome.json)
             summary=$(jq -r '.summary // ""' .checkly-heal/outcome.json)
+          else
+            # No trustworthy outcome — say WHY, so the Slack message diagnoses
+            # itself. An outcome written before a kill is unverified: escalate
+            # rather than open a PR for it.
+            if [ "$AGENT_CONCLUSION" = "skipped" ]; then
+              summary="The heal agent never ran — a setup step failed before it."
+            elif [ "$AGENT_CONCLUSION" != "success" ]; then
+              summary="The heal agent was killed before finishing (step conclusion: ${AGENT_CONCLUSION:-unknown} — usually the 35-minute timeout)."
+            fi
+            agent_error=$(jq -r 'select(.is_error == true) | .result // empty' /tmp/checkly-heal-agent-output.json 2>/dev/null | head -c 300)
+            [ -n "$agent_error" ] && summary="$summary Agent error: $agent_error"
           fi
 
           files=$(git status --porcelain | awk '{print $NF}' | grep -v "^\.checkly-heal/" || true)
@@ -291,28 +312,48 @@ jobs:
         run: |
           case "${MODE}:${CLASSIFICATION}" in
             repeat:*)
-              text=":rotating_light: *Checkly: \`$CHECK_NAME\` is failing again* — a heal PR for this check merged within the last $REPEAT_WINDOW_DAYS days, so no second automatic fix was attempted. This needs a developer." ;;
+              title=":rotating_light: *\`$CHECK_NAME\` is failing again*"
+              body="A heal PR for this check merged within the last $REPEAT_WINDOW_DAYS days, so no second automatic fix was attempted. This needs a developer." ;;
             *:healed)
-              text=":wrench: *Checkly: \`$CHECK_NAME\` failed — fix ready for review.* $SUMMARY\nPR: $PR_URL" ;;
+              title=":wrench: *\`$CHECK_NAME\` failed — fix ready for review*"
+              body="$SUMMARY" ;;
             *:flaky)
-              text=":wrench: *Checkly: \`$CHECK_NAME\` was flaky — test hardened, fix ready for review.* $SUMMARY\nPR: $PR_URL" ;;
+              title=":wrench: *\`$CHECK_NAME\` was flaky — test hardened, fix ready for review*"
+              body="$SUMMARY" ;;
             *:regression)
-              text=":rotating_light: *Checkly: \`$CHECK_NAME\` failed — production regression.* The test is correct; the site is broken. $SUMMARY" ;;
+              title=":rotating_light: *\`$CHECK_NAME\` failed — production regression*"
+              body="The test is correct; the site is broken. $SUMMARY" ;;
             *:needs-app-change)
-              text=":warning: *Checkly: \`$CHECK_NAME\` failed — fix needs an app code change.* $SUMMARY" ;;
+              title=":warning: *\`$CHECK_NAME\` failed — fix needs an app code change*"
+              body="$SUMMARY" ;;
             *:not-reproducible)
               # Transient blip, nothing to act on — summary only, no ping.
               echo "Not reproducible — no Slack notification." >> "$GITHUB_STEP_SUMMARY"
               exit 0 ;;
             *)
-              text=":rotating_light: *Checkly: \`$CHECK_NAME\` failed and the auto-heal run did not finish cleanly.* ${SUMMARY:-Check the run logs.} This needs a developer." ;;
+              title=":rotating_light: *\`$CHECK_NAME\` failed and the auto-heal run did not finish cleanly*"
+              body="${SUMMARY:-Check the run logs.} This needs a developer." ;;
           esac
-          # Routing tag: Slack Workflow Builder rules match the full bracketed
-          # tag ([e2e:<repo>]) to fan messages out from the shared e2e channel
-          # to project channels without substring collisions.
-          text="[e2e:${GITHUB_REPOSITORY##*/}] $text"
-          [ "$APP_CHANGE" = "true" ] && text="$text\nIncludes app-side test hooks — the PR's preview run validates them."
-          [ -n "$RESULT_LINK" ] && text="$text\nCheckly result: $RESULT_LINK"
+
+          # Repeat mode skips the agent, so there is no outcome classification;
+          # label it as what it is rather than defaulting to agent-failed.
+          label="${CLASSIFICATION:-agent-failed}"
+          [ "$MODE" = "repeat" ] && label="repeat-offender"
+
+          # Failing test titles, best effort from the Checkly result's embedded
+          # Playwright report — the recursive search tolerates schema changes,
+          # and an empty result simply omits the field.
+          failed_tests=""
+          if [ -f .checkly-heal/result.json ]; then
+            failed_tests=$(jq -r '[.. | objects | select((.status? == "failed") or (.outcome? == "unexpected")) | .title? // empty] | unique | .[0:3] | join(", ")' .checkly-heal/result.json 2>/dev/null || true)
+          fi
+
+          # Plain-text fallback: what notifications, search, and the Slack
+          # routing rule see — the Workflow Builder rule matches the [e2e:<repo>]
+          # tag here and forwards this text, so it stays a complete message.
+          repo="${GITHUB_REPOSITORY##*/}"
+          text="[e2e:$repo] $title — $body"
+          [ -n "$PR_URL" ] && text="$text\nPR: $PR_URL"
           text="$text\nRun: $RUN_URL"
 
           if [ -z "$SLACK_WEBHOOK_URL" ]; then
@@ -320,7 +361,51 @@ jobs:
             printf '%b\n' "$text" >> "$GITHUB_STEP_SUMMARY"
             exit 0
           fi
-          jq -n --arg text "$(printf '%b' "$text")" '{text: $text}' \
+
+          # Block Kit body: title, classification/repo fields, failing tests
+          # when known, the diagnosis, and link buttons assembled per outcome —
+          # a button never renders without its target.
+          jq -n \
+            --arg text "$(printf '%b' "$text")" \
+            --arg title "$title" \
+            --arg body "$body" \
+            --arg classification "$label" \
+            --arg repo "$repo" \
+            --arg failed "$failed_tests" \
+            --arg app_change "$APP_CHANGE" \
+            --arg pr "$PR_URL" \
+            --arg run "$RUN_URL" \
+            --arg result "$RESULT_LINK" \
+            --arg rerun "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/workflows/checkly-heal.yml" \
+            '
+            def btn($label; $url): {type: "button", text: {type: "plain_text", text: $label}, url: $url};
+            {
+              text: $text,
+              blocks: (
+                [
+                  {type: "section", text: {type: "mrkdwn", text: $title}},
+                  {type: "section", fields: ([
+                    {type: "mrkdwn", text: ("*Classification*\n" + $classification)},
+                    {type: "mrkdwn", text: ("*Repository*\n" + $repo)}
+                  ] + (if $failed != "" then
+                    [{type: "mrkdwn", text: ("*Failed test*\n" + $failed)}]
+                  else [] end))}
+                ]
+                + (if $body != "" then
+                    [{type: "section", text: {type: "mrkdwn", text: $body}}]
+                  else [] end)
+                + (if $app_change == "true" then
+                    [{type: "context", elements: [{type: "mrkdwn", text: "Includes app-side test hooks — the PR'\''s preview run validates them."}]}]
+                  else [] end)
+                + [{type: "actions", elements: (
+                    (if $pr != "" then [btn("Review PR"; $pr)] else [] end)
+                    + [btn("View run"; $run)]
+                    + (if $result != "" then [btn("Checkly result"; $result)] else [] end)
+                    + (if $classification == "agent-failed" then [btn("Re-run heal"; $rerun)] else [] end)
+                  )}]
+                + [{type: "context", elements: [{type: "mrkdwn", text: ("[e2e:" + $repo + "]")}]}]
+              )
+            }' \
             | curl -sf --max-time 15 -X POST -H "Content-Type: application/json" -d @- "$SLACK_WEBHOOK_URL" \
             || echo "Slack notification failed — see summary." >> "$GITHUB_STEP_SUMMARY"
 
@@ -332,5 +417,8 @@ jobs:
           path: |
             .checkly-heal/
             /tmp/checkly-heal-agent-output.json
+          # .checkly-heal is a dot-directory; without this it is silently
+          # excluded and the debug artifact loses context/outcome/result.
+          include-hidden-files: true
           if-no-files-found: ignore
           retention-days: 14