@andwhy/e2e-pipeline 0.12.2 → 0.13.1

package/dist/bin/init.js

@@ -117,8 +117,10 @@ const run = async () => {
             "                                             deployment protection",
             "  SLACK_WEBHOOK_URL                        — auto-heal notifications",
             "(ANTHROPIC_API_KEY is an org-level secret inherited by all repos —",
-            "nothing to add.) The auto-heal workflow stays inert until a Checkly",
-            "webhook alert channel dispatches to it — the skill walks through that.",
+            "nothing to add; if the heal agent fails to authenticate, a repo",
+            "secret of the same name overrides it.) The auto-heal workflow stays",
+            "inert until a Checkly webhook alert channel dispatches to it — the",
+            "skill walks through that.",
         ].join("\n"));
     }
     console.log("Next steps:");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andwhy/e2e-pipeline",
-  "version": "0.12.2",
+  "version": "0.13.1",
   "description": "Scaffolds the full e2e testing pipeline: agent-assisted Playwright tests, CI runs against Vercel previews, and Checkly monitors that follow production deploys",
   "type": "module",
   "license": "MIT",

package/templates/claude/skills/playwright-test-pipeline/SKILL.md

@@ -92,8 +92,13 @@ The scaffold writes templates, not a working setup. Walk these with the user:
      value from 1Password ("e2e Slack webhook"). Same value in every repo;
      do not walk the user through creating a Slack app or webhook — routing
      to the project channel happens via a Slack rule in Phase 5.
-   - `ANTHROPIC_API_KEY` needs **no action**: it is an organization-level
-     secret inherited by every repo. Do not tell the user to add it.
+   - `ANTHROPIC_API_KEY` usually needs **no action**: it is an
+     organization-level secret inherited by every repo — do not walk the
+     user through adding it up front. But it can fail to reach a repo
+     (org-secret scoping, or a stale stored value): if a heal run's agent
+     step fails to authenticate, have the user add a repository-level
+     `ANTHROPIC_API_KEY` — a repo secret with the same name overrides the
+     organization one.
 5. **Existing tests under `tests/`** (unit, a11y — the scaffold lists them as
    "outside tests/e2e"): leave `testDir` at `./tests/e2e` and give that suite
    its own Playwright project with its own `testDir` if it should run under
@@ -181,7 +186,9 @@ overwritten):
 `CHECKLY_API_KEY` and `CHECKLY_ACCOUNT_ID` for the deploy and auto-heal,
 `VERCEL_AUTOMATION_BYPASS_SECRET` for the PR run through deployment
 protection, `SLACK_WEBHOOK_URL` for auto-heal notifications
-(`ANTHROPIC_API_KEY` is inherited from the organization — nothing to add).
+(`ANTHROPIC_API_KEY` is inherited from the organization — nothing to add
+unless a heal run fails to authenticate; then a repo-level secret of the
+same name overrides it, see Phase 0).
 If the scaffold just created the workflows, say so explicitly and list the
 secrets — a merge without them fails CI and looks like a pipeline bug.
 
@@ -201,7 +208,9 @@ Set up once per project:
 
 1. Confirm the `SLACK_WEBHOOK_URL` repository secret from Phase 0 exists —
    the workflow escalates nothing without it. (`ANTHROPIC_API_KEY` comes
-   from the organization automatically.)
+   from the organization automatically — but if the dry run in step 6 ends
+   agent-failed with an authentication error, override it with a repo-level
+   secret of the same name, per Phase 0.)
 2. The shared `GH_DISPATCH_TOKEN` secret exists in Checkly and covers this
    repo. This is **one-time organization setup, not yours to perform**: the
    token (`checkly-heal-dispatch`, a fine-grained PAT) is created once and

package/templates/github/workflows/checkly-heal.yml

@@ -11,7 +11,9 @@ name: Checkly auto-heal
 # commit, PR, and Slack are workflow steps.
 # Needs repo secrets CHECKLY_API_KEY, CHECKLY_ACCOUNT_ID, SLACK_WEBHOOK_URL
 # (the org-shared e2e-channel webhook; values in 1Password); ANTHROPIC_API_KEY
-# is inherited from the organization. Slack messages carry an [e2e:<repo>]
+# is inherited from the organization (a repo secret of the same name
+# overrides it if the org value fails to authenticate). Slack messages
+# carry an [e2e:<repo>]
 # tag so Slack Workflow Builder rules route them from the shared channel to
 # project channels. Setup docs: the playwright-test-pipeline skill walks
 # through the Checkly webhook channel and the Slack rule.
@@ -259,7 +261,12 @@ jobs:
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git checkout -b "$branch"
-          git add -A -- ':(exclude).checkly-heal'
+          # Stage everything, then unstage the agent's scratch dir. An
+          # ':(exclude).checkly-heal' pathspec looks equivalent but exits 1
+          # when the path is gitignored (as init makes it), which killed
+          # every PR step before a commit existed.
+          git add -A
+          git reset -q -- .checkly-heal
           git commit -m "Heal Checkly check: $CHECK_NAME"
           git push -u origin "$branch"
 
@@ -307,31 +314,65 @@ jobs:
           APP_CHANGE: ${{ steps.outcome.outputs.app_change }}
           PR_URL: ${{ steps.pr.outputs.url }}
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          BRANCH: checkly-heal/${{ steps.guard.outputs.slug }}-${{ github.run_id }}
         run: |
-          case "${MODE}:${CLASSIFICATION}" in
+          # A fix only reaches review through a PR. When the PR step failed,
+          # "fix ready for review" would be false: report the failure and
+          # offer the re-run instead.
+          outcome="$CLASSIFICATION"
+          if { [ "$CLASSIFICATION" = "healed" ] || [ "$CLASSIFICATION" = "flaky" ]; } && [ -z "$PR_URL" ]; then
+            outcome="pr-failed"
+          fi
+
+          case "${MODE}:${outcome}" in
             repeat:*)
-              text=":rotating_light: *Checkly: \`$CHECK_NAME\` is failing again* — a heal PR for this check merged within the last $REPEAT_WINDOW_DAYS days, so no second automatic fix was attempted. This needs a developer." ;;
+              title=":rotating_light: *\`$CHECK_NAME\` is failing again*"
+              body="A heal PR for this check merged within the last $REPEAT_WINDOW_DAYS days, so no second automatic fix was attempted. This needs a developer." ;;
+            *:pr-failed)
+              title=":rotating_light: *\`$CHECK_NAME\` failed — fix made, but the PR could not be opened*"
+              body="The agent classified this as $CLASSIFICATION and made a fix, then opening the PR failed, so nothing reached review. If branch \`$BRANCH\` exists, open the PR from it; otherwise check the run logs and re-run the heal. Agent diagnosis: $SUMMARY" ;;
             *:healed)
-              text=":wrench: *Checkly: \`$CHECK_NAME\` failed — fix ready for review.* $SUMMARY\nPR: $PR_URL" ;;
+              title=":wrench: *\`$CHECK_NAME\` failed — fix ready for review*"
+              body="$SUMMARY" ;;
             *:flaky)
-              text=":wrench: *Checkly: \`$CHECK_NAME\` was flaky — test hardened, fix ready for review.* $SUMMARY\nPR: $PR_URL" ;;
+              title=":wrench: *\`$CHECK_NAME\` was flaky — test hardened, fix ready for review*"
+              body="$SUMMARY" ;;
             *:regression)
-              text=":rotating_light: *Checkly: \`$CHECK_NAME\` failed — production regression.* The test is correct; the site is broken. $SUMMARY" ;;
+              title=":rotating_light: *\`$CHECK_NAME\` failed — production regression*"
+              body="The test is correct; the site is broken. $SUMMARY" ;;
             *:needs-app-change)
-              text=":warning: *Checkly: \`$CHECK_NAME\` failed — fix needs an app code change.* $SUMMARY" ;;
+              title=":warning: *\`$CHECK_NAME\` failed — fix needs an app code change*"
+              body="$SUMMARY" ;;
             *:not-reproducible)
               # Transient blip, nothing to act on — summary only, no ping.
               echo "Not reproducible — no Slack notification." >> "$GITHUB_STEP_SUMMARY"
               exit 0 ;;
             *)
-              text=":rotating_light: *Checkly: \`$CHECK_NAME\` failed and the auto-heal run did not finish cleanly.* ${SUMMARY:-Check the run logs.} This needs a developer." ;;
+              title=":rotating_light: *\`$CHECK_NAME\` failed and the auto-heal run did not finish cleanly*"
+              body="${SUMMARY:-Check the run logs.} This needs a developer." ;;
           esac
-          # Routing tag: Slack Workflow Builder rules match the full bracketed
-          # tag ([e2e:<repo>]) to fan messages out from the shared e2e channel
-          # to project channels without substring collisions.
-          text="[e2e:${GITHUB_REPOSITORY##*/}] $text"
-          [ "$APP_CHANGE" = "true" ] && text="$text\nIncludes app-side test hooks — the PR's preview run validates them."
-          [ -n "$RESULT_LINK" ] && text="$text\nCheckly result: $RESULT_LINK"
+
+          # Repeat mode skips the agent, so there is no outcome classification;
+          # label it as what it is rather than defaulting to agent-failed.
+          label="${CLASSIFICATION:-agent-failed}"
+          [ "$MODE" = "repeat" ] && label="repeat-offender"
+          # agent-failed also renders the Re-run heal button.
+          [ "$outcome" = "pr-failed" ] && label="agent-failed"
+
+          # Failing test titles, best effort from the Checkly result's embedded
+          # Playwright report — the recursive search tolerates schema changes,
+          # and an empty result simply omits the field.
+          failed_tests=""
+          if [ -f .checkly-heal/result.json ]; then
+            failed_tests=$(jq -r '[.. | objects | select((.status? == "failed") or (.outcome? == "unexpected")) | .title? // empty] | unique | .[0:3] | join(", ")' .checkly-heal/result.json 2>/dev/null || true)
+          fi
+
+          # Plain-text fallback: what notifications, search, and the Slack
+          # routing rule see — the Workflow Builder rule matches the [e2e:<repo>]
+          # tag here and forwards this text, so it stays a complete message.
+          repo="${GITHUB_REPOSITORY##*/}"
+          text="[e2e:$repo] $title — $body"
+          [ -n "$PR_URL" ] && text="$text\nPR: $PR_URL"
           text="$text\nRun: $RUN_URL"
 
           if [ -z "$SLACK_WEBHOOK_URL" ]; then
@@ -339,7 +380,51 @@ jobs:
             printf '%b\n' "$text" >> "$GITHUB_STEP_SUMMARY"
             exit 0
           fi
-          jq -n --arg text "$(printf '%b' "$text")" '{text: $text}' \
+
+          # Block Kit body: title, classification/repo fields, failing tests
+          # when known, the diagnosis, and link buttons assembled per outcome —
+          # a button never renders without its target.
+          jq -n \
+            --arg text "$(printf '%b' "$text")" \
+            --arg title "$title" \
+            --arg body "$body" \
+            --arg classification "$label" \
+            --arg repo "$repo" \
+            --arg failed "$failed_tests" \
+            --arg app_change "$APP_CHANGE" \
+            --arg pr "$PR_URL" \
+            --arg run "$RUN_URL" \
+            --arg result "$RESULT_LINK" \
+            --arg rerun "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/workflows/checkly-heal.yml" \
+            '
+            def btn($label; $url): {type: "button", text: {type: "plain_text", text: $label}, url: $url};
+            {
+              text: $text,
+              blocks: (
+                [
+                  {type: "section", text: {type: "mrkdwn", text: $title}},
+                  {type: "section", fields: ([
+                    {type: "mrkdwn", text: ("*Classification*\n" + $classification)},
+                    {type: "mrkdwn", text: ("*Repository*\n" + $repo)}
+                  ] + (if $failed != "" then
+                    [{type: "mrkdwn", text: ("*Failed test*\n" + $failed)}]
+                  else [] end))}
+                ]
+                + (if $body != "" then
+                    [{type: "section", text: {type: "mrkdwn", text: $body}}]
+                  else [] end)
+                + (if $app_change == "true" then
+                    [{type: "context", elements: [{type: "mrkdwn", text: "Includes app-side test hooks — the PR'\''s preview run validates them."}]}]
+                  else [] end)
+                + [{type: "actions", elements: (
+                    (if $pr != "" then [btn("Review PR"; $pr)] else [] end)
+                    + [btn("View run"; $run)]
+                    + (if $result != "" then [btn("Checkly result"; $result)] else [] end)
+                    + (if $classification == "agent-failed" then [btn("Re-run heal"; $rerun)] else [] end)
+                  )}]
+                + [{type: "context", elements: [{type: "mrkdwn", text: ("[e2e:" + $repo + "]")}]}]
+              )
+            }' \
             | curl -sf --max-time 15 -X POST -H "Content-Type: application/json" -d @- "$SLACK_WEBHOOK_URL" \
             || echo "Slack notification failed — see summary." >> "$GITHUB_STEP_SUMMARY"