@andwhy/e2e-pipeline 0.13.1 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,22 +1,14 @@
1
1
  name: Checkly auto-heal
2
2
 
3
3
  # A failing Checkly monitor dispatches here (webhook alert channel →
4
- # repository_dispatch). The agent triages first: a stale or flaky test gets
5
- # fixed and opened as a human-review PR; a real production regression, a fix
6
- # needing behavioral app code, or any agent failure escalates to Slack
7
- # instead. Fixes may include behavior-neutral test hooks in app source
8
- # (data-testid) — the PR's own preview run (pr-playwright-tests.yml)
9
- # validates those; a diff guard aborts changes to protected paths
10
- # (.github, .claude, configs, lockfiles). The agent never pushes: branch,
11
- # commit, PR, and Slack are workflow steps.
4
+ # repository_dispatch). The triage-and-heal agent, the diff guard, the PR
5
+ # and the Slack notification live in the pipeline's reusable workflow, so a
6
+ # pipeline release updates them without a change here. This file owns the
7
+ # triggers, the permissions and the per-repo settings passed under `with`.
12
8
  # Needs repo secrets CHECKLY_API_KEY, CHECKLY_ACCOUNT_ID, SLACK_WEBHOOK_URL
13
- # (the org-shared e2e-channel webhook; values in 1Password); ANTHROPIC_API_KEY
14
- # is inherited from the organization (a repo secret of the same name
15
- # overrides it if the org value fails to authenticate). Slack messages
16
- # carry an [e2e:<repo>]
17
- # tag so Slack Workflow Builder rules route them from the shared channel to
18
- # project channels. Setup docs: the playwright-test-pipeline skill walks
19
- # through the Checkly webhook channel and the Slack rule.
9
+ # (values in 1Password); ANTHROPIC_API_KEY is inherited from the organization.
10
+ # Setup docs: the playwright-test-pipeline skill walks through the Checkly
11
+ # webhook channel and the Slack rule.
20
12
  on:
21
13
  repository_dispatch:
22
14
  types: [checkly-alert]
@@ -39,405 +31,18 @@ permissions:
39
31
  pull-requests: write
40
32
  issues: write
41
33
 
42
- # One heal at a time; queued alerts (e.g. an outage failing several checks)
43
- # run serially and the dedup below makes repeats cheap no-ops.
44
- concurrency:
45
- group: checkly-heal
46
- cancel-in-progress: false
47
-
48
- env:
49
- # The e2e test root. Changes outside it (behavior-neutral test hooks in app
50
- # source) are allowed but flagged on the PR and in Slack.
51
- # Match your playwright.config testDir.
52
- E2E_ROOT: tests/e2e
53
- # Paths the agent must never change — anything here aborts and escalates.
54
- # Extended regex against the changed-file list.
55
- PROTECTED_PATHS: '^(\.github/|\.claude/|\.mcp\.json$|.*\.config\.(ts|mjs|js|cjs)$|package\.json$|pnpm-lock\.yaml$|package-lock\.json$|yarn\.lock$|\.env)'
56
- # How long a merged heal PR blocks a second silent attempt on the same check.
57
- REPEAT_WINDOW_DAYS: 7
58
-
59
34
  jobs:
60
35
  heal:
61
- # Recovery/degraded notifications are not work; only failures are.
62
- if: >-
63
- github.event_name == 'workflow_dispatch' ||
64
- contains(github.event.client_payload.alert_type || 'ALERT_FAILURE', 'FAILURE')
65
- runs-on: ubuntu-latest
66
- timeout-minutes: 45
67
-
68
- steps:
69
- - name: Check out the default branch
70
- uses: actions/checkout@v5
71
- with:
72
- fetch-depth: 0
73
-
74
- # Dedup and repeat-offender guard. An open heal PR for this check means
75
- # the alert is already handled. A heal PR merged recently means the fix
76
- # did not hold — a second silent attempt would mask instability, so that
77
- # escalates to a human instead.
78
- - name: Guard against duplicate and repeated heals
79
- id: guard
80
- env:
81
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
82
- CHECK_NAME: ${{ github.event.client_payload.check_name || inputs.check_name }}
83
- run: |
84
- slug=$(printf '%s' "$CHECK_NAME" | tr -c 'a-zA-Z0-9' '-' | tr -s '-' | sed 's/^-//;s/-$//' | cut -c1-40 | tr '[:upper:]' '[:lower:]')
85
- echo "slug=${slug:-check}" >> "$GITHUB_OUTPUT"
86
-
87
- open=$(gh pr list --state open --label checkly-heal --json headRefName \
88
- --jq "[.[] | select(.headRefName | startswith(\"checkly-heal/$slug\"))] | length")
89
- if [ "$open" -gt 0 ]; then
90
- echo "mode=skip" >> "$GITHUB_OUTPUT"
91
- echo "A heal PR for this check is already open — nothing to do." >> "$GITHUB_STEP_SUMMARY"
92
- exit 0
93
- fi
94
-
95
- cutoff=$(date -u -d "$REPEAT_WINDOW_DAYS days ago" +%Y-%m-%dT%H:%M:%SZ)
96
- recent=$(gh pr list --state merged --label checkly-heal --json headRefName,mergedAt \
97
- --jq "[.[] | select((.headRefName | startswith(\"checkly-heal/$slug\")) and .mergedAt > \"$cutoff\")] | length")
98
- if [ "$recent" -gt 0 ]; then
99
- echo "mode=repeat" >> "$GITHUB_OUTPUT"
100
- exit 0
101
- fi
102
- echo "mode=heal" >> "$GITHUB_OUTPUT"
103
-
104
- - name: Set up pnpm
105
- if: steps.guard.outputs.mode == 'heal'
106
- uses: pnpm/action-setup@v4
107
-
108
- - name: Set up Node
109
- if: steps.guard.outputs.mode == 'heal'
110
- uses: actions/setup-node@v5
111
- with:
112
- node-version: 22
113
- cache: pnpm
114
-
115
- # --ignore-scripts: only the tests run here; project postinstall hooks
116
- # are not needed and may want CMS tokens CI does not have.
117
- - name: Install dependencies
118
- if: steps.guard.outputs.mode == 'heal'
119
- run: pnpm install --frozen-lockfile --ignore-scripts
120
-
121
- - name: Install Playwright browsers
122
- if: steps.guard.outputs.mode == 'heal'
123
- run: pnpm playwright install --with-deps chromium
124
-
125
- # Checkly already diagnosed the failure — error, failing step, trace.
126
- # Best effort: the agent reproduces from scratch when this is missing.
127
- - name: Fetch the Checkly result and write agent context
128
- if: steps.guard.outputs.mode == 'heal'
129
- env:
130
- CHECKLY_API_KEY: ${{ secrets.CHECKLY_API_KEY }}
131
- CHECKLY_ACCOUNT_ID: ${{ secrets.CHECKLY_ACCOUNT_ID }}
132
- CHECK_NAME: ${{ github.event.client_payload.check_name || inputs.check_name }}
133
- CHECK_ID: ${{ github.event.client_payload.check_id || inputs.check_id }}
134
- RESULT_ID: ${{ github.event.client_payload.result_id || inputs.result_id }}
135
- RESULT_LINK: ${{ github.event.client_payload.result_link }}
136
- ALERT_TYPE: ${{ github.event.client_payload.alert_type }}
137
- run: |
138
- mkdir -p .checkly-heal
139
- if [ -n "$CHECK_ID" ] && [ -n "$RESULT_ID" ]; then
140
- curl -sf --max-time 30 \
141
- -H "Authorization: Bearer $CHECKLY_API_KEY" \
142
- -H "X-Checkly-Account: $CHECKLY_ACCOUNT_ID" \
143
- "https://api.checklyhq.com/v1/check-results/$CHECK_ID/$RESULT_ID" \
144
- -o .checkly-heal/result.json \
145
- || echo "Checkly result fetch failed — agent reproduces from scratch."
146
- fi
147
- jq -n \
148
- --arg check_name "$CHECK_NAME" \
149
- --arg alert_type "${ALERT_TYPE:-ALERT_FAILURE}" \
150
- --arg result_link "${RESULT_LINK:-}" \
151
- --arg result_file "$([ -f .checkly-heal/result.json ] && echo .checkly-heal/result.json)" \
152
- '{check_name: $check_name, alert_type: $alert_type, result_link: $result_link, result_file: $result_file}' \
153
- > .checkly-heal/context.json
154
-
155
- # The agent gets test-fixing tools and Checkly credentials (for
156
- # pw-test verification) — deliberately no GitHub token: it cannot push,
157
- # comment, or open PRs even if it tries.
158
- - name: Run the triage-and-heal agent
159
- if: steps.guard.outputs.mode == 'heal'
160
- id: agent
161
- timeout-minutes: 35
162
- env:
163
- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
164
- CHECKLY_API_KEY: ${{ secrets.CHECKLY_API_KEY }}
165
- CHECKLY_ACCOUNT_ID: ${{ secrets.CHECKLY_ACCOUNT_ID }}
166
- run: |
167
- npm install -g @anthropic-ai/claude-code@2
168
- # Transcript lives OUTSIDE .checkly-heal so it survives whatever the
169
- # agent does to its working directory, and outside the repo tree so
170
- # the diff guard never counts it. stream-json writes each event as it
171
- # happens — a run killed by the timeout still leaves the full history
172
- # (plain json buffers everything and a kill leaves an empty file).
173
- claude -p "$(cat .github/prompts/checkly-heal.md)" \
174
- --model claude-opus-5 \
175
- --allowedTools "Bash,Read,Edit,Write,Glob,Grep" \
176
- --permission-mode dontAsk \
177
- --max-turns 150 \
178
- --output-format stream-json --verbose \
179
- > /tmp/checkly-heal-agent-output.json || echo "claude exited non-zero: $?" | tee -a "$GITHUB_STEP_SUMMARY"
180
- # Diagnostics go to the job log AND the summary: the log is readable
181
- # over the API, the summary only in the browser.
182
- {
183
- echo "### Agent step diagnostics"
184
- echo '```'
185
- ls -la .checkly-heal 2>&1 || echo ".checkly-heal missing"
186
- tail -c 2000 /tmp/checkly-heal-agent-output.json 2>/dev/null || echo "no agent output"
187
- echo '```'
188
- } | tee -a "$GITHUB_STEP_SUMMARY"
189
-
190
- # The guard is deterministic on purpose: the PR path requires an outcome
191
- # file claiming a fix AND a diff clear of protected paths. App-source
192
- # changes (test hooks) are allowed but derived from the diff, not the
193
- # agent's claim, and flagged downstream. Everything else — protected
194
- # paths, missing outcome, other classifications — escalates.
195
- # always(): a killed agent step (timeout, cancel) must still produce a
196
- # classification, or Slack reports an unexplained failure.
197
- - name: Read the outcome and guard the diff
198
- if: always() && steps.guard.outputs.mode == 'heal'
199
- id: outcome
200
- env:
201
- AGENT_CONCLUSION: ${{ steps.agent.conclusion }}
202
- run: |
203
- classification="agent-failed"
204
- summary="The heal agent did not produce an outcome."
205
- if [ "$AGENT_CONCLUSION" = "success" ] && [ -f .checkly-heal/outcome.json ] && jq -e . .checkly-heal/outcome.json > /dev/null 2>&1; then
206
- classification=$(jq -r '.classification // "agent-failed"' .checkly-heal/outcome.json)
207
- summary=$(jq -r '.summary // ""' .checkly-heal/outcome.json)
208
- else
209
- # No trustworthy outcome — say WHY, so the Slack message diagnoses
210
- # itself. An outcome written before a kill is unverified: escalate
211
- # rather than open a PR for it.
212
- if [ "$AGENT_CONCLUSION" = "skipped" ]; then
213
- summary="The heal agent never ran — a setup step failed before it."
214
- elif [ "$AGENT_CONCLUSION" != "success" ]; then
215
- summary="The heal agent was killed before finishing (step conclusion: ${AGENT_CONCLUSION:-unknown} — usually the 35-minute timeout)."
216
- fi
217
- agent_error=$(jq -r 'select(.is_error == true) | .result // empty' /tmp/checkly-heal-agent-output.json 2>/dev/null | head -c 300)
218
- [ -n "$agent_error" ] && summary="$summary Agent error: $agent_error"
219
- fi
220
-
221
- files=$(git status --porcelain | awk '{print $NF}' | grep -v "^\.checkly-heal/" || true)
222
- protected=$(printf '%s' "$files" | grep -Ec "$PROTECTED_PATHS" || true)
223
- changed=$(printf '%s' "$files" | grep -c . || true)
224
- app_change=$(printf '%s' "$files" | grep -cv "^$E2E_ROOT/" || true)
225
-
226
- if [ "$classification" = "healed" ] || [ "$classification" = "flaky" ]; then
227
- if [ "$protected" -gt 0 ]; then
228
- classification="agent-failed"
229
- summary="Diff guard tripped: protected files (.github, .claude, configs, lockfiles) were changed. Nothing was pushed."
230
- elif [ "$changed" -eq 0 ]; then
231
- classification="agent-failed"
232
- summary="The agent reported a fix but changed no files."
233
- fi
234
- fi
235
-
236
- echo "app_change=$([ "$app_change" -gt 0 ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
237
- echo "classification=$classification" >> "$GITHUB_OUTPUT"
238
- {
239
- echo "summary<<CHECKLY_HEAL_EOF"
240
- echo "$summary"
241
- echo "CHECKLY_HEAL_EOF"
242
- } >> "$GITHUB_OUTPUT"
243
- echo "## Checkly auto-heal: $classification" >> "$GITHUB_STEP_SUMMARY"
244
- echo "$summary" >> "$GITHUB_STEP_SUMMARY"
245
-
246
- - name: Open the heal PR for human review
247
- if: >-
248
- steps.guard.outputs.mode == 'heal' &&
249
- (steps.outcome.outputs.classification == 'healed' ||
250
- steps.outcome.outputs.classification == 'flaky')
251
- id: pr
252
- env:
253
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
254
- CHECK_NAME: ${{ github.event.client_payload.check_name || inputs.check_name }}
255
- RESULT_LINK: ${{ github.event.client_payload.result_link }}
256
- SLUG: ${{ steps.guard.outputs.slug }}
257
- CLASSIFICATION: ${{ steps.outcome.outputs.classification }}
258
- APP_CHANGE: ${{ steps.outcome.outputs.app_change }}
259
- run: |
260
- branch="checkly-heal/$SLUG-${{ github.run_id }}"
261
- git config user.name "github-actions[bot]"
262
- git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
263
- git checkout -b "$branch"
264
- # Stage everything, then unstage the agent's scratch dir. An
265
- # ':(exclude).checkly-heal' pathspec looks equivalent but exits 1
266
- # when the path is gitignored (as init makes it), which killed
267
- # every PR step before a commit existed.
268
- git add -A
269
- git reset -q -- .checkly-heal
270
- git commit -m "Heal Checkly check: $CHECK_NAME"
271
- git push -u origin "$branch"
272
-
273
- gh label create checkly-heal --force \
274
- --description "Automated fix for a failing Checkly monitor" \
275
- --color BFD4F2 || true
276
-
277
- case "$CLASSIFICATION" in
278
- flaky) kind="a flaky test (hardened, stability proven with repeated runs)" ;;
279
- *) kind="a stale test" ;;
280
- esac
281
- {
282
- echo "A Checkly monitor failed and the auto-heal agent classified it as $kind. **Human review required — do not merge unread.**"
283
- echo ""
284
- echo "- Check: \`$CHECK_NAME\`"
285
- [ -n "$RESULT_LINK" ] && echo "- Failing result: $RESULT_LINK"
286
- echo "- Heal run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
287
- if [ "$APP_CHANGE" = "true" ]; then
288
- echo ""
289
- echo "> [!IMPORTANT]"
290
- echo "> This fix includes app-source changes (behavior-neutral test hooks such as \`data-testid\`). Production does not carry them yet, so the authoritative validation is **this PR's own preview run** (pr-playwright-tests) — check it before merging."
291
- fi
292
- echo ""
293
- jq -r '.details // ""' .checkly-heal/outcome.json
294
- } > pr-body.md
295
-
296
- url=$(gh pr create \
297
- --title "Heal Checkly check: $CHECK_NAME" \
298
- --body-file pr-body.md \
299
- --label checkly-heal)
300
- echo "url=$url" >> "$GITHUB_OUTPUT"
301
-
302
- # The single alert path: every outcome that needs eyes lands in Slack,
303
- # written after triage. Runs on step failure too (always() + the guard
304
- # conditions), so a crashed agent still alerts.
305
- - name: Notify Slack
306
- if: always() && steps.guard.outputs.mode != 'skip'
307
- env:
308
- SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
309
- CHECK_NAME: ${{ github.event.client_payload.check_name || inputs.check_name }}
310
- RESULT_LINK: ${{ github.event.client_payload.result_link }}
311
- MODE: ${{ steps.guard.outputs.mode }}
312
- CLASSIFICATION: ${{ steps.outcome.outputs.classification }}
313
- SUMMARY: ${{ steps.outcome.outputs.summary }}
314
- APP_CHANGE: ${{ steps.outcome.outputs.app_change }}
315
- PR_URL: ${{ steps.pr.outputs.url }}
316
- RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
317
- BRANCH: checkly-heal/${{ steps.guard.outputs.slug }}-${{ github.run_id }}
318
- run: |
319
- # A fix only reaches review through a PR. When the PR step failed,
320
- # "fix ready for review" would be false: report the failure and
321
- # offer the re-run instead.
322
- outcome="$CLASSIFICATION"
323
- if { [ "$CLASSIFICATION" = "healed" ] || [ "$CLASSIFICATION" = "flaky" ]; } && [ -z "$PR_URL" ]; then
324
- outcome="pr-failed"
325
- fi
326
-
327
- case "${MODE}:${outcome}" in
328
- repeat:*)
329
- title=":rotating_light: *\`$CHECK_NAME\` is failing again*"
330
- body="A heal PR for this check merged within the last $REPEAT_WINDOW_DAYS days, so no second automatic fix was attempted. This needs a developer." ;;
331
- *:pr-failed)
332
- title=":rotating_light: *\`$CHECK_NAME\` failed — fix made, but the PR could not be opened*"
333
- body="The agent classified this as $CLASSIFICATION and made a fix, then opening the PR failed, so nothing reached review. If branch \`$BRANCH\` exists, open the PR from it; otherwise check the run logs and re-run the heal. Agent diagnosis: $SUMMARY" ;;
334
- *:healed)
335
- title=":wrench: *\`$CHECK_NAME\` failed — fix ready for review*"
336
- body="$SUMMARY" ;;
337
- *:flaky)
338
- title=":wrench: *\`$CHECK_NAME\` was flaky — test hardened, fix ready for review*"
339
- body="$SUMMARY" ;;
340
- *:regression)
341
- title=":rotating_light: *\`$CHECK_NAME\` failed — production regression*"
342
- body="The test is correct; the site is broken. $SUMMARY" ;;
343
- *:needs-app-change)
344
- title=":warning: *\`$CHECK_NAME\` failed — fix needs an app code change*"
345
- body="$SUMMARY" ;;
346
- *:not-reproducible)
347
- # Transient blip, nothing to act on — summary only, no ping.
348
- echo "Not reproducible — no Slack notification." >> "$GITHUB_STEP_SUMMARY"
349
- exit 0 ;;
350
- *)
351
- title=":rotating_light: *\`$CHECK_NAME\` failed and the auto-heal run did not finish cleanly*"
352
- body="${SUMMARY:-Check the run logs.} This needs a developer." ;;
353
- esac
354
-
355
- # Repeat mode skips the agent, so there is no outcome classification;
356
- # label it as what it is rather than defaulting to agent-failed.
357
- label="${CLASSIFICATION:-agent-failed}"
358
- [ "$MODE" = "repeat" ] && label="repeat-offender"
359
- # agent-failed also renders the Re-run heal button.
360
- [ "$outcome" = "pr-failed" ] && label="agent-failed"
361
-
362
- # Failing test titles, best effort from the Checkly result's embedded
363
- # Playwright report — the recursive search tolerates schema changes,
364
- # and an empty result simply omits the field.
365
- failed_tests=""
366
- if [ -f .checkly-heal/result.json ]; then
367
- failed_tests=$(jq -r '[.. | objects | select((.status? == "failed") or (.outcome? == "unexpected")) | .title? // empty] | unique | .[0:3] | join(", ")' .checkly-heal/result.json 2>/dev/null || true)
368
- fi
369
-
370
- # Plain-text fallback: what notifications, search, and the Slack
371
- # routing rule see — the Workflow Builder rule matches the [e2e:<repo>]
372
- # tag here and forwards this text, so it stays a complete message.
373
- repo="${GITHUB_REPOSITORY##*/}"
374
- text="[e2e:$repo] $title — $body"
375
- [ -n "$PR_URL" ] && text="$text\nPR: $PR_URL"
376
- text="$text\nRun: $RUN_URL"
377
-
378
- if [ -z "$SLACK_WEBHOOK_URL" ]; then
379
- echo "SLACK_WEBHOOK_URL is not set — notification skipped:" >> "$GITHUB_STEP_SUMMARY"
380
- printf '%b\n' "$text" >> "$GITHUB_STEP_SUMMARY"
381
- exit 0
382
- fi
383
-
384
- # Block Kit body: title, classification/repo fields, failing tests
385
- # when known, the diagnosis, and link buttons assembled per outcome —
386
- # a button never renders without its target.
387
- jq -n \
388
- --arg text "$(printf '%b' "$text")" \
389
- --arg title "$title" \
390
- --arg body "$body" \
391
- --arg classification "$label" \
392
- --arg repo "$repo" \
393
- --arg failed "$failed_tests" \
394
- --arg app_change "$APP_CHANGE" \
395
- --arg pr "$PR_URL" \
396
- --arg run "$RUN_URL" \
397
- --arg result "$RESULT_LINK" \
398
- --arg rerun "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/workflows/checkly-heal.yml" \
399
- '
400
- def btn($label; $url): {type: "button", text: {type: "plain_text", text: $label}, url: $url};
401
- {
402
- text: $text,
403
- blocks: (
404
- [
405
- {type: "section", text: {type: "mrkdwn", text: $title}},
406
- {type: "section", fields: ([
407
- {type: "mrkdwn", text: ("*Classification*\n" + $classification)},
408
- {type: "mrkdwn", text: ("*Repository*\n" + $repo)}
409
- ] + (if $failed != "" then
410
- [{type: "mrkdwn", text: ("*Failed test*\n" + $failed)}]
411
- else [] end))}
412
- ]
413
- + (if $body != "" then
414
- [{type: "section", text: {type: "mrkdwn", text: $body}}]
415
- else [] end)
416
- + (if $app_change == "true" then
417
- [{type: "context", elements: [{type: "mrkdwn", text: "Includes app-side test hooks — the PR'\''s preview run validates them."}]}]
418
- else [] end)
419
- + [{type: "actions", elements: (
420
- (if $pr != "" then [btn("Review PR"; $pr)] else [] end)
421
- + [btn("View run"; $run)]
422
- + (if $result != "" then [btn("Checkly result"; $result)] else [] end)
423
- + (if $classification == "agent-failed" then [btn("Re-run heal"; $rerun)] else [] end)
424
- )}]
425
- + [{type: "context", elements: [{type: "mrkdwn", text: ("[e2e:" + $repo + "]")}]}]
426
- )
427
- }' \
428
- | curl -sf --max-time 15 -X POST -H "Content-Type: application/json" -d @- "$SLACK_WEBHOOK_URL" \
429
- || echo "Slack notification failed — see summary." >> "$GITHUB_STEP_SUMMARY"
430
-
431
- - name: Upload agent artifacts
432
- if: always() && steps.guard.outputs.mode == 'heal'
433
- uses: actions/upload-artifact@v4
434
- with:
435
- name: checkly-heal-${{ github.run_id }}
436
- path: |
437
- .checkly-heal/
438
- /tmp/checkly-heal-agent-output.json
439
- # .checkly-heal is a dot-directory; without this it is silently
440
- # excluded and the debug artifact loses context/outcome/result.
441
- include-hidden-files: true
442
- if-no-files-found: ignore
443
- retention-days: 14
36
+ uses: why-agency/why-e2e-pipeline/.github/workflows/reusable-checkly-heal.yml@main
37
+ with:
38
+ check_name: ${{ github.event.client_payload.check_name || inputs.check_name }}
39
+ check_id: ${{ github.event.client_payload.check_id || inputs.check_id }}
40
+ result_id: ${{ github.event.client_payload.result_id || inputs.result_id }}
41
+ # Per-repo settings, defaults shown. Uncomment to change.
42
+ # e2e_root: tests/e2e # match your playwright.config testDir
43
+ # heal_commit_scope: tests # fix(<scope>): heal check "<name>"
44
+ # repeat_window_days: 7
45
+ # node_version: "22"
46
+ # pnpm_version: "" # set when package.json has no packageManager
47
+ # install_command: pnpm install --frozen-lockfile --ignore-scripts
48
+ secrets: inherit
@@ -1,14 +1,17 @@
1
1
  name: PR Playwright tests
2
2
 
3
- # Runs the suite against the PR's own Vercel preview. `deployment_status`
4
- # fires once the preview is live and carries its URL; the preview is built
5
- # from the PR branch, so the PR's changes exist on the target.
3
+ # Runs the suite against the PR's own Vercel preview and comments the result
4
+ # on the PR. The work happens in the pipeline's reusable workflow, so a
5
+ # pipeline release updates it without a change here. This file owns the
6
+ # trigger and the permissions only.
6
7
  # Needs repo secret VERCEL_AUTOMATION_BYPASS_SECRET when previews are
7
- # protected; the scaffolded helper turns it into Vercel's bypass cookie.
8
+ # protected. Optional repo variable PREVIEW_LOCALE_HOST: which production
9
+ # domain's locale config preview hosts resolve to.
8
10
  on:
9
11
  deployment_status
10
12
 
11
- # Vercel's payload has no branch, only the SHA — the job below resolves it.
13
+ # Vercel's payload has no branch, only the SHA — the reusable workflow
14
+ # resolves it.
12
15
  run-name: Playwright · ${{ github.event.deployment.environment }}
13
16
 
14
17
  permissions:
@@ -16,142 +19,13 @@ permissions:
16
19
  pull-requests: write
17
20
 
18
21
  jobs:
19
- resolve-branch:
20
- if: >-
21
- github.event.deployment_status.state == 'success' &&
22
- github.event.deployment.environment == 'Preview'
23
- runs-on: ubuntu-latest
24
- timeout-minutes: 2
25
- outputs:
26
- branch: ${{ steps.pr.outputs.branch }}
27
- label: ${{ steps.pr.outputs.label }}
28
- number: ${{ steps.pr.outputs.number }}
29
- steps:
30
- - name: Resolve the PR branch for this commit
31
- id: pr
32
- env:
33
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
34
- SHA: ${{ github.event.deployment.ref }}
35
- REPO: ${{ github.repository }}
36
- run: |
37
- # No associated PR (plain branch push) falls back to the SHA.
38
- pr=$(gh api "repos/$REPO/commits/$SHA/pulls" --jq \
39
- 'if length > 0 then "\(.[0].number)\t\(.[0].head.ref)\t\(.[0].title)" else "" end' \
40
- 2>/dev/null || true)
41
- branch=$(printf '%s' "$pr" | cut -f2)
42
- number=$(printf '%s' "$pr" | cut -f1)
43
- title=$(printf '%s' "$pr" | cut -f3)
44
-
45
- echo "branch=${branch:-$SHA}" >> "$GITHUB_OUTPUT"
46
- echo "number=${number}" >> "$GITHUB_OUTPUT"
47
- if [ -n "$number" ]; then
48
- echo "label=#${number} ${title}" >> "$GITHUB_OUTPUT"
49
- else
50
- echo "label=${branch:-$SHA}" >> "$GITHUB_OUTPUT"
51
- fi
52
-
53
22
  playwright:
54
- name: Playwright · ${{ needs.resolve-branch.outputs.label }}
55
- needs: resolve-branch
56
- # A re-push cancels the stale run instead of racing an outdated preview.
57
- concurrency:
58
- group: pr-playwright-${{ needs.resolve-branch.outputs.branch }}
59
- cancel-in-progress: true
60
- runs-on: ubuntu-latest
61
- # Red informs, branch protection decides — the check is not required.
62
- timeout-minutes: 10
63
-
64
- steps:
65
- - name: Check out the deployed ref
66
- uses: actions/checkout@v5
67
- with:
68
- ref: ${{ github.event.deployment.ref }}
69
-
70
- - name: Set up pnpm
71
- uses: pnpm/action-setup@v4
72
-
73
- - name: Set up Node
74
- uses: actions/setup-node@v5
75
- with:
76
- node-version: 22
77
- cache: pnpm
78
-
79
- # --ignore-scripts: only the tests run here; project postinstall hooks
80
- # are not needed and may want CMS tokens CI does not have.
81
- - name: Install dependencies
82
- run: pnpm install --frozen-lockfile --ignore-scripts
83
-
84
- - name: Install Playwright browsers
85
- run: pnpm playwright install --with-deps chromium
86
-
87
- # Chrome only for speed; Firefox/Edge run on Checkly against production.
88
- # --global-timeout instead of a step timeout so reports still flush.
89
- - name: Run Playwright tests against the preview
90
- id: tests
91
- timeout-minutes: 9
92
- env:
93
- BASE_URL: ${{ github.event.deployment_status.target_url }}
94
- VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}
95
- # Which production domain's locale config preview hosts resolve to
96
- # (repo variable, optional — specs default to the primary domain).
97
- PREVIEW_LOCALE_HOST: ${{ vars.PREVIEW_LOCALE_HOST }}
98
- PLAYWRIGHT_JSON_OUTPUT_NAME: playwright-results.json
99
- run: |
100
- pnpm playwright test \
101
- --project=Chrome \
102
- --global-timeout 360000 \
103
- --reporter=list,html,json
104
-
105
- - name: Upload HTML report
106
- if: always()
107
- uses: actions/upload-artifact@v4
108
- with:
109
- name: playwright-report-${{ github.run_id }}
110
- path: |
111
- playwright-report/
112
- playwright-results.json
113
- if-no-files-found: warn
114
- retention-days: 7
115
-
116
- # Reporting only — must not turn the check red.
117
- - name: Comment the result on the PR
118
- if: always() && needs.resolve-branch.outputs.number != ''
119
- continue-on-error: true
120
- env:
121
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
122
- REPO: ${{ github.repository }}
123
- PR: ${{ needs.resolve-branch.outputs.number }}
124
- PREVIEW_URL: ${{ github.event.deployment_status.target_url }}
125
- RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
126
- run: |
127
- node .github/scripts/playwright-pr-comment.mjs \
128
- playwright-results.json > comment.md
129
-
130
- # Update the existing comment so re-pushes do not stack.
131
- marker='<!-- playwright-preview-report -->'
132
- id=$(gh api "repos/$REPO/issues/$PR/comments" --paginate \
133
- --jq "map(select(.body | startswith(\"$marker\"))) | .[0].id // empty")
134
-
135
- if [ -n "$id" ]; then
136
- gh api -X PATCH "repos/$REPO/issues/comments/$id" -F body=@comment.md
137
- else
138
- gh api -X POST "repos/$REPO/issues/$PR/comments" -F body=@comment.md
139
- fi
140
-
141
- - name: Write result to job summary
142
- if: always()
143
- env:
144
- OUTCOME: ${{ steps.tests.outcome }}
145
- PREVIEW_URL: ${{ github.event.deployment_status.target_url }}
146
- run: |
147
- {
148
- echo "## Playwright (preview)"
149
- echo ""
150
- echo "- Target: $PREVIEW_URL"
151
- echo "- Browser: Chrome only"
152
- if [ "$OUTCOME" = "success" ]; then
153
- echo "- Result: all tests passed"
154
- else
155
- echo "- Result: **failures present** — see the uploaded \`playwright-report\` artifact"
156
- fi
157
- } >> "$GITHUB_STEP_SUMMARY"
23
+ uses: why-agency/why-e2e-pipeline/.github/workflows/reusable-pr-playwright-tests.yml@main
24
+ with:
25
+ preview_locale_host: ${{ vars.PREVIEW_LOCALE_HOST }}
26
+ # Per-repo settings, defaults shown. Uncomment to change.
27
+ # node_version: "22" # match the repo's engines/.nvmrc
28
+ # pnpm_version: "" # set when package.json has no packageManager
29
+ # install_command: pnpm install --frozen-lockfile --ignore-scripts
30
+ # skip_branches: "" # e.g. "review|development" for environment branches
31
+ secrets: inherit