@andwhy/e2e-pipeline 0.13.1 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -10
- package/dist/bin/init.js +29 -5
- package/dist/utils/copy.d.ts +15 -1
- package/dist/utils/copy.js +152 -91
- package/package.json +6 -3
- package/plugin/.claude-plugin/plugin.json +18 -0
- package/plugin/agents/playwright-test-generator.md +273 -0
- package/plugin/agents/playwright-test-healer.md +159 -0
- package/plugin/agents/playwright-test-planner.md +135 -0
- package/plugin/hooks/hooks.json +16 -0
- package/plugin/hooks/session-context.md +9 -0
- package/plugin/hooks/session-start.mjs +49 -0
- package/plugin/skills/playwright-test-pipeline/SKILL.md +545 -0
- package/templates/github/prompts/checkly-heal.md +10 -3
- package/templates/github/workflows/checkly-deploy.yml +19 -52
- package/templates/github/workflows/checkly-heal.yml +20 -415
- package/templates/github/workflows/pr-playwright-tests.yml +17 -143
- package/templates/claude/rules/playwright-workflow.md +0 -219
- package/templates/claude/skills/playwright-test-pipeline/SKILL.md +0 -286
|
@@ -1,22 +1,14 @@
|
|
|
1
1
|
name: Checkly auto-heal
|
|
2
2
|
|
|
3
3
|
# A failing Checkly monitor dispatches here (webhook alert channel →
|
|
4
|
-
# repository_dispatch). The agent
|
|
5
|
-
#
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
# (data-testid) — the PR's own preview run (pr-playwright-tests.yml)
|
|
9
|
-
# validates those; a diff guard aborts changes to protected paths
|
|
10
|
-
# (.github, .claude, configs, lockfiles). The agent never pushes: branch,
|
|
11
|
-
# commit, PR, and Slack are workflow steps.
|
|
4
|
+
# repository_dispatch). The triage-and-heal agent, the diff guard, the PR
|
|
5
|
+
# and the Slack notification live in the pipeline's reusable workflow, so a
|
|
6
|
+
# pipeline release updates them without a change here. This file owns the
|
|
7
|
+
# triggers, the permissions and the per-repo settings passed under `with`.
|
|
12
8
|
# Needs repo secrets CHECKLY_API_KEY, CHECKLY_ACCOUNT_ID, SLACK_WEBHOOK_URL
|
|
13
|
-
# (
|
|
14
|
-
#
|
|
15
|
-
#
|
|
16
|
-
# carry an [e2e:<repo>]
|
|
17
|
-
# tag so Slack Workflow Builder rules route them from the shared channel to
|
|
18
|
-
# project channels. Setup docs: the playwright-test-pipeline skill walks
|
|
19
|
-
# through the Checkly webhook channel and the Slack rule.
|
|
9
|
+
# (values in 1Password); ANTHROPIC_API_KEY is inherited from the organization.
|
|
10
|
+
# Setup docs: the playwright-test-pipeline skill walks through the Checkly
|
|
11
|
+
# webhook channel and the Slack rule.
|
|
20
12
|
on:
|
|
21
13
|
repository_dispatch:
|
|
22
14
|
types: [checkly-alert]
|
|
@@ -39,405 +31,18 @@ permissions:
|
|
|
39
31
|
pull-requests: write
|
|
40
32
|
issues: write
|
|
41
33
|
|
|
42
|
-
# One heal at a time; queued alerts (e.g. an outage failing several checks)
|
|
43
|
-
# run serially and the dedup below makes repeats cheap no-ops.
|
|
44
|
-
concurrency:
|
|
45
|
-
group: checkly-heal
|
|
46
|
-
cancel-in-progress: false
|
|
47
|
-
|
|
48
|
-
env:
|
|
49
|
-
# The e2e test root. Changes outside it (behavior-neutral test hooks in app
|
|
50
|
-
# source) are allowed but flagged on the PR and in Slack.
|
|
51
|
-
# Match your playwright.config testDir.
|
|
52
|
-
E2E_ROOT: tests/e2e
|
|
53
|
-
# Paths the agent must never change — anything here aborts and escalates.
|
|
54
|
-
# Extended regex against the changed-file list.
|
|
55
|
-
PROTECTED_PATHS: '^(\.github/|\.claude/|\.mcp\.json$|.*\.config\.(ts|mjs|js|cjs)$|package\.json$|pnpm-lock\.yaml$|package-lock\.json$|yarn\.lock$|\.env)'
|
|
56
|
-
# How long a merged heal PR blocks a second silent attempt on the same check.
|
|
57
|
-
REPEAT_WINDOW_DAYS: 7
|
|
58
|
-
|
|
59
34
|
jobs:
|
|
60
35
|
heal:
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
github.
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
# Dedup and repeat-offender guard. An open heal PR for this check means
|
|
75
|
-
# the alert is already handled. A heal PR merged recently means the fix
|
|
76
|
-
# did not hold — a second silent attempt would mask instability, so that
|
|
77
|
-
# escalates to a human instead.
|
|
78
|
-
- name: Guard against duplicate and repeated heals
|
|
79
|
-
id: guard
|
|
80
|
-
env:
|
|
81
|
-
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
82
|
-
CHECK_NAME: ${{ github.event.client_payload.check_name || inputs.check_name }}
|
|
83
|
-
run: |
|
|
84
|
-
slug=$(printf '%s' "$CHECK_NAME" | tr -c 'a-zA-Z0-9' '-' | tr -s '-' | sed 's/^-//;s/-$//' | cut -c1-40 | tr '[:upper:]' '[:lower:]')
|
|
85
|
-
echo "slug=${slug:-check}" >> "$GITHUB_OUTPUT"
|
|
86
|
-
|
|
87
|
-
open=$(gh pr list --state open --label checkly-heal --json headRefName \
|
|
88
|
-
--jq "[.[] | select(.headRefName | startswith(\"checkly-heal/$slug\"))] | length")
|
|
89
|
-
if [ "$open" -gt 0 ]; then
|
|
90
|
-
echo "mode=skip" >> "$GITHUB_OUTPUT"
|
|
91
|
-
echo "A heal PR for this check is already open — nothing to do." >> "$GITHUB_STEP_SUMMARY"
|
|
92
|
-
exit 0
|
|
93
|
-
fi
|
|
94
|
-
|
|
95
|
-
cutoff=$(date -u -d "$REPEAT_WINDOW_DAYS days ago" +%Y-%m-%dT%H:%M:%SZ)
|
|
96
|
-
recent=$(gh pr list --state merged --label checkly-heal --json headRefName,mergedAt \
|
|
97
|
-
--jq "[.[] | select((.headRefName | startswith(\"checkly-heal/$slug\")) and .mergedAt > \"$cutoff\")] | length")
|
|
98
|
-
if [ "$recent" -gt 0 ]; then
|
|
99
|
-
echo "mode=repeat" >> "$GITHUB_OUTPUT"
|
|
100
|
-
exit 0
|
|
101
|
-
fi
|
|
102
|
-
echo "mode=heal" >> "$GITHUB_OUTPUT"
|
|
103
|
-
|
|
104
|
-
- name: Set up pnpm
|
|
105
|
-
if: steps.guard.outputs.mode == 'heal'
|
|
106
|
-
uses: pnpm/action-setup@v4
|
|
107
|
-
|
|
108
|
-
- name: Set up Node
|
|
109
|
-
if: steps.guard.outputs.mode == 'heal'
|
|
110
|
-
uses: actions/setup-node@v5
|
|
111
|
-
with:
|
|
112
|
-
node-version: 22
|
|
113
|
-
cache: pnpm
|
|
114
|
-
|
|
115
|
-
# --ignore-scripts: only the tests run here; project postinstall hooks
|
|
116
|
-
# are not needed and may want CMS tokens CI does not have.
|
|
117
|
-
- name: Install dependencies
|
|
118
|
-
if: steps.guard.outputs.mode == 'heal'
|
|
119
|
-
run: pnpm install --frozen-lockfile --ignore-scripts
|
|
120
|
-
|
|
121
|
-
- name: Install Playwright browsers
|
|
122
|
-
if: steps.guard.outputs.mode == 'heal'
|
|
123
|
-
run: pnpm playwright install --with-deps chromium
|
|
124
|
-
|
|
125
|
-
# Checkly already diagnosed the failure — error, failing step, trace.
|
|
126
|
-
# Best effort: the agent reproduces from scratch when this is missing.
|
|
127
|
-
- name: Fetch the Checkly result and write agent context
|
|
128
|
-
if: steps.guard.outputs.mode == 'heal'
|
|
129
|
-
env:
|
|
130
|
-
CHECKLY_API_KEY: ${{ secrets.CHECKLY_API_KEY }}
|
|
131
|
-
CHECKLY_ACCOUNT_ID: ${{ secrets.CHECKLY_ACCOUNT_ID }}
|
|
132
|
-
CHECK_NAME: ${{ github.event.client_payload.check_name || inputs.check_name }}
|
|
133
|
-
CHECK_ID: ${{ github.event.client_payload.check_id || inputs.check_id }}
|
|
134
|
-
RESULT_ID: ${{ github.event.client_payload.result_id || inputs.result_id }}
|
|
135
|
-
RESULT_LINK: ${{ github.event.client_payload.result_link }}
|
|
136
|
-
ALERT_TYPE: ${{ github.event.client_payload.alert_type }}
|
|
137
|
-
run: |
|
|
138
|
-
mkdir -p .checkly-heal
|
|
139
|
-
if [ -n "$CHECK_ID" ] && [ -n "$RESULT_ID" ]; then
|
|
140
|
-
curl -sf --max-time 30 \
|
|
141
|
-
-H "Authorization: Bearer $CHECKLY_API_KEY" \
|
|
142
|
-
-H "X-Checkly-Account: $CHECKLY_ACCOUNT_ID" \
|
|
143
|
-
"https://api.checklyhq.com/v1/check-results/$CHECK_ID/$RESULT_ID" \
|
|
144
|
-
-o .checkly-heal/result.json \
|
|
145
|
-
|| echo "Checkly result fetch failed — agent reproduces from scratch."
|
|
146
|
-
fi
|
|
147
|
-
jq -n \
|
|
148
|
-
--arg check_name "$CHECK_NAME" \
|
|
149
|
-
--arg alert_type "${ALERT_TYPE:-ALERT_FAILURE}" \
|
|
150
|
-
--arg result_link "${RESULT_LINK:-}" \
|
|
151
|
-
--arg result_file "$([ -f .checkly-heal/result.json ] && echo .checkly-heal/result.json)" \
|
|
152
|
-
'{check_name: $check_name, alert_type: $alert_type, result_link: $result_link, result_file: $result_file}' \
|
|
153
|
-
> .checkly-heal/context.json
|
|
154
|
-
|
|
155
|
-
# The agent gets test-fixing tools and Checkly credentials (for
|
|
156
|
-
# pw-test verification) — deliberately no GitHub token: it cannot push,
|
|
157
|
-
# comment, or open PRs even if it tries.
|
|
158
|
-
- name: Run the triage-and-heal agent
|
|
159
|
-
if: steps.guard.outputs.mode == 'heal'
|
|
160
|
-
id: agent
|
|
161
|
-
timeout-minutes: 35
|
|
162
|
-
env:
|
|
163
|
-
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
164
|
-
CHECKLY_API_KEY: ${{ secrets.CHECKLY_API_KEY }}
|
|
165
|
-
CHECKLY_ACCOUNT_ID: ${{ secrets.CHECKLY_ACCOUNT_ID }}
|
|
166
|
-
run: |
|
|
167
|
-
npm install -g @anthropic-ai/claude-code@2
|
|
168
|
-
# Transcript lives OUTSIDE .checkly-heal so it survives whatever the
|
|
169
|
-
# agent does to its working directory, and outside the repo tree so
|
|
170
|
-
# the diff guard never counts it. stream-json writes each event as it
|
|
171
|
-
# happens — a run killed by the timeout still leaves the full history
|
|
172
|
-
# (plain json buffers everything and a kill leaves an empty file).
|
|
173
|
-
claude -p "$(cat .github/prompts/checkly-heal.md)" \
|
|
174
|
-
--model claude-opus-5 \
|
|
175
|
-
--allowedTools "Bash,Read,Edit,Write,Glob,Grep" \
|
|
176
|
-
--permission-mode dontAsk \
|
|
177
|
-
--max-turns 150 \
|
|
178
|
-
--output-format stream-json --verbose \
|
|
179
|
-
> /tmp/checkly-heal-agent-output.json || echo "claude exited non-zero: $?" | tee -a "$GITHUB_STEP_SUMMARY"
|
|
180
|
-
# Diagnostics go to the job log AND the summary: the log is readable
|
|
181
|
-
# over the API, the summary only in the browser.
|
|
182
|
-
{
|
|
183
|
-
echo "### Agent step diagnostics"
|
|
184
|
-
echo '```'
|
|
185
|
-
ls -la .checkly-heal 2>&1 || echo ".checkly-heal missing"
|
|
186
|
-
tail -c 2000 /tmp/checkly-heal-agent-output.json 2>/dev/null || echo "no agent output"
|
|
187
|
-
echo '```'
|
|
188
|
-
} | tee -a "$GITHUB_STEP_SUMMARY"
|
|
189
|
-
|
|
190
|
-
# The guard is deterministic on purpose: the PR path requires an outcome
|
|
191
|
-
# file claiming a fix AND a diff clear of protected paths. App-source
|
|
192
|
-
# changes (test hooks) are allowed but derived from the diff, not the
|
|
193
|
-
# agent's claim, and flagged downstream. Everything else — protected
|
|
194
|
-
# paths, missing outcome, other classifications — escalates.
|
|
195
|
-
# always(): a killed agent step (timeout, cancel) must still produce a
|
|
196
|
-
# classification, or Slack reports an unexplained failure.
|
|
197
|
-
- name: Read the outcome and guard the diff
|
|
198
|
-
if: always() && steps.guard.outputs.mode == 'heal'
|
|
199
|
-
id: outcome
|
|
200
|
-
env:
|
|
201
|
-
AGENT_CONCLUSION: ${{ steps.agent.conclusion }}
|
|
202
|
-
run: |
|
|
203
|
-
classification="agent-failed"
|
|
204
|
-
summary="The heal agent did not produce an outcome."
|
|
205
|
-
if [ "$AGENT_CONCLUSION" = "success" ] && [ -f .checkly-heal/outcome.json ] && jq -e . .checkly-heal/outcome.json > /dev/null 2>&1; then
|
|
206
|
-
classification=$(jq -r '.classification // "agent-failed"' .checkly-heal/outcome.json)
|
|
207
|
-
summary=$(jq -r '.summary // ""' .checkly-heal/outcome.json)
|
|
208
|
-
else
|
|
209
|
-
# No trustworthy outcome — say WHY, so the Slack message diagnoses
|
|
210
|
-
# itself. An outcome written before a kill is unverified: escalate
|
|
211
|
-
# rather than open a PR for it.
|
|
212
|
-
if [ "$AGENT_CONCLUSION" = "skipped" ]; then
|
|
213
|
-
summary="The heal agent never ran — a setup step failed before it."
|
|
214
|
-
elif [ "$AGENT_CONCLUSION" != "success" ]; then
|
|
215
|
-
summary="The heal agent was killed before finishing (step conclusion: ${AGENT_CONCLUSION:-unknown} — usually the 35-minute timeout)."
|
|
216
|
-
fi
|
|
217
|
-
agent_error=$(jq -r 'select(.is_error == true) | .result // empty' /tmp/checkly-heal-agent-output.json 2>/dev/null | head -c 300)
|
|
218
|
-
[ -n "$agent_error" ] && summary="$summary Agent error: $agent_error"
|
|
219
|
-
fi
|
|
220
|
-
|
|
221
|
-
files=$(git status --porcelain | awk '{print $NF}' | grep -v "^\.checkly-heal/" || true)
|
|
222
|
-
protected=$(printf '%s' "$files" | grep -Ec "$PROTECTED_PATHS" || true)
|
|
223
|
-
changed=$(printf '%s' "$files" | grep -c . || true)
|
|
224
|
-
app_change=$(printf '%s' "$files" | grep -cv "^$E2E_ROOT/" || true)
|
|
225
|
-
|
|
226
|
-
if [ "$classification" = "healed" ] || [ "$classification" = "flaky" ]; then
|
|
227
|
-
if [ "$protected" -gt 0 ]; then
|
|
228
|
-
classification="agent-failed"
|
|
229
|
-
summary="Diff guard tripped: protected files (.github, .claude, configs, lockfiles) were changed. Nothing was pushed."
|
|
230
|
-
elif [ "$changed" -eq 0 ]; then
|
|
231
|
-
classification="agent-failed"
|
|
232
|
-
summary="The agent reported a fix but changed no files."
|
|
233
|
-
fi
|
|
234
|
-
fi
|
|
235
|
-
|
|
236
|
-
echo "app_change=$([ "$app_change" -gt 0 ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
|
237
|
-
echo "classification=$classification" >> "$GITHUB_OUTPUT"
|
|
238
|
-
{
|
|
239
|
-
echo "summary<<CHECKLY_HEAL_EOF"
|
|
240
|
-
echo "$summary"
|
|
241
|
-
echo "CHECKLY_HEAL_EOF"
|
|
242
|
-
} >> "$GITHUB_OUTPUT"
|
|
243
|
-
echo "## Checkly auto-heal: $classification" >> "$GITHUB_STEP_SUMMARY"
|
|
244
|
-
echo "$summary" >> "$GITHUB_STEP_SUMMARY"
|
|
245
|
-
|
|
246
|
-
- name: Open the heal PR for human review
|
|
247
|
-
if: >-
|
|
248
|
-
steps.guard.outputs.mode == 'heal' &&
|
|
249
|
-
(steps.outcome.outputs.classification == 'healed' ||
|
|
250
|
-
steps.outcome.outputs.classification == 'flaky')
|
|
251
|
-
id: pr
|
|
252
|
-
env:
|
|
253
|
-
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
254
|
-
CHECK_NAME: ${{ github.event.client_payload.check_name || inputs.check_name }}
|
|
255
|
-
RESULT_LINK: ${{ github.event.client_payload.result_link }}
|
|
256
|
-
SLUG: ${{ steps.guard.outputs.slug }}
|
|
257
|
-
CLASSIFICATION: ${{ steps.outcome.outputs.classification }}
|
|
258
|
-
APP_CHANGE: ${{ steps.outcome.outputs.app_change }}
|
|
259
|
-
run: |
|
|
260
|
-
branch="checkly-heal/$SLUG-${{ github.run_id }}"
|
|
261
|
-
git config user.name "github-actions[bot]"
|
|
262
|
-
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
263
|
-
git checkout -b "$branch"
|
|
264
|
-
# Stage everything, then unstage the agent's scratch dir. An
|
|
265
|
-
# ':(exclude).checkly-heal' pathspec looks equivalent but exits 1
|
|
266
|
-
# when the path is gitignored (as init makes it), which killed
|
|
267
|
-
# every PR step before a commit existed.
|
|
268
|
-
git add -A
|
|
269
|
-
git reset -q -- .checkly-heal
|
|
270
|
-
git commit -m "Heal Checkly check: $CHECK_NAME"
|
|
271
|
-
git push -u origin "$branch"
|
|
272
|
-
|
|
273
|
-
gh label create checkly-heal --force \
|
|
274
|
-
--description "Automated fix for a failing Checkly monitor" \
|
|
275
|
-
--color BFD4F2 || true
|
|
276
|
-
|
|
277
|
-
case "$CLASSIFICATION" in
|
|
278
|
-
flaky) kind="a flaky test (hardened, stability proven with repeated runs)" ;;
|
|
279
|
-
*) kind="a stale test" ;;
|
|
280
|
-
esac
|
|
281
|
-
{
|
|
282
|
-
echo "A Checkly monitor failed and the auto-heal agent classified it as $kind. **Human review required — do not merge unread.**"
|
|
283
|
-
echo ""
|
|
284
|
-
echo "- Check: \`$CHECK_NAME\`"
|
|
285
|
-
[ -n "$RESULT_LINK" ] && echo "- Failing result: $RESULT_LINK"
|
|
286
|
-
echo "- Heal run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
|
287
|
-
if [ "$APP_CHANGE" = "true" ]; then
|
|
288
|
-
echo ""
|
|
289
|
-
echo "> [!IMPORTANT]"
|
|
290
|
-
echo "> This fix includes app-source changes (behavior-neutral test hooks such as \`data-testid\`). Production does not carry them yet, so the authoritative validation is **this PR's own preview run** (pr-playwright-tests) — check it before merging."
|
|
291
|
-
fi
|
|
292
|
-
echo ""
|
|
293
|
-
jq -r '.details // ""' .checkly-heal/outcome.json
|
|
294
|
-
} > pr-body.md
|
|
295
|
-
|
|
296
|
-
url=$(gh pr create \
|
|
297
|
-
--title "Heal Checkly check: $CHECK_NAME" \
|
|
298
|
-
--body-file pr-body.md \
|
|
299
|
-
--label checkly-heal)
|
|
300
|
-
echo "url=$url" >> "$GITHUB_OUTPUT"
|
|
301
|
-
|
|
302
|
-
# The single alert path: every outcome that needs eyes lands in Slack,
|
|
303
|
-
# written after triage. Runs on step failure too (always() + the guard
|
|
304
|
-
# conditions), so a crashed agent still alerts.
|
|
305
|
-
- name: Notify Slack
|
|
306
|
-
if: always() && steps.guard.outputs.mode != 'skip'
|
|
307
|
-
env:
|
|
308
|
-
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
|
309
|
-
CHECK_NAME: ${{ github.event.client_payload.check_name || inputs.check_name }}
|
|
310
|
-
RESULT_LINK: ${{ github.event.client_payload.result_link }}
|
|
311
|
-
MODE: ${{ steps.guard.outputs.mode }}
|
|
312
|
-
CLASSIFICATION: ${{ steps.outcome.outputs.classification }}
|
|
313
|
-
SUMMARY: ${{ steps.outcome.outputs.summary }}
|
|
314
|
-
APP_CHANGE: ${{ steps.outcome.outputs.app_change }}
|
|
315
|
-
PR_URL: ${{ steps.pr.outputs.url }}
|
|
316
|
-
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
317
|
-
BRANCH: checkly-heal/${{ steps.guard.outputs.slug }}-${{ github.run_id }}
|
|
318
|
-
run: |
|
|
319
|
-
# A fix only reaches review through a PR. When the PR step failed,
|
|
320
|
-
# "fix ready for review" would be false: report the failure and
|
|
321
|
-
# offer the re-run instead.
|
|
322
|
-
outcome="$CLASSIFICATION"
|
|
323
|
-
if { [ "$CLASSIFICATION" = "healed" ] || [ "$CLASSIFICATION" = "flaky" ]; } && [ -z "$PR_URL" ]; then
|
|
324
|
-
outcome="pr-failed"
|
|
325
|
-
fi
|
|
326
|
-
|
|
327
|
-
case "${MODE}:${outcome}" in
|
|
328
|
-
repeat:*)
|
|
329
|
-
title=":rotating_light: *\`$CHECK_NAME\` is failing again*"
|
|
330
|
-
body="A heal PR for this check merged within the last $REPEAT_WINDOW_DAYS days, so no second automatic fix was attempted. This needs a developer." ;;
|
|
331
|
-
*:pr-failed)
|
|
332
|
-
title=":rotating_light: *\`$CHECK_NAME\` failed — fix made, but the PR could not be opened*"
|
|
333
|
-
body="The agent classified this as $CLASSIFICATION and made a fix, then opening the PR failed, so nothing reached review. If branch \`$BRANCH\` exists, open the PR from it; otherwise check the run logs and re-run the heal. Agent diagnosis: $SUMMARY" ;;
|
|
334
|
-
*:healed)
|
|
335
|
-
title=":wrench: *\`$CHECK_NAME\` failed — fix ready for review*"
|
|
336
|
-
body="$SUMMARY" ;;
|
|
337
|
-
*:flaky)
|
|
338
|
-
title=":wrench: *\`$CHECK_NAME\` was flaky — test hardened, fix ready for review*"
|
|
339
|
-
body="$SUMMARY" ;;
|
|
340
|
-
*:regression)
|
|
341
|
-
title=":rotating_light: *\`$CHECK_NAME\` failed — production regression*"
|
|
342
|
-
body="The test is correct; the site is broken. $SUMMARY" ;;
|
|
343
|
-
*:needs-app-change)
|
|
344
|
-
title=":warning: *\`$CHECK_NAME\` failed — fix needs an app code change*"
|
|
345
|
-
body="$SUMMARY" ;;
|
|
346
|
-
*:not-reproducible)
|
|
347
|
-
# Transient blip, nothing to act on — summary only, no ping.
|
|
348
|
-
echo "Not reproducible — no Slack notification." >> "$GITHUB_STEP_SUMMARY"
|
|
349
|
-
exit 0 ;;
|
|
350
|
-
*)
|
|
351
|
-
title=":rotating_light: *\`$CHECK_NAME\` failed and the auto-heal run did not finish cleanly*"
|
|
352
|
-
body="${SUMMARY:-Check the run logs.} This needs a developer." ;;
|
|
353
|
-
esac
|
|
354
|
-
|
|
355
|
-
# Repeat mode skips the agent, so there is no outcome classification;
|
|
356
|
-
# label it as what it is rather than defaulting to agent-failed.
|
|
357
|
-
label="${CLASSIFICATION:-agent-failed}"
|
|
358
|
-
[ "$MODE" = "repeat" ] && label="repeat-offender"
|
|
359
|
-
# agent-failed also renders the Re-run heal button.
|
|
360
|
-
[ "$outcome" = "pr-failed" ] && label="agent-failed"
|
|
361
|
-
|
|
362
|
-
# Failing test titles, best effort from the Checkly result's embedded
|
|
363
|
-
# Playwright report — the recursive search tolerates schema changes,
|
|
364
|
-
# and an empty result simply omits the field.
|
|
365
|
-
failed_tests=""
|
|
366
|
-
if [ -f .checkly-heal/result.json ]; then
|
|
367
|
-
failed_tests=$(jq -r '[.. | objects | select((.status? == "failed") or (.outcome? == "unexpected")) | .title? // empty] | unique | .[0:3] | join(", ")' .checkly-heal/result.json 2>/dev/null || true)
|
|
368
|
-
fi
|
|
369
|
-
|
|
370
|
-
# Plain-text fallback: what notifications, search, and the Slack
|
|
371
|
-
# routing rule see — the Workflow Builder rule matches the [e2e:<repo>]
|
|
372
|
-
# tag here and forwards this text, so it stays a complete message.
|
|
373
|
-
repo="${GITHUB_REPOSITORY##*/}"
|
|
374
|
-
text="[e2e:$repo] $title — $body"
|
|
375
|
-
[ -n "$PR_URL" ] && text="$text\nPR: $PR_URL"
|
|
376
|
-
text="$text\nRun: $RUN_URL"
|
|
377
|
-
|
|
378
|
-
if [ -z "$SLACK_WEBHOOK_URL" ]; then
|
|
379
|
-
echo "SLACK_WEBHOOK_URL is not set — notification skipped:" >> "$GITHUB_STEP_SUMMARY"
|
|
380
|
-
printf '%b\n' "$text" >> "$GITHUB_STEP_SUMMARY"
|
|
381
|
-
exit 0
|
|
382
|
-
fi
|
|
383
|
-
|
|
384
|
-
# Block Kit body: title, classification/repo fields, failing tests
|
|
385
|
-
# when known, the diagnosis, and link buttons assembled per outcome —
|
|
386
|
-
# a button never renders without its target.
|
|
387
|
-
jq -n \
|
|
388
|
-
--arg text "$(printf '%b' "$text")" \
|
|
389
|
-
--arg title "$title" \
|
|
390
|
-
--arg body "$body" \
|
|
391
|
-
--arg classification "$label" \
|
|
392
|
-
--arg repo "$repo" \
|
|
393
|
-
--arg failed "$failed_tests" \
|
|
394
|
-
--arg app_change "$APP_CHANGE" \
|
|
395
|
-
--arg pr "$PR_URL" \
|
|
396
|
-
--arg run "$RUN_URL" \
|
|
397
|
-
--arg result "$RESULT_LINK" \
|
|
398
|
-
--arg rerun "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/workflows/checkly-heal.yml" \
|
|
399
|
-
'
|
|
400
|
-
def btn($label; $url): {type: "button", text: {type: "plain_text", text: $label}, url: $url};
|
|
401
|
-
{
|
|
402
|
-
text: $text,
|
|
403
|
-
blocks: (
|
|
404
|
-
[
|
|
405
|
-
{type: "section", text: {type: "mrkdwn", text: $title}},
|
|
406
|
-
{type: "section", fields: ([
|
|
407
|
-
{type: "mrkdwn", text: ("*Classification*\n" + $classification)},
|
|
408
|
-
{type: "mrkdwn", text: ("*Repository*\n" + $repo)}
|
|
409
|
-
] + (if $failed != "" then
|
|
410
|
-
[{type: "mrkdwn", text: ("*Failed test*\n" + $failed)}]
|
|
411
|
-
else [] end))}
|
|
412
|
-
]
|
|
413
|
-
+ (if $body != "" then
|
|
414
|
-
[{type: "section", text: {type: "mrkdwn", text: $body}}]
|
|
415
|
-
else [] end)
|
|
416
|
-
+ (if $app_change == "true" then
|
|
417
|
-
[{type: "context", elements: [{type: "mrkdwn", text: "Includes app-side test hooks — the PR'\''s preview run validates them."}]}]
|
|
418
|
-
else [] end)
|
|
419
|
-
+ [{type: "actions", elements: (
|
|
420
|
-
(if $pr != "" then [btn("Review PR"; $pr)] else [] end)
|
|
421
|
-
+ [btn("View run"; $run)]
|
|
422
|
-
+ (if $result != "" then [btn("Checkly result"; $result)] else [] end)
|
|
423
|
-
+ (if $classification == "agent-failed" then [btn("Re-run heal"; $rerun)] else [] end)
|
|
424
|
-
)}]
|
|
425
|
-
+ [{type: "context", elements: [{type: "mrkdwn", text: ("[e2e:" + $repo + "]")}]}]
|
|
426
|
-
)
|
|
427
|
-
}' \
|
|
428
|
-
| curl -sf --max-time 15 -X POST -H "Content-Type: application/json" -d @- "$SLACK_WEBHOOK_URL" \
|
|
429
|
-
|| echo "Slack notification failed — see summary." >> "$GITHUB_STEP_SUMMARY"
|
|
430
|
-
|
|
431
|
-
- name: Upload agent artifacts
|
|
432
|
-
if: always() && steps.guard.outputs.mode == 'heal'
|
|
433
|
-
uses: actions/upload-artifact@v4
|
|
434
|
-
with:
|
|
435
|
-
name: checkly-heal-${{ github.run_id }}
|
|
436
|
-
path: |
|
|
437
|
-
.checkly-heal/
|
|
438
|
-
/tmp/checkly-heal-agent-output.json
|
|
439
|
-
# .checkly-heal is a dot-directory; without this it is silently
|
|
440
|
-
# excluded and the debug artifact loses context/outcome/result.
|
|
441
|
-
include-hidden-files: true
|
|
442
|
-
if-no-files-found: ignore
|
|
443
|
-
retention-days: 14
|
|
36
|
+
uses: why-agency/why-e2e-pipeline/.github/workflows/reusable-checkly-heal.yml@main
|
|
37
|
+
with:
|
|
38
|
+
check_name: ${{ github.event.client_payload.check_name || inputs.check_name }}
|
|
39
|
+
check_id: ${{ github.event.client_payload.check_id || inputs.check_id }}
|
|
40
|
+
result_id: ${{ github.event.client_payload.result_id || inputs.result_id }}
|
|
41
|
+
# Per-repo settings, defaults shown. Uncomment to change.
|
|
42
|
+
# e2e_root: tests/e2e # match your playwright.config testDir
|
|
43
|
+
# heal_commit_scope: tests # fix(<scope>): heal check "<name>"
|
|
44
|
+
# repeat_window_days: 7
|
|
45
|
+
# node_version: "22"
|
|
46
|
+
# pnpm_version: "" # set when package.json has no packageManager
|
|
47
|
+
# install_command: pnpm install --frozen-lockfile --ignore-scripts
|
|
48
|
+
secrets: inherit
|
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
name: PR Playwright tests
|
|
2
2
|
|
|
3
|
-
# Runs the suite against the PR's own Vercel preview
|
|
4
|
-
#
|
|
5
|
-
#
|
|
3
|
+
# Runs the suite against the PR's own Vercel preview and comments the result
|
|
4
|
+
# on the PR. The work happens in the pipeline's reusable workflow, so a
|
|
5
|
+
# pipeline release updates it without a change here. This file owns the
|
|
6
|
+
# trigger and the permissions only.
|
|
6
7
|
# Needs repo secret VERCEL_AUTOMATION_BYPASS_SECRET when previews are
|
|
7
|
-
# protected
|
|
8
|
+
# protected. Optional repo variable PREVIEW_LOCALE_HOST: which production
|
|
9
|
+
# domain's locale config preview hosts resolve to.
|
|
8
10
|
on:
|
|
9
11
|
deployment_status
|
|
10
12
|
|
|
11
|
-
# Vercel's payload has no branch, only the SHA — the
|
|
13
|
+
# Vercel's payload has no branch, only the SHA — the reusable workflow
|
|
14
|
+
# resolves it.
|
|
12
15
|
run-name: Playwright · ${{ github.event.deployment.environment }}
|
|
13
16
|
|
|
14
17
|
permissions:
|
|
@@ -16,142 +19,13 @@ permissions:
|
|
|
16
19
|
pull-requests: write
|
|
17
20
|
|
|
18
21
|
jobs:
|
|
19
|
-
resolve-branch:
|
|
20
|
-
if: >-
|
|
21
|
-
github.event.deployment_status.state == 'success' &&
|
|
22
|
-
github.event.deployment.environment == 'Preview'
|
|
23
|
-
runs-on: ubuntu-latest
|
|
24
|
-
timeout-minutes: 2
|
|
25
|
-
outputs:
|
|
26
|
-
branch: ${{ steps.pr.outputs.branch }}
|
|
27
|
-
label: ${{ steps.pr.outputs.label }}
|
|
28
|
-
number: ${{ steps.pr.outputs.number }}
|
|
29
|
-
steps:
|
|
30
|
-
- name: Resolve the PR branch for this commit
|
|
31
|
-
id: pr
|
|
32
|
-
env:
|
|
33
|
-
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
34
|
-
SHA: ${{ github.event.deployment.ref }}
|
|
35
|
-
REPO: ${{ github.repository }}
|
|
36
|
-
run: |
|
|
37
|
-
# No associated PR (plain branch push) falls back to the SHA.
|
|
38
|
-
pr=$(gh api "repos/$REPO/commits/$SHA/pulls" --jq \
|
|
39
|
-
'if length > 0 then "\(.[0].number)\t\(.[0].head.ref)\t\(.[0].title)" else "" end' \
|
|
40
|
-
2>/dev/null || true)
|
|
41
|
-
branch=$(printf '%s' "$pr" | cut -f2)
|
|
42
|
-
number=$(printf '%s' "$pr" | cut -f1)
|
|
43
|
-
title=$(printf '%s' "$pr" | cut -f3)
|
|
44
|
-
|
|
45
|
-
echo "branch=${branch:-$SHA}" >> "$GITHUB_OUTPUT"
|
|
46
|
-
echo "number=${number}" >> "$GITHUB_OUTPUT"
|
|
47
|
-
if [ -n "$number" ]; then
|
|
48
|
-
echo "label=#${number} ${title}" >> "$GITHUB_OUTPUT"
|
|
49
|
-
else
|
|
50
|
-
echo "label=${branch:-$SHA}" >> "$GITHUB_OUTPUT"
|
|
51
|
-
fi
|
|
52
|
-
|
|
53
22
|
playwright:
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
steps:
|
|
65
|
-
- name: Check out the deployed ref
|
|
66
|
-
uses: actions/checkout@v5
|
|
67
|
-
with:
|
|
68
|
-
ref: ${{ github.event.deployment.ref }}
|
|
69
|
-
|
|
70
|
-
- name: Set up pnpm
|
|
71
|
-
uses: pnpm/action-setup@v4
|
|
72
|
-
|
|
73
|
-
- name: Set up Node
|
|
74
|
-
uses: actions/setup-node@v5
|
|
75
|
-
with:
|
|
76
|
-
node-version: 22
|
|
77
|
-
cache: pnpm
|
|
78
|
-
|
|
79
|
-
# --ignore-scripts: only the tests run here; project postinstall hooks
|
|
80
|
-
# are not needed and may want CMS tokens CI does not have.
|
|
81
|
-
- name: Install dependencies
|
|
82
|
-
run: pnpm install --frozen-lockfile --ignore-scripts
|
|
83
|
-
|
|
84
|
-
- name: Install Playwright browsers
|
|
85
|
-
run: pnpm playwright install --with-deps chromium
|
|
86
|
-
|
|
87
|
-
# Chrome only for speed; Firefox/Edge run on Checkly against production.
|
|
88
|
-
# --global-timeout instead of a step timeout so reports still flush.
|
|
89
|
-
- name: Run Playwright tests against the preview
|
|
90
|
-
id: tests
|
|
91
|
-
timeout-minutes: 9
|
|
92
|
-
env:
|
|
93
|
-
BASE_URL: ${{ github.event.deployment_status.target_url }}
|
|
94
|
-
VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}
|
|
95
|
-
# Which production domain's locale config preview hosts resolve to
|
|
96
|
-
# (repo variable, optional — specs default to the primary domain).
|
|
97
|
-
PREVIEW_LOCALE_HOST: ${{ vars.PREVIEW_LOCALE_HOST }}
|
|
98
|
-
PLAYWRIGHT_JSON_OUTPUT_NAME: playwright-results.json
|
|
99
|
-
run: |
|
|
100
|
-
pnpm playwright test \
|
|
101
|
-
--project=Chrome \
|
|
102
|
-
--global-timeout 360000 \
|
|
103
|
-
--reporter=list,html,json
|
|
104
|
-
|
|
105
|
-
- name: Upload HTML report
|
|
106
|
-
if: always()
|
|
107
|
-
uses: actions/upload-artifact@v4
|
|
108
|
-
with:
|
|
109
|
-
name: playwright-report-${{ github.run_id }}
|
|
110
|
-
path: |
|
|
111
|
-
playwright-report/
|
|
112
|
-
playwright-results.json
|
|
113
|
-
if-no-files-found: warn
|
|
114
|
-
retention-days: 7
|
|
115
|
-
|
|
116
|
-
# Reporting only — must not turn the check red.
|
|
117
|
-
- name: Comment the result on the PR
|
|
118
|
-
if: always() && needs.resolve-branch.outputs.number != ''
|
|
119
|
-
continue-on-error: true
|
|
120
|
-
env:
|
|
121
|
-
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
122
|
-
REPO: ${{ github.repository }}
|
|
123
|
-
PR: ${{ needs.resolve-branch.outputs.number }}
|
|
124
|
-
PREVIEW_URL: ${{ github.event.deployment_status.target_url }}
|
|
125
|
-
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
126
|
-
run: |
|
|
127
|
-
node .github/scripts/playwright-pr-comment.mjs \
|
|
128
|
-
playwright-results.json > comment.md
|
|
129
|
-
|
|
130
|
-
# Update the existing comment so re-pushes do not stack.
|
|
131
|
-
marker='<!-- playwright-preview-report -->'
|
|
132
|
-
id=$(gh api "repos/$REPO/issues/$PR/comments" --paginate \
|
|
133
|
-
--jq "map(select(.body | startswith(\"$marker\"))) | .[0].id // empty")
|
|
134
|
-
|
|
135
|
-
if [ -n "$id" ]; then
|
|
136
|
-
gh api -X PATCH "repos/$REPO/issues/comments/$id" -F body=@comment.md
|
|
137
|
-
else
|
|
138
|
-
gh api -X POST "repos/$REPO/issues/$PR/comments" -F body=@comment.md
|
|
139
|
-
fi
|
|
140
|
-
|
|
141
|
-
- name: Write result to job summary
|
|
142
|
-
if: always()
|
|
143
|
-
env:
|
|
144
|
-
OUTCOME: ${{ steps.tests.outcome }}
|
|
145
|
-
PREVIEW_URL: ${{ github.event.deployment_status.target_url }}
|
|
146
|
-
run: |
|
|
147
|
-
{
|
|
148
|
-
echo "## Playwright (preview)"
|
|
149
|
-
echo ""
|
|
150
|
-
echo "- Target: $PREVIEW_URL"
|
|
151
|
-
echo "- Browser: Chrome only"
|
|
152
|
-
if [ "$OUTCOME" = "success" ]; then
|
|
153
|
-
echo "- Result: all tests passed"
|
|
154
|
-
else
|
|
155
|
-
echo "- Result: **failures present** — see the uploaded \`playwright-report\` artifact"
|
|
156
|
-
fi
|
|
157
|
-
} >> "$GITHUB_STEP_SUMMARY"
|
|
23
|
+
uses: why-agency/why-e2e-pipeline/.github/workflows/reusable-pr-playwright-tests.yml@main
|
|
24
|
+
with:
|
|
25
|
+
preview_locale_host: ${{ vars.PREVIEW_LOCALE_HOST }}
|
|
26
|
+
# Per-repo settings, defaults shown. Uncomment to change.
|
|
27
|
+
# node_version: "22" # match the repo's engines/.nvmrc
|
|
28
|
+
# pnpm_version: "" # set when package.json has no packageManager
|
|
29
|
+
# install_command: pnpm install --frozen-lockfile --ignore-scripts
|
|
30
|
+
# skip_branches: "" # e.g. "review|development" for environment branches
|
|
31
|
+
secrets: inherit
|