@andwhy/e2e-pipeline 0.12.0 → 0.12.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andwhy/e2e-pipeline",
-  "version": "0.12.0",
+  "version": "0.12.1",
   "description": "Scaffolds the full e2e testing pipeline: agent-assisted Playwright tests, CI runs against Vercel previews, and Checkly monitors that follow production deploys",
   "type": "module",
   "license": "MIT",

package/templates/github/prompts/checkly-heal.md

@@ -39,6 +39,10 @@ negotiable:
    touch `.github/`, `.claude/`, `package.json`, lockfiles, or any
    `*.config.*` file.
 4. **Never run git commit, push, or any GitHub CLI/API command.**
+5. **Never delete, clean, or recreate the `.checkly-heal/` directory** — the
+   workflow reads your outcome from it after you finish, and destroying it
+   turns your entire run into an unexplained failure. No `rm -rf`, no
+   `git clean`. Only add files to it.
 
 ## Procedure
 
@@ -47,6 +51,13 @@ negotiable:
 Read the context and result files. Identify the failing spec file, test, and
 step. Read the spec. Resolve the production `BASE_URL` (override 1).
 
+If the check name from the context matches **nothing** in
+`checkly.config.{ts,mjs}` — a webhook test, a renamed check, a synthetic
+dispatch — there is nothing to reproduce: write the outcome immediately with
+classification `not-reproducible` and a summary saying the check name is
+unknown to this repo, then stop. Do not skip the outcome file because the
+run "wasn't real".
+
 ### 2. Reproduce
 
 Run only the failing spec against production:

package/templates/github/workflows/checkly-heal.yml

@@ -163,13 +163,23 @@ jobs:
           CHECKLY_ACCOUNT_ID: ${{ secrets.CHECKLY_ACCOUNT_ID }}
         run: |
           npm install -g @anthropic-ai/claude-code@2
+          # Transcript lives OUTSIDE .checkly-heal so it survives whatever the
+          # agent does to its working directory, and outside the repo tree so
+          # the diff guard never counts it.
           claude -p "$(cat .github/prompts/checkly-heal.md)" \
             --model claude-opus-5 \
             --allowedTools "Bash,Read,Edit,Write,Glob,Grep" \
             --permission-mode dontAsk \
             --max-turns 150 \
             --output-format json \
-            > .checkly-heal/agent-output.json || true
+            > /tmp/checkly-heal-agent-output.json || echo "claude exited non-zero: $?" >> "$GITHUB_STEP_SUMMARY"
+          {
+            echo "### Agent step diagnostics"
+            echo '```'
+            ls -la .checkly-heal 2>&1 || echo ".checkly-heal missing"
+            tail -c 2000 /tmp/checkly-heal-agent-output.json 2>/dev/null || echo "no agent output"
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
 
       # The guard is deterministic on purpose: the PR path requires an outcome
       # file claiming a fix AND a diff clear of protected paths. App-source
@@ -319,6 +329,8 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: checkly-heal-${{ github.run_id }}
-          path: .checkly-heal/
+          path: |
+            .checkly-heal/
+            /tmp/checkly-heal-agent-output.json
           if-no-files-found: ignore
           retention-days: 14