@andwhy/e2e-pipeline 0.10.0

package/templates/github/workflows/pr-playwright-tests.yml

@@ -0,0 +1,154 @@
+name: PR Playwright tests
+
+# Runs the suite against the PR's own Vercel preview. `deployment_status`
+# fires once the preview is live and carries its URL; the preview is built
+# from the PR branch, so the PR's changes exist on the target.
+# Needs repo secret VERCEL_AUTOMATION_BYPASS_SECRET when previews are
+# protected; the scaffolded helper turns it into Vercel's bypass cookie.
+on:
+  deployment_status
+
+# Vercel's payload has no branch, only the SHA — the job below resolves it.
+run-name: Playwright · ${{ github.event.deployment.environment }}
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  resolve-branch:
+    if: >-
+      github.event.deployment_status.state == 'success' &&
+      github.event.deployment.environment == 'Preview'
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    outputs:
+      branch: ${{ steps.pr.outputs.branch }}
+      label: ${{ steps.pr.outputs.label }}
+      number: ${{ steps.pr.outputs.number }}
+    steps:
+      - name: Resolve the PR branch for this commit
+        id: pr
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SHA: ${{ github.event.deployment.ref }}
+          REPO: ${{ github.repository }}
+        run: |
+          # No associated PR (plain branch push) falls back to the SHA.
+          pr=$(gh api "repos/$REPO/commits/$SHA/pulls" --jq \
+            'if length > 0 then "\(.[0].number)\t\(.[0].head.ref)\t\(.[0].title)" else "" end' \
+            2>/dev/null || true)
+          branch=$(printf '%s' "$pr" | cut -f2)
+          number=$(printf '%s' "$pr" | cut -f1)
+          title=$(printf '%s' "$pr" | cut -f3)
+
+          echo "branch=${branch:-$SHA}" >> "$GITHUB_OUTPUT"
+          echo "number=${number}" >> "$GITHUB_OUTPUT"
+          if [ -n "$number" ]; then
+            echo "label=#${number} ${title}" >> "$GITHUB_OUTPUT"
+          else
+            echo "label=${branch:-$SHA}" >> "$GITHUB_OUTPUT"
+          fi
+
+  playwright:
+    name: Playwright · ${{ needs.resolve-branch.outputs.label }}
+    needs: resolve-branch
+    # A re-push cancels the stale run instead of racing an outdated preview.
+    concurrency:
+      group: pr-playwright-${{ needs.resolve-branch.outputs.branch }}
+      cancel-in-progress: true
+    runs-on: ubuntu-latest
+    # Red informs, branch protection decides — the check is not required.
+    timeout-minutes: 10
+
+    steps:
+      - name: Check out the deployed ref
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ github.event.deployment.ref }}
+
+      - name: Set up pnpm
+        uses: pnpm/action-setup@v4
+
+      - name: Set up Node
+        uses: actions/setup-node@v5
+        with:
+          node-version: 22
+          cache: pnpm
+
+      # --ignore-scripts: only the tests run here; project postinstall hooks
+      # are not needed and may want CMS tokens CI does not have.
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile --ignore-scripts
+
+      - name: Install Playwright browsers
+        run: pnpm playwright install --with-deps chromium
+
+      # Chrome only for speed; Firefox/Edge run on Checkly against production.
+      # --global-timeout instead of a step timeout so reports still flush.
+      - name: Run Playwright tests against the preview
+        id: tests
+        timeout-minutes: 9
+        env:
+          BASE_URL: ${{ github.event.deployment_status.target_url }}
+          VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}
+          PLAYWRIGHT_JSON_OUTPUT_NAME: playwright-results.json
+        run: |
+          pnpm playwright test \
+            --project=Chrome \
+            --global-timeout 360000 \
+            --reporter=list,html,json
+
+      - name: Upload HTML report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report-${{ github.run_id }}
+          path: |
+            playwright-report/
+            playwright-results.json
+          if-no-files-found: warn
+          retention-days: 7
+
+      # Reporting only — must not turn the check red.
+      - name: Comment the result on the PR
+        if: always() && needs.resolve-branch.outputs.number != ''
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          PR: ${{ needs.resolve-branch.outputs.number }}
+          PREVIEW_URL: ${{ github.event.deployment_status.target_url }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          node .github/scripts/playwright-pr-comment.mjs \
+            playwright-results.json > comment.md
+
+          # Update the existing comment so re-pushes do not stack.
+          marker='<!-- playwright-preview-report -->'
+          id=$(gh api "repos/$REPO/issues/$PR/comments" --paginate \
+            --jq "map(select(.body | startswith(\"$marker\"))) | .[0].id // empty")
+
+          if [ -n "$id" ]; then
+            gh api -X PATCH "repos/$REPO/issues/comments/$id" -F body=@comment.md
+          else
+            gh api -X POST "repos/$REPO/issues/$PR/comments" -F body=@comment.md
+          fi
+
+      - name: Write result to job summary
+        if: always()
+        env:
+          OUTCOME: ${{ steps.tests.outcome }}
+          PREVIEW_URL: ${{ github.event.deployment_status.target_url }}
+        run: |
+          {
+            echo "## Playwright (preview)"
+            echo ""
+            echo "- Target: $PREVIEW_URL"
+            echo "- Browser: Chrome only"
+            if [ "$OUTCOME" = "success" ]; then
+              echo "- Result: all tests passed"
+            else
+              echo "- Result: **failures present** — see the uploaded \`playwright-report\` artifact"
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"

package/templates/shared/checkly.config.template.mjs

@@ -0,0 +1,58 @@
+import { defineConfig } from "checkly";
+
+// -- Multi-domain example (uncomment and adapt) --
+// const domains = [
+//   { name: "example.com", url: "https://example.com" },
+//   { name: "example.de", url: "https://example.de" },
+// ]
+
+/**
+ * See https://www.checklyhq.com/docs/cli/project-structure/
+ */
+export default defineConfig({
+  projectName: "my-app",
+  /** A logical ID that needs to be unique across your Checkly account.
+   * See https://www.checklyhq.com/docs/cli/constructs/
+   */
+  logicalId: "my-app",
+  repoUrl: "https://github.com/your-org/your-repo",
+  checks: {
+    frequency: 180,
+    locations: ["eu-central-1"],
+    runtimeId: "2025.04",
+    playwrightConfigPath: "./playwright.config.mjs",
+    playwrightChecks: [
+      {
+        logicalId: "my-app-check-suite",
+        // Change the name to match your brand or product.
+        name: "my-app",
+        // Checkly's runner gets a bundle of test files, not the repo, so a
+        // project postinstall (codegen etc.) fails there on missing files.
+        // pnpm 10 skips dependency build scripts by default, so this only
+        // disables the project's own hooks; browsers come from the runtime.
+        installCommand: "pnpm install --ignore-scripts",
+        // Deploy only the e2e suite as monitors. Without this the whole
+        // Playwright suite runs per domain, so unit tests and regression
+        // guards that never needed a deployment become recurring paid checks.
+        testCommand: "npx playwright test tests/e2e",
+        environmentVariables: [
+          // Set this to your production URL.
+          { key: "BASE_URL", value: "http://localhost:3000" },
+        ],
+      },
+      // Add more suites for additional domains:
+      // {
+      //   logicalId: "my-app-check-suite-de",
+      //   name: "my-app (DE)",
+      //   environmentVariables: [
+      //     { key: "BASE_URL", value: "https://example.de" },
+      //   ],
+      // },
+    ],
+  },
+  cli: {
+    runLocation: "eu-west-1",
+    reporters: ["list"],
+    retries: 1,
+  },
+});

package/templates/shared/checkly.config.template.ts

@@ -0,0 +1,58 @@
+import { defineConfig } from "checkly";
+
+// -- Multi-domain example (uncomment and adapt) --
+// const domains = [
+//   { name: "example.com", url: "https://example.com" },
+//   { name: "example.de", url: "https://example.de" },
+// ]
+
+/**
+ * See https://www.checklyhq.com/docs/cli/project-structure/
+ */
+export default defineConfig({
+  projectName: "my-app",
+  /** A logical ID that needs to be unique across your Checkly account.
+   * See https://www.checklyhq.com/docs/cli/constructs/
+   */
+  logicalId: "my-app",
+  repoUrl: "https://github.com/your-org/your-repo",
+  checks: {
+    frequency: 180,
+    locations: ["eu-central-1"],
+    runtimeId: "2025.04",
+    playwrightConfigPath: "./playwright.config.ts",
+    playwrightChecks: [
+      {
+        logicalId: "my-app-check-suite",
+        // Change the name to match your brand or product.
+        name: "my-app",
+        // Checkly's runner gets a bundle of test files, not the repo, so a
+        // project postinstall (codegen etc.) fails there on missing files.
+        // pnpm 10 skips dependency build scripts by default, so this only
+        // disables the project's own hooks; browsers come from the runtime.
+        installCommand: "pnpm install --ignore-scripts",
+        // Deploy only the e2e suite as monitors. Without this the whole
+        // Playwright suite runs per domain, so unit tests and regression
+        // guards that never needed a deployment become recurring paid checks.
+        testCommand: "npx playwright test tests/e2e",
+        environmentVariables: [
+          // Set this to your production URL.
+          { key: "BASE_URL", value: "http://localhost:3000" },
+        ],
+      },
+      // Add more suites for additional domains:
+      // {
+      //   logicalId: "my-app-check-suite-de",
+      //   name: "my-app (DE)",
+      //   environmentVariables: [
+      //     { key: "BASE_URL", value: "https://example.de" },
+      //   ],
+      // },
+    ],
+  },
+  cli: {
+    runLocation: "eu-west-1",
+    reporters: ["list"],
+    retries: 1,
+  },
+});

package/templates/shared/playwright.config.template.mjs

@@ -0,0 +1,44 @@
+import { defineConfig, devices } from "@playwright/test";
+
+export default defineConfig({
+  // e2e specs live here so other suites elsewhere under tests/ are not picked
+  // up. If the repo already has such a suite (unit, a11y), give it its own
+  // project below with its own testDir — do NOT widen this one to ./tests,
+  // which would run that suite in every browser project and mix the suites.
+  testDir: "./tests/e2e",
+  // Seeds are run directly by the Playwright MCP, not as tests of their own.
+  testIgnore: "**/seed.spec.*",
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: process.env.CI ? 1 : undefined,
+  reporter: "html",
+  use: {
+    baseURL: process.env.BASE_URL ?? "http://localhost:3000",
+    trace: "on",
+  },
+  projects: [
+    {
+      name: "Chrome",
+      use: {
+        ...devices["Desktop Chrome"],
+        userAgent: "Chrome_Checkly_0759687ceba6",
+      },
+    },
+    {
+      name: "Firefox",
+      use: {
+        ...devices["Desktop Firefox"],
+        userAgent: "Firefox_Checkly_0759687ceba6",
+      },
+    },
+    {
+      name: "Edge",
+      use: {
+        ...devices["Desktop Edge"],
+        userAgent: "Edge_Checkly_0759687ceba6",
+      },
+    },
+  ],
+  timeout: 60_000,
+});

package/templates/shared/playwright.config.template.ts

@@ -0,0 +1,44 @@
+import { defineConfig, devices } from "@playwright/test";
+
+export default defineConfig({
+  // e2e specs live here so other suites elsewhere under tests/ are not picked
+  // up. If the repo already has such a suite (unit, a11y), give it its own
+  // project below with its own testDir — do NOT widen this one to ./tests,
+  // which would run that suite in every browser project and mix the suites.
+  testDir: "./tests/e2e",
+  // Seeds are run directly by the Playwright MCP, not as tests of their own.
+  testIgnore: "**/seed.spec.*",
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: process.env.CI ? 1 : undefined,
+  reporter: "html",
+  use: {
+    baseURL: process.env.BASE_URL ?? "http://localhost:3000",
+    trace: "on",
+  },
+  projects: [
+    {
+      name: "Chrome",
+      use: {
+        ...devices["Desktop Chrome"],
+        userAgent: "Chrome_Checkly_0759687ceba6",
+      },
+    },
+    {
+      name: "Firefox",
+      use: {
+        ...devices["Desktop Firefox"],
+        userAgent: "Firefox_Checkly_0759687ceba6",
+      },
+    },
+    {
+      name: "Edge",
+      use: {
+        ...devices["Desktop Edge"],
+        userAgent: "Edge_Checkly_0759687ceba6",
+      },
+    },
+  ],
+  timeout: 60_000,
+});

package/templates/shared/playwright.mcp.config.template.mjs

@@ -0,0 +1,12 @@
+import { defineConfig } from "@playwright/test";
+import config from "./playwright.config.mjs";
+
+// Used only by the Playwright MCP server — .mcp.json points here, nothing else
+// does. The main config ignores seed.spec.* so seeds never run as tests in
+// local, CI, or Checkly runs; but the planner/generator setup_page tools
+// execute seeds through the test runner, so the MCP's config must be able to
+// discover them. Do not point CI or Checkly at this file.
+export default defineConfig({
+  ...config,
+  testIgnore: [],
+});

package/templates/shared/playwright.mcp.config.template.ts

@@ -0,0 +1,12 @@
+import { defineConfig } from "@playwright/test";
+import config from "./playwright.config";
+
+// Used only by the Playwright MCP server — .mcp.json points here, nothing else
+// does. The main config ignores seed.spec.* so seeds never run as tests in
+// local, CI, or Checkly runs; but the planner/generator setup_page tools
+// execute seeds through the test runner, so the MCP's config must be able to
+// discover them. Do not point CI or Checkly at this file.
+export default defineConfig({
+  ...config,
+  testIgnore: [],
+});

package/templates/shared/preview-consent-setup.template.js

@@ -0,0 +1,32 @@
+
+// Vercel deployment protection returns 401 for anonymous visitors, which is
+// what Checkly's runners are. With the project's automation bypass secret in
+// VERCEL_AUTOMATION_BYPASS_SECRET (Vercel project settings → Deployment
+// Protection; keep the value in 1Password and pass it per run:
+//   pnpm checkly pw-test --env BASE_URL="<preview>" \
+//     --env VERCEL_AUTOMATION_BYPASS_SECRET="<secret>" -- tests/e2e/<feature>/
+// ) the first navigation exchanges the secret for Vercel's bypass cookie and
+// the rest of the test uses plain relative URLs. Without the secret this is a
+// plain goto, so the same specs run unchanged against localhost and production.
+// Never commit the secret into checkly.config — deployed monitors run against
+// production and do not need it.
+export const prepareVercelPreviewPage = async (page, baseURL) => {
+  const secret = process.env.VERCEL_AUTOMATION_BYPASS_SECRET;
+  if (secret) {
+    await page.context().setExtraHTTPHeaders({
+      "x-vercel-protection-bypass": secret,
+      "x-vercel-set-bypass-cookie": "true",
+    });
+  }
+  await page.goto(baseURL ?? "/");
+  if (secret) {
+    // The bypass cookie is set now. Stop sending the secret with every
+    // request — extraHTTPHeaders are context-wide and would carry it to
+    // third-party origins too.
+    await page.context().setExtraHTTPHeaders({});
+  }
+};
+
+// Projects extend this file with their own consent helpers, e.g.
+// gotoPathAndAcceptCookies(page, baseURL, path) for the local cookie banner —
+// that part is CMP-specific and stays project-owned.

package/templates/shared/preview-consent-setup.template.ts

@@ -0,0 +1,33 @@
+import type { Page } from "@playwright/test";
+
+// Vercel deployment protection returns 401 for anonymous visitors, which is
+// what Checkly's runners are. With the project's automation bypass secret in
+// VERCEL_AUTOMATION_BYPASS_SECRET (Vercel project settings → Deployment
+// Protection; keep the value in 1Password and pass it per run:
+//   pnpm checkly pw-test --env BASE_URL="<preview>" \
+//     --env VERCEL_AUTOMATION_BYPASS_SECRET="<secret>" -- tests/e2e/<feature>/
+// ) the first navigation exchanges the secret for Vercel's bypass cookie and
+// the rest of the test uses plain relative URLs. Without the secret this is a
+// plain goto, so the same specs run unchanged against localhost and production.
+// Never commit the secret into checkly.config — deployed monitors run against
+// production and do not need it.
+export const prepareVercelPreviewPage = async (page: Page, baseURL?: string) => {
+  const secret = process.env.VERCEL_AUTOMATION_BYPASS_SECRET;
+  if (secret) {
+    await page.context().setExtraHTTPHeaders({
+      "x-vercel-protection-bypass": secret,
+      "x-vercel-set-bypass-cookie": "true",
+    });
+  }
+  await page.goto(baseURL ?? "/");
+  if (secret) {
+    // The bypass cookie is set now. Stop sending the secret with every
+    // request — extraHTTPHeaders are context-wide and would carry it to
+    // third-party origins too.
+    await page.context().setExtraHTTPHeaders({});
+  }
+};
+
+// Projects extend this file with their own consent helpers, e.g.
+// gotoPathAndAcceptCookies(page, baseURL, path) for the local cookie banner —
+// that part is CMP-specific and stays project-owned.

package/templates/shared/seed.spec.js

@@ -0,0 +1,5 @@
+import { test } from "@playwright/test";
+
+test("seed", async ({ page }) => {
+  await page.goto("/");
+});

package/templates/shared/seed.spec.ts

@@ -0,0 +1,5 @@
+import { test } from "@playwright/test";
+
+test("seed", async ({ page }) => {
+  await page.goto("/");
+});