@andwhy/e2e-pipeline 0.15.1 → 0.15.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andwhy/e2e-pipeline",
-  "version": "0.15.1",
+  "version": "0.15.3",
   "description": "Scaffolds the full e2e testing pipeline: agent-assisted Playwright tests, CI runs against Vercel previews, and Checkly monitors that follow production deploys",
   "type": "module",
   "license": "MIT",

package/plugin/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "e2e-pipeline",
   "displayName": "&why e2e pipeline",
-  "version": "0.15.1",
+  "version": "0.15.3",
   "description": "Agent-assisted Playwright e2e tests: planner, generator and healer agents plus the playwright-test-pipeline skill that drives Plan → Generate → Heal, Checkly preview validation and deployment.",
   "author": {
     "name": "&why",

package/plugin/agents/playwright-test-generator.md

@@ -133,7 +133,8 @@ Follow these when writing tests to prevent flaky-test and cross-environment fail
   exact-match lookup silently skips the entire suite on every PR run and in
   `checkly pw-test` — green checks that tested nothing. The lookup must resolve preview
   hosts to the config of the production domain the preview serves (ask the user which one
-  if it isn't obvious; `PREVIEW_LOCALE_HOST` overrides it at runtime):
+  if it isn't obvious; `PREVIEW_LOCALE_HOST` overrides it at runtime, and CI may pass it
+  empty, so fall back with `||`, never `??`):
 
   ```ts
   const getLocaleConfig = (baseURL: string) => {
@@ -142,7 +143,7 @@ Follow these when writing tests to prevent flaky-test and cross-environment fail
     if (config) return config;
     // Previews serve <primary domain>'s content; without this every preview run skips.
     if (hostname.endsWith(".vercel.app"))
-      return LOCALE_CONFIG[process.env.PREVIEW_LOCALE_HOST ?? "<primary domain>"];
+      return LOCALE_CONFIG[process.env.PREVIEW_LOCALE_HOST || "<primary domain>"];
     return undefined;
   };
   ```

package/plugin/agents/playwright-test-healer.md

@@ -130,7 +130,8 @@ Robustness rules (apply these BEFORE increasing timeouts or adding retries):
   exact-match lookup silently skips the entire suite on every PR run and in
   `checkly pw-test` — green checks that tested nothing. The lookup must resolve preview
   hosts to the config of the production domain the preview serves (ask the user which one
-  if it isn't obvious; `PREVIEW_LOCALE_HOST` overrides it at runtime):
+  if it isn't obvious; `PREVIEW_LOCALE_HOST` overrides it at runtime, and CI may pass it
+  empty, so fall back with `||`, never `??`):
 
   ```ts
   const getLocaleConfig = (baseURL: string) => {
@@ -139,7 +140,7 @@ Robustness rules (apply these BEFORE increasing timeouts or adding retries):
     if (config) return config;
     // Previews serve <primary domain>'s content; without this every preview run skips.
     if (hostname.endsWith(".vercel.app"))
-      return LOCALE_CONFIG[process.env.PREVIEW_LOCALE_HOST ?? "<primary domain>"];
+      return LOCALE_CONFIG[process.env.PREVIEW_LOCALE_HOST || "<primary domain>"];
     return undefined;
   };
   ```

package/plugin/skills/playwright-test-pipeline/SKILL.md

@@ -7,7 +7,7 @@ description: Guides the full workflow for creating and maintaining Playwright
   tests, asks to "test" a page/feature/flow, or asks how the test pipeline
   works.
 metadata:
-  version: "0.15.1"
+  version: "0.15.3"
 ---
 
 # Playwright Test Pipeline
@@ -200,8 +200,9 @@ Include them verbatim in every Agent prompt to the generator or healer.
 4. Never hardcode a single locale's URL pattern. Use a locale config map keyed
    by bare hostname and resolve at runtime. Preview hosts (`*.vercel.app`) are
    never in the map and must fall back to the primary production domain's
-   config (overridable via `PREVIEW_LOCALE_HOST`) — an exact-match-only lookup
-   skips the whole suite on every PR run.
+   config (overridable via `PREVIEW_LOCALE_HOST`, read with `||` so an empty
+   value falls back too) — an exact-match-only lookup skips the whole suite on
+   every PR run.
 
 5. After all tests pass locally, verify with
    `BASE_URL=<dev-url> pnpm playwright test` via the terminal tool. If failures

package/templates/claude/partials/_robustness-core.md

@@ -72,7 +72,8 @@
   exact-match lookup silently skips the entire suite on every PR run and in
   `checkly pw-test` — green checks that tested nothing. The lookup must resolve preview
   hosts to the config of the production domain the preview serves (ask the user which one
-  if it isn't obvious; `PREVIEW_LOCALE_HOST` overrides it at runtime):
+  if it isn't obvious; `PREVIEW_LOCALE_HOST` overrides it at runtime, and CI may pass it
+  empty, so fall back with `||`, never `??`):
 
   ```ts
   const getLocaleConfig = (baseURL: string) => {
@@ -81,7 +82,7 @@
     if (config) return config;
     // Previews serve <primary domain>'s content; without this every preview run skips.
     if (hostname.endsWith(".vercel.app"))
-      return LOCALE_CONFIG[process.env.PREVIEW_LOCALE_HOST ?? "<primary domain>"];
+      return LOCALE_CONFIG[process.env.PREVIEW_LOCALE_HOST || "<primary domain>"];
     return undefined;
   };
   ```

package/templates/github/scripts/playwright-pr-comment.mjs

@@ -13,6 +13,24 @@ const clean = text =>
     .replace(/\u001b\[[0-9;]*m/g, '')
     .trim()
 
+// Skip reasons with counts. A skipped test proves nothing, so a run that
+// skips must say why instead of reading as a pass.
+const skipReasons = (suite, acc = new Map()) => {
+  for (const spec of suite.specs ?? []) {
+    for (const test of spec.tests ?? []) {
+      const last = test.results?.[test.results.length - 1]
+      if (last?.status !== 'skipped') continue
+      const note = [...(test.annotations ?? []), ...(last.annotations ?? [])].find(
+        a => a.type === 'skip'
+      )
+      const reason = clean(note?.description) || 'no reason given'
+      acc.set(reason, (acc.get(reason) ?? 0) + 1)
+    }
+  }
+  for (const child of suite.suites ?? []) skipReasons(child, acc)
+  return acc
+}
+
 const collect = (suite, acc = []) => {
   for (const spec of suite.specs ?? []) {
     for (const test of spec.tests ?? []) {
@@ -41,8 +59,10 @@ const render = report => {
 
   const lines = [MARKER, '## Playwright on the preview', '']
 
-  if (!failures.length && !truncated) {
-    lines.push(`All tests passed. ${expected} passed, ${skipped} skipped.`)
+  if (!failures.length && !truncated && !skipped) {
+    lines.push(`All tests passed. ${expected} passed.`)
+  } else if (!failures.length && !truncated) {
+    lines.push(`No failures. ${expected} passed, **${skipped} skipped**.`)
   } else {
     lines.push(
       `**${failures.length} failed**, ${expected} passed, ${skipped} skipped` +
@@ -58,6 +78,19 @@ const render = report => {
     )
   }
 
+  if (skipped) {
+    const reasons = new Map()
+    for (const s of report.suites ?? []) skipReasons(s, reasons)
+    lines.push(
+      '',
+      '> [!WARNING]',
+      `> ${skipped} skipped tests did not run on this preview. Check that they are meant to skip here:`
+    )
+    for (const [reason, count] of reasons) {
+      lines.push(`> - ${truncate(reason, 200)} (${count})`)
+    }
+  }
+
   for (const f of failures) {
     lines.push('', `### \`${f.file}:${f.line}\` — ${f.title}`)
     if (f.error) {

package/templates/github/workflows/pr-playwright-tests.yml

@@ -5,8 +5,9 @@ name: PR Playwright tests
 # pipeline release updates it without a change here. This file owns the
 # trigger and the permissions only.
 # Needs repo secret VERCEL_AUTOMATION_BYPASS_SECRET when previews are
-# protected. Optional repo variable PREVIEW_LOCALE_HOST: which production
-# domain's locale config preview hosts resolve to.
+# protected. Optional repo variable PREVIEW_TESTS_AS: the production domain
+# whose test settings previews use (e.g. shop.de), one value per repo.
+# Unset, the specs use the primary domain.
 on:
   deployment_status
 
@@ -22,7 +23,7 @@ jobs:
   playwright:
     uses: why-agency/why-e2e-pipeline/.github/workflows/reusable-pr-playwright-tests.yml@main
     with:
-      preview_locale_host: ${{ vars.PREVIEW_LOCALE_HOST }}
+      preview_tests_as: ${{ vars.PREVIEW_TESTS_AS }}
       # Per-repo settings, defaults shown. Uncomment to change.
       # node_version: "22"             # match the repo's engines/.nvmrc
       # pnpm_version: ""               # set when package.json has no packageManager