groundwork-method 0.0.1 → 0.10.0

package/src/generators/system-test-runner/files/tests/system/test_a11y_smoke.py.template

@@ -0,0 +1,132 @@
+<% if (!surfaces) { -%>
+"""Accessibility + console-hygiene smoke for the graphical UI.
+
+Loads the frontend root (discovered via conftest.frontend_base_url) and asserts:
+
+  1. No severe console errors during initial load — uncaught page errors and
+     console.error output both fail the smoke.
+  2. An axe-core scan (injected from CDN — no extra pip dependency) reports
+     ZERO critical violations. Lower-impact findings are reported as warnings
+     so they surface without blocking the loop.
+
+Skips cleanly when no frontend service is discovered (frontend_base_url skips)
+and when the axe-core CDN is unreachable (offline runs must not false-fail).
+"""
+
+import warnings
+
+import pytest
+from playwright.sync_api import Page
+
+AXE_CDN_URL = "https://cdn.jsdelivr.net/npm/axe-core@4.10.2/axe.min.js"
+
+
+def test_frontend_root_a11y_smoke(cluster, frontend_base_url, page: Page):
+    console_errors: list[str] = []
+    page.on(
+        "console",
+        lambda msg: console_errors.append(msg.text) if msg.type == "error" else None,
+    )
+    page.on("pageerror", lambda exc: console_errors.append(f"pageerror: {exc}"))
+
+    page.goto(frontend_base_url, wait_until="load")
+    # Let hydration and deferred requests settle so late errors are captured.
+    page.wait_for_load_state("networkidle")
+
+    assert not console_errors, (
+        f"frontend root emitted severe console errors on load:\n  "
+        + "\n  ".join(console_errors)
+    )
+
+    try:
+        page.add_script_tag(url=AXE_CDN_URL)
+    except Exception as exc:  # noqa: BLE001 — any injection failure means no scan
+        pytest.skip(f"axe-core CDN unreachable, a11y scan skipped: {exc}")
+
+    results = page.evaluate(
+        "() => axe.run(document, { resultTypes: ['violations'] })"
+    )
+    violations = results.get("violations", [])
+
+    non_critical = [v for v in violations if v.get("impact") != "critical"]
+    if non_critical:
+        summary = ", ".join(f"{v['id']} ({v.get('impact')})" for v in non_critical)
+        warnings.warn(f"non-critical a11y violations on frontend root: {summary}")
+
+    critical = [v for v in violations if v.get("impact") == "critical"]
+    assert not critical, (
+        "axe-core found CRITICAL accessibility violations on the frontend root:\n  "
+        + "\n  ".join(
+            f"{v['id']}: {v['help']} ({len(v.get('nodes', []))} node(s))"
+            for v in critical
+        )
+    )
+<% } else { -%>
+"""Accessibility + console-hygiene smoke, one test per graphical surface.
+
+Each test loads the surface's root via its generated per-surface page fixture
+(see conftest's surface fixtures) and asserts:
+
+  1. No severe console errors during initial load — uncaught page errors and
+     console.error output both fail the smoke.
+  2. An axe-core scan (injected from CDN — no extra pip dependency) reports
+     ZERO critical violations. Lower-impact findings are reported as warnings
+     so they surface without blocking the loop.
+
+Skips cleanly when a surface is not reachable (its page fixture skips) and
+when the axe-core CDN is unreachable (offline runs must not false-fail).
+"""
+
+import warnings
+
+import pytest
+from playwright.sync_api import Page
+
+AXE_CDN_URL = "https://cdn.jsdelivr.net/npm/axe-core@4.10.2/axe.min.js"
+
+
+def _root_a11y_smoke(page: Page, surface_slug: str) -> None:
+    console_errors: list[str] = []
+    page.on(
+        "console",
+        lambda msg: console_errors.append(msg.text) if msg.type == "error" else None,
+    )
+    page.on("pageerror", lambda exc: console_errors.append(f"pageerror: {exc}"))
+
+    page.goto("/", wait_until="load")
+    # Let hydration and deferred requests settle so late errors are captured.
+    page.wait_for_load_state("networkidle")
+
+    assert not console_errors, (
+        f"{surface_slug} root emitted severe console errors on load:\n  "
+        + "\n  ".join(console_errors)
+    )
+
+    try:
+        page.add_script_tag(url=AXE_CDN_URL)
+    except Exception as exc:  # noqa: BLE001 — any injection failure means no scan
+        pytest.skip(f"axe-core CDN unreachable, a11y scan skipped: {exc}")
+
+    results = page.evaluate(
+        "() => axe.run(document, { resultTypes: ['violations'] })"
+    )
+    violations = results.get("violations", [])
+
+    non_critical = [v for v in violations if v.get("impact") != "critical"]
+    if non_critical:
+        summary = ", ".join(f"{v['id']} ({v.get('impact')})" for v in non_critical)
+        warnings.warn(f"non-critical a11y violations on {surface_slug} root: {summary}")
+
+    critical = [v for v in violations if v.get("impact") == "critical"]
+    assert not critical, (
+        f"axe-core found CRITICAL accessibility violations on the {surface_slug} root:\n  "
+        + "\n  ".join(
+            f"{v['id']}: {v['help']} ({len(v.get('nodes', []))} node(s))"
+            for v in critical
+        )
+    )
+<% graphicalSurfaces.forEach((s) => { %>
+
+def test_<%= s.ident %>_root_a11y_smoke(cluster, <%= s.ident %>_page: Page):
+    _root_a11y_smoke(<%= s.ident %>_page, "<%= s.slug %>")
+<% }) %><% } -%>

package/src/generators/system-test-runner/files/tests/system/test_contract_conformance.py.template

@@ -0,0 +1,140 @@
+"""Contract conformance: the served API vs the promoted design spec.
+
+For each discovered APP service (see conftest.services_manifest) with a
+canonical promoted spec at docs/architecture/api/<service>/openapi.yaml, fetch the spec the
+RUNNING service serves (Huma on Go and FastAPI both expose /openapi.json) and
+structurally compare paths + methods + response codes.
+
+Drift taxonomy:
+  BREAKING — surface present in the design spec but missing from the served
+             spec (a path, method, or response code the design promises and
+             the implementation does not deliver). Fails the test.
+  ADDITIVE — surface the service serves that the design spec does not declare.
+             Reported as a warning: no consumer breaks, but the canonical spec
+             is stale and should be re-promoted (see the bet validation phase).
+
+Skips cleanly when no design spec is committed for a service, or the service
+does not expose a machine-readable spec.
+"""
+
+import re
+import warnings
+
+import httpx
+import pytest
+import yaml
+
+from conftest import WORKSPACE_ROOT, _base_url, _discover_services
+
+DESIGN_SPEC_ROOT = WORKSPACE_ROOT / "docs" / "api"
+# Huma serves both; FastAPI serves /openapi.json. Try JSON first.
+SERVED_SPEC_PATHS = ("/openapi.json", "/openapi.yaml")
+HTTP_METHODS = {"get", "put", "post", "delete", "options", "head", "patch", "trace"}
+
+# Build params at import time so pytest shows one case per service.
+_SERVICES = _discover_services()
+_PARAMS = [pytest.param(s, id=s["name"]) for s in _SERVICES]
+
+
+def _normalize_path(path: str) -> str:
+    """Collapse path-parameter names so /users/{id} == /users/{userId}.
+
+    Parameter naming is a spec-authoring choice, not an API surface change —
+    comparing it would report false breaking drift.
+    """
+    return re.sub(r"\{[^}]*\}", "{}", path)
+
+
+def _surface(spec: dict) -> dict[tuple[str, str], set[str]]:
+    """Flatten an OpenAPI document to {(path, method): {response codes}}."""
+    surface: dict[tuple[str, str], set[str]] = {}
+    for raw_path, item in (spec.get("paths") or {}).items():
+        if not isinstance(item, dict):
+            continue
+        for method, op in item.items():
+            if method.lower() not in HTTP_METHODS or not isinstance(op, dict):
+                continue
+            codes = {str(c) for c in (op.get("responses") or {})}
+            surface[(_normalize_path(raw_path), method.lower())] = codes
+    return surface
+
+
+def _load_design_spec(service_name: str) -> dict | None:
+    spec_path = DESIGN_SPEC_ROOT / service_name / "openapi.yaml"
+    if not spec_path.exists():
+        return None
+    return yaml.safe_load(spec_path.read_text()) or {}
+
+
+def _fetch_served_spec(svc: dict) -> dict | None:
+    for path in SERVED_SPEC_PATHS:
+        try:
+            resp = httpx.get(f"{_base_url(svc)}{path}", timeout=5.0)
+        except httpx.HTTPError:
+            continue
+        if resp.status_code != 200:
+            continue
+        if path.endswith(".json"):
+            return resp.json()
+        return yaml.safe_load(resp.text)
+    return None
+
+
+def _diff(design: dict, served: dict) -> tuple[list[str], list[str]]:
+    """Compare flattened surfaces. Returns (breaking, additive) drift lines."""
+    breaking: list[str] = []
+    additive: list[str] = []
+    for (path, method), designed_codes in design.items():
+        label = f"{method.upper()} {path}"
+        served_codes = served.get((path, method))
+        if served_codes is None:
+            breaking.append(f"{label} — designed but not served")
+            continue
+        missing = designed_codes - served_codes
+        if missing:
+            breaking.append(
+                f"{label} — designed response codes not served: {sorted(missing)}"
+            )
+        extra = served_codes - designed_codes
+        if extra:
+            additive.append(f"{label} — served response codes not in design: {sorted(extra)}")
+    for path, method in sorted(served.keys() - design.keys()):
+        additive.append(f"{method.upper()} {path} — served but not in design spec")
+    return breaking, additive
+
+
+@pytest.mark.parametrize("svc", _PARAMS)
+def test_served_api_conforms_to_design_spec(cluster, svc):
+    """The running service implements at least the surface its promoted spec declares."""
+    design_doc = _load_design_spec(svc["name"])
+    if design_doc is None:
+        pytest.skip(
+            f"no promoted design spec at docs/architecture/api/{svc['name']}/openapi.yaml — "
+            "promote one in bet validation to bring this service under contract test"
+        )
+    if svc["host_port"] is None:
+        pytest.skip(f"{svc['name']} has no published host port")
+
+    served_doc = _fetch_served_spec(svc)
+    if served_doc is None:
+        pytest.skip(f"{svc['name']} does not expose a served spec at {SERVED_SPEC_PATHS}")
+
+    design = _surface(design_doc)
+    if not design:
+        pytest.skip(f"design spec for {svc['name']} declares no paths — nothing to conform to")
+    served = _surface(served_doc)
+
+    breaking, additive = _diff(design, served)
+
+    if additive:
+        warnings.warn(
+            f"ADDITIVE drift for {svc['name']} — the served API exceeds "
+            f"docs/architecture/api/{svc['name']}/openapi.yaml; re-promote the spec:\n  "
+            + "\n  ".join(additive)
+        )
+
+    assert not breaking, (
+        f"BREAKING drift for {svc['name']} — the design spec at "
+        f"docs/architecture/api/{svc['name']}/openapi.yaml promises surface the running "
+        "service does not deliver:\n  " + "\n  ".join(breaking)
+    )

package/src/generators/system-test-runner/files/tests/system/test_layout_geometry.py.template

@@ -0,0 +1,109 @@
+<% if (!surfaces) { -%>
+"""Layout geometry gate — Tier 1 of the visual verification loop.
+
+A deliberately small handful of deterministic layout invariants that responsive
+breakage violates — kept narrow to avoid heuristic flakiness. Today it asserts the
+highest-signal one: no horizontal overflow at the mobile breakpoint, the defect that
+ships a sideways-scrolling phone layout unseen. Grow the invariant set per project
+(landmark-in-viewport, no-clipped-primary-content) only where it stays deterministic.
+
+Skips cleanly when no frontend service is discovered (frontend_base_url skips).
+"""
+
+import json
+import pathlib
+
+from playwright.sync_api import Page
+
+MOBILE = (375, 812)
+# Sub-pixel rounding can add a fraction; a small tolerance avoids false failures.
+_OVERFLOW_TOLERANCE_PX = 2
+
+
+def _load_routes() -> tuple[str, ...]:
+    manifest = pathlib.Path("tests/system/routes.json")
+    if manifest.exists():
+        try:
+            routes = json.loads(manifest.read_text())
+            if isinstance(routes, list) and routes:
+                return tuple(routes)
+        except Exception:  # noqa: BLE001 — a malformed manifest falls back to root
+            pass
+    return ("/",)
+
+
+ROUTES = _load_routes()
+
+
+def _no_horizontal_overflow(page: Page, surface_slug: str, base_url: str | None) -> None:
+    page.set_viewport_size({"width": MOBILE[0], "height": MOBILE[1]})
+    for route in ROUTES:
+        target = (base_url.rstrip("/") + route) if base_url else route
+        page.goto(target, wait_until="load")
+        page.wait_for_load_state("networkidle")
+        widths = page.evaluate(
+            "() => ({ scroll: document.documentElement.scrollWidth,"
+            " client: document.documentElement.clientWidth })"
+        )
+        assert widths["scroll"] <= widths["client"] + _OVERFLOW_TOLERANCE_PX, (
+            f"{surface_slug} {route} overflows horizontally at {MOBILE[0]}px "
+            f"(scrollWidth {widths['scroll']} > clientWidth {widths['client']})"
+        )
+
+
+def test_frontend_no_horizontal_overflow(cluster, frontend_base_url, page: Page):
+    _no_horizontal_overflow(page, "frontend", frontend_base_url)
+<% } else { -%>
+"""Layout geometry gate, one test per graphical surface — Tier 1 of the visual loop.
+
+A deliberately small handful of deterministic layout invariants that responsive
+breakage violates — kept narrow to avoid heuristic flakiness. Today it asserts the
+highest-signal one: no horizontal overflow at the mobile breakpoint, the defect that
+ships a sideways-scrolling phone layout unseen. Grow the invariant set per project
+only where it stays deterministic.
+
+Skips cleanly when a surface is not reachable (its page fixture skips).
+"""
+
+import json
+import pathlib
+
+from playwright.sync_api import Page
+
+MOBILE = (375, 812)
+_OVERFLOW_TOLERANCE_PX = 2
+
+
+def _load_routes() -> tuple[str, ...]:
+    manifest = pathlib.Path("tests/system/routes.json")
+    if manifest.exists():
+        try:
+            routes = json.loads(manifest.read_text())
+            if isinstance(routes, list) and routes:
+                return tuple(routes)
+        except Exception:  # noqa: BLE001 — a malformed manifest falls back to root
+            pass
+    return ("/",)
+
+
+ROUTES = _load_routes()
+
+
+def _no_horizontal_overflow(page: Page, surface_slug: str) -> None:
+    page.set_viewport_size({"width": MOBILE[0], "height": MOBILE[1]})
+    for route in ROUTES:
+        page.goto(route, wait_until="load")
+        page.wait_for_load_state("networkidle")
+        widths = page.evaluate(
+            "() => ({ scroll: document.documentElement.scrollWidth,"
+            " client: document.documentElement.clientWidth })"
+        )
+        assert widths["scroll"] <= widths["client"] + _OVERFLOW_TOLERANCE_PX, (
+            f"{surface_slug} {route} overflows horizontally at {MOBILE[0]}px "
+            f"(scrollWidth {widths['scroll']} > clientWidth {widths['client']})"
+        )
+<% graphicalSurfaces.forEach((s) => { %>
+
+def test_<%= s.ident %>_no_horizontal_overflow(cluster, <%= s.ident %>_page: Page):
+    _no_horizontal_overflow(<%= s.ident %>_page, "<%= s.slug %>")
+<% }) %><% } -%>

package/src/generators/system-test-runner/files/tests/system/test_render_smoke.py.template

@@ -0,0 +1,227 @@
+<% if (!surfaces) { -%>
+"""Render-smoke gate for the graphical UI — Tier 1 of the visual verification loop.
+
+A behavioural test asserting a selector exists passes while the page is a blank
+screen, throwing a hydration error, unstyled because the CSS 404'd, or showing an
+error-boundary fallback. This gate catches that class: it boots the app and
+asserts the objective facts a broken page violates, across a viewport x theme
+matrix, and writes a screenshot of each (route, viewport, theme) so the delivery
+agent (Tier 2) and the fidelity critic (Tier 3) can read what actually rendered.
+
+What this gate sees: render correctness, console/asset hygiene, blank-render, and
+basic theme/viewport breakage. What it does NOT see: motion (easing, durations,
+press physics) and perceived latency — those stay behaviour-tested, never asserted
+from a static screenshot.
+
+Skips cleanly when no frontend service is discovered (frontend_base_url skips).
+"""
+
+import json
+import pathlib
+
+
+def _load_routes() -> tuple[str, ...]:
+    """Route inventory — the screens the route-driven gates sweep. A bet declares
+    the routes it touched in tests/system/routes.json (a JSON array of paths);
+    absent that file, the gate covers the app root."""
+    manifest = pathlib.Path("tests/system/routes.json")
+    if manifest.exists():
+        try:
+            routes = json.loads(manifest.read_text())
+            if isinstance(routes, list) and routes:
+                return tuple(routes)
+        except Exception:  # noqa: BLE001 — a malformed manifest falls back to root
+            pass
+    return ("/",)
+
+import pytest
+from playwright.sync_api import Page
+
+# The design system commits dual themes and responsive breakpoints; the matrix is
+# the test projection of those commitments. Extend per project as the spec grows.
+VIEWPORTS = {"mobile": (375, 812), "desktop": (1280, 800)}
+THEMES = ("light", "dark")
+# Route inventory: the bet's declared routes (tests/system/routes.json) or the root.
+ROUTES = _load_routes()
+# DOM node floor a non-blank page clears — catches the blank-screen and the
+# dark-mode-invisible-text cases.
+_MIN_DOM_NODES = 8
+_VISUAL_DIR = pathlib.Path(".groundwork/cache/visual/_smoke")
+
+
+def _render_smoke(page: Page, surface_slug: str, base_url: str | None) -> None:
+    console_errors: list[str] = []
+    failed_requests: list[str] = []
+    page.on(
+        "console",
+        lambda msg: console_errors.append(msg.text) if msg.type == "error" else None,
+    )
+    page.on("pageerror", lambda exc: console_errors.append(f"pageerror: {exc}"))
+    page.on(
+        "response",
+        lambda r: failed_requests.append(f"{r.status} {r.url}")
+        if r.status >= 400 and (base_url is None or r.url.startswith(base_url))
+        else None,
+    )
+
+    for vp_name, (w, h) in VIEWPORTS.items():
+        for theme in THEMES:
+            page.set_viewport_size({"width": w, "height": h})
+            page.emulate_media(color_scheme=theme)
+            for route in ROUTES:
+                console_errors.clear()
+                failed_requests.clear()
+                target = (base_url.rstrip("/") + route) if base_url else route
+                resp = page.goto(target, wait_until="load")
+                page.wait_for_load_state("networkidle")
+
+                ctx = f"{surface_slug} {route} [{vp_name}/{theme}]"
+
+                # 1. Navigation succeeded (never a 4xx/5xx document response).
+                assert resp is not None and resp.status < 400, (
+                    f"{ctx}: navigation returned "
+                    f"{resp.status if resp else 'no response'}"
+                )
+
+                # 2. No severe console output and no uncaught exceptions.
+                assert not console_errors, (
+                    f"{ctx}: severe console output on load:\n  "
+                    + "\n  ".join(console_errors)
+                )
+
+                # 3. No failed same-origin asset/API requests (the unstyled-CSS-404 case).
+                assert not failed_requests, (
+                    f"{ctx}: failed same-origin requests:\n  "
+                    + "\n  ".join(failed_requests)
+                )
+
+                # 4. No framework error overlay or app error-boundary fallback.
+                overlay = page.locator(
+                    "nextjs-portal, [data-nextjs-dialog], [data-testid='error-boundary']"
+                )
+                assert overlay.count() == 0, f"{ctx}: an error overlay/boundary is present"
+
+                # 5. The page rendered meaningful content (blank-screen case).
+                metrics = page.evaluate(
+                    "() => ({ nodes: document.querySelectorAll('*').length,"
+                    " text: (document.body.innerText || '').trim().length })"
+                )
+                assert metrics["nodes"] >= _MIN_DOM_NODES and metrics["text"] > 0, (
+                    f"{ctx}: page rendered blank "
+                    f"({metrics['nodes']} nodes, {metrics['text']} chars of text)"
+                )
+
+                # On pass, persist the screenshot for Tiers 2-3 to read.
+                out = _VISUAL_DIR / surface_slug
+                out.mkdir(parents=True, exist_ok=True)
+                slug = route.strip("/").replace("/", "_") or "root"
+                page.screenshot(path=str(out / f"{slug}__{vp_name}__{theme}.png"))
+
+
+def test_frontend_root_render_smoke(cluster, frontend_base_url, page: Page):
+    _render_smoke(page, "frontend", frontend_base_url)
+<% } else { -%>
+"""Render-smoke gate, one test per graphical surface — Tier 1 of the visual loop.
+
+A behavioural test asserting a selector exists passes while the page is a blank
+screen, throwing a hydration error, unstyled because the CSS 404'd, or showing an
+error-boundary fallback. This gate catches that class: it boots each surface and
+asserts the objective facts a broken page violates, across a viewport x theme
+matrix, and writes a screenshot of each (route, viewport, theme) so the delivery
+agent (Tier 2) and the fidelity critic (Tier 3) can read what actually rendered.
+
+What this gate sees: render correctness, console/asset hygiene, blank-render, and
+basic theme/viewport breakage. What it does NOT see: motion (easing, durations,
+press physics) and perceived latency — those stay behaviour-tested, never asserted
+from a static screenshot.
+
+Skips cleanly when a surface is not reachable (its page fixture skips).
+"""
+
+import json
+import pathlib
+
+
+def _load_routes() -> tuple[str, ...]:
+    """Route inventory — the screens the route-driven gates sweep. A bet declares
+    the routes it touched in tests/system/routes.json (a JSON array of paths);
+    absent that file, the gate covers the app root."""
+    manifest = pathlib.Path("tests/system/routes.json")
+    if manifest.exists():
+        try:
+            routes = json.loads(manifest.read_text())
+            if isinstance(routes, list) and routes:
+                return tuple(routes)
+        except Exception:  # noqa: BLE001 — a malformed manifest falls back to root
+            pass
+    return ("/",)
+
+from playwright.sync_api import Page
+
+VIEWPORTS = {"mobile": (375, 812), "desktop": (1280, 800)}
+THEMES = ("light", "dark")
+ROUTES = _load_routes()
+_MIN_DOM_NODES = 8
+_VISUAL_DIR = pathlib.Path(".groundwork/cache/visual/_smoke")
+
+
+def _render_smoke(page: Page, surface_slug: str) -> None:
+    console_errors: list[str] = []
+    failed_requests: list[str] = []
+    page.on(
+        "console",
+        lambda msg: console_errors.append(msg.text) if msg.type == "error" else None,
+    )
+    page.on("pageerror", lambda exc: console_errors.append(f"pageerror: {exc}"))
+    page.on(
+        "response",
+        lambda r: failed_requests.append(f"{r.status} {r.url}")
+        if r.status >= 400
+        else None,
+    )
+
+    for vp_name, (w, h) in VIEWPORTS.items():
+        for theme in THEMES:
+            page.set_viewport_size({"width": w, "height": h})
+            page.emulate_media(color_scheme=theme)
+            for route in ROUTES:
+                console_errors.clear()
+                failed_requests.clear()
+                resp = page.goto(route, wait_until="load")
+                page.wait_for_load_state("networkidle")
+
+                ctx = f"{surface_slug} {route} [{vp_name}/{theme}]"
+
+                assert resp is not None and resp.status < 400, (
+                    f"{ctx}: navigation returned "
+                    f"{resp.status if resp else 'no response'}"
+                )
+                assert not console_errors, (
+                    f"{ctx}: severe console output on load:\n  "
+                    + "\n  ".join(console_errors)
+                )
+                assert not failed_requests, (
+                    f"{ctx}: failed requests:\n  " + "\n  ".join(failed_requests)
+                )
+                overlay = page.locator(
+                    "nextjs-portal, [data-nextjs-dialog], [data-testid='error-boundary']"
+                )
+                assert overlay.count() == 0, f"{ctx}: an error overlay/boundary is present"
+                metrics = page.evaluate(
+                    "() => ({ nodes: document.querySelectorAll('*').length,"
+                    " text: (document.body.innerText || '').trim().length })"
+                )
+                assert metrics["nodes"] >= _MIN_DOM_NODES and metrics["text"] > 0, (
+                    f"{ctx}: page rendered blank "
+                    f"({metrics['nodes']} nodes, {metrics['text']} chars of text)"
+                )
+
+                out = _VISUAL_DIR / surface_slug
+                out.mkdir(parents=True, exist_ok=True)
+                slug = route.strip("/").replace("/", "_") or "root"
+                page.screenshot(path=str(out / f"{slug}__{vp_name}__{theme}.png"))
+<% graphicalSurfaces.forEach((s) => { %>
+
+def test_<%= s.ident %>_render_smoke(cluster, <%= s.ident %>_page: Page):
+    _render_smoke(<%= s.ident %>_page, "<%= s.slug %>")
+<% }) %><% } -%>