groundwork-method 0.0.1 → 0.11.0

package/src/generators/system-test-runner/files/tests/conftest.py.template

@@ -0,0 +1,503 @@
+"""Shared fixtures for the system and bet-progress test suites.
+
+This file is the single source of truth for service discovery and state
+management. Both tests/system/ and tests/bets/ import from here.
+
+Services and their ports/DBs are DISCOVERED at runtime by parsing the
+workspace docker-compose.yml — there are no hardcoded ports and no
+manifest file. Adding a service to the workspace automatically brings
+it under test.
+
+FAIL LOUD, NOT SILENT: when GROUNDWORK_REQUIRE_SERVICES=1 (set by CI and
+by `./dev test integration`), an unreachable service is a FAILURE. Locally
+(flag unset) the same condition skips, so the inner loop stays ergonomic.
+"""
+
+<% if (flutterSurfaces.length) { -%>
+import json
+<% } -%>
+import os
+<% if (cliSurfaces.length || flutterSurfaces.length || electronSurfaces.length) { -%>
+import shlex
+<% } -%>
+<% if (flutterSurfaces.length) { -%>
+import shutil
+<% } -%>
+<% if (cliSurfaces.length || flutterSurfaces.length || electronSurfaces.length) { -%>
+import subprocess
+<% } -%>
+import uuid
+from pathlib import Path
+
+import httpx
+import psycopg
+import pytest
+import yaml
+from tenacity import RetryError, retry, stop_after_delay, wait_exponential
+
+# tests/ -> workspace root (conftest.py sits at tests/, one level above system/)
+WORKSPACE_ROOT = Path(__file__).resolve().parent.parent
+COMPOSE_PATH = WORKSPACE_ROOT / "docker-compose.yml"
+SERVICES_DIR = WORKSPACE_ROOT / "services"
+
+# Postgres on the dev stack is published to host localhost:5432.
+PG_HOST = "localhost"
+PG_PORT = 5432
+PG_USER = "postgres"
+PG_PASSWORD = "postgres"
+
+# Jaeger query API (UI + /api/services + /api/traces) on the dev stack.
+JAEGER_URL = os.environ.get("TRACE_BACKEND_URL", "http://localhost:16686")
+
+REQUIRE_SERVICES = os.environ.get("GROUNDWORK_REQUIRE_SERVICES") == "1"
+
+
+def _infer_type(name: str) -> str:
+    """Infer service type from on-disk markers under services/<name>/."""
+    svc = SERVICES_DIR / name
+    if (svc / "go.mod").exists():
+        return "go"
+    if (svc / "pyproject.toml").exists():
+        return "python"
+    if (svc / "package.json").exists():
+        return "next"
+    return "unknown"
+
+
+def _health_path_for(svc_type: str) -> str:
+    if svc_type == "next":
+        return "/api/healthz"
+    # Go (huma) and Python (FastAPI) both expose /health.
+    return "/health"
+
+
+def _probe_path_for(svc_type: str) -> str:
+    """A real (traced) GET route per service type, used by the span tests.
+    /health is excluded from tracing by the otelhttp middleware, so use a
+    domain route. Go ships /api/v1/entities; the Python scaffold's example
+    router is mounted at /examples."""
+    if svc_type == "python":
+        return "/examples"
+    return "/api/v1/entities"
+
+
+def _discover_services() -> list[dict]:
+    """Parse the workspace docker-compose.yml and return APP services only.
+
+    Discriminator: an APP service has `build.context` starting with
+    `./services/`; infra services use `image:` and are skipped.
+    """
+    if not COMPOSE_PATH.exists():
+        return []
+    doc = yaml.safe_load(COMPOSE_PATH.read_text()) or {}
+    services = doc.get("services", {}) or {}
+    discovered: list[dict] = []
+    for name, spec in services.items():
+        spec = spec or {}
+        build = spec.get("build")
+        context = build.get("context") if isinstance(build, dict) else None
+        if not (isinstance(context, str) and context.startswith("./services/")):
+            continue  # infra (image:-based) service
+
+        # host_port = left side of the first "X:Y" ports mapping.
+        host_port = None
+        for mapping in spec.get("ports", []) or []:
+            left = str(mapping).split(":")[0]
+            if left.isdigit():
+                host_port = int(left)
+                break
+
+        # db_name from environment DB_NAME (supports list or dict form),
+        # default to the service key.
+        db_name = name
+        env = spec.get("environment")
+        env_pairs: dict[str, str] = {}
+        if isinstance(env, list):
+            for item in env:
+                if "=" in str(item):
+                    k, v = str(item).split("=", 1)
+                    env_pairs[k] = v
+        elif isinstance(env, dict):
+            env_pairs = {k: str(v) for k, v in env.items()}
+        if "DB_NAME" in env_pairs:
+            # value may be "${DB_NAME:-svc}"; take the default after :- if present.
+            raw = env_pairs["DB_NAME"]
+            if ":-" in raw:
+                db_name = raw.split(":-", 1)[1].rstrip("}")
+            elif not raw.startswith("$"):
+                db_name = raw
+
+        svc_type = _infer_type(name)
+        discovered.append(
+            {
+                "name": name,
+                "host_port": host_port,
+                "db_name": db_name,
+                # otel_name == compose service key == Go otelhttp.NewMiddleware name.
+                "otel_name": name,
+                "health_path": _health_path_for(svc_type),
+                "probe_path": _probe_path_for(svc_type),
+                "type": svc_type,
+            }
+        )
+    return discovered
+
+
+@pytest.fixture(scope="session")
+def services_manifest() -> list[dict]:
+    """Discovered APP services. Empty list is allowed (no services yet)."""
+    return _discover_services()
+
+
+def _base_url(svc: dict) -> str:
+    return f"http://localhost:{svc['host_port']}"
+
+
+@pytest.fixture(scope="session")
+def cluster(services_manifest):
+    """Health-gate the dev stack: poll every discovered service's health
+    endpoint plus the jaeger query API. FAIL if REQUIRE_SERVICES and anything
+    is unreachable after the timeout; otherwise skip (local inner loop)."""
+
+    @retry(stop=stop_after_delay(300), wait=wait_exponential(multiplier=1, min=1, max=8))
+    def _wait():
+        unreachable = []
+        for svc in services_manifest:
+            if svc["host_port"] is None:
+                continue
+            url = f"{_base_url(svc)}{svc['health_path']}"
+            try:
+                r = httpx.get(url, timeout=2.0)
+                if r.status_code != 200:
+                    unreachable.append(url)
+            except httpx.HTTPError:
+                unreachable.append(url)
+        try:
+            j = httpx.get(f"{JAEGER_URL}/api/services", timeout=2.0)
+            if j.status_code != 200:
+                unreachable.append(f"{JAEGER_URL}/api/services")
+        except httpx.HTTPError:
+            unreachable.append(f"{JAEGER_URL}/api/services")
+        if unreachable:
+            raise RuntimeError(f"unreachable: {unreachable}")
+
+    try:
+        _wait()
+    except (RetryError, RuntimeError) as exc:
+        if REQUIRE_SERVICES:
+            pytest.fail(
+                f"dev stack not healthy and GROUNDWORK_REQUIRE_SERVICES=1: {exc}"
+            )
+        pytest.skip(f"dev stack not reachable (run `./dev start`): {exc}")
+    return services_manifest
+
+
+@pytest.fixture
+def trace_id():
+    """A W3C trace id (32 hex chars)."""
+    return uuid.uuid4().hex
+
+
+@pytest.fixture
+async def api_client(trace_id):
+    """Async HTTPX client that stamps a W3C traceparent so test traffic is
+    identifiable in jaeger."""
+    span_id = uuid.uuid4().hex[:16]
+    traceparent = f"00-{trace_id}-{span_id}-01"
+    headers = {"traceparent": traceparent, "x-test-run": "system-test"}
+    async with httpx.AsyncClient(headers=headers, timeout=10.0) as client:
+        yield client
+
+
+@pytest.fixture(scope="function", autouse=True)
+def pure_state_reset(services_manifest):
+    """Truncate every table in each discovered service's database before each
+    test. Runs for BOTH system/ and bets/ suites (autouse is rootdir-scoped).
+    Tolerates a missing DB (not yet created) and an empty table set."""
+    for svc in services_manifest:
+        dsn = (
+            f"postgresql://{PG_USER}:{PG_PASSWORD}@{PG_HOST}:{PG_PORT}/{svc['db_name']}"
+        )
+        try:
+            with psycopg.connect(dsn, autocommit=True) as conn:
+                with conn.cursor() as cur:
+                    cur.execute(
+                        "SELECT tablename FROM pg_tables WHERE schemaname='public'"
+                    )
+                    tables = [row[0] for row in cur.fetchall()]
+                    if tables:
+                        joined = ", ".join(f'"{t}"' for t in tables)
+                        cur.execute(
+                            f"TRUNCATE {joined} RESTART IDENTITY CASCADE"
+                        )
+        except psycopg.OperationalError:
+            pass  # DB may not exist yet (pre-migrate); tolerate it.
+    yield
+
+
+<% if (!surfaces && interfaceMedium === 'graphical-ui') { %>
+@pytest.fixture(scope="session")
+def frontend_base_url(services_manifest):
+    """Base URL for the Next.js frontend service. Used by graphical-ui bet-progress tests.
+
+    Discovers the first service of type 'next' from the workspace docker-compose.yml
+    and returns its host URL. Skips cleanly when no Next.js service is present.
+    """
+    next_svcs = [s for s in services_manifest if s.get("type") == "next"]
+    if not next_svcs:
+        pytest.skip("No Next.js service found in docker-compose.yml — frontend_base_url unavailable")
+    svc = next_svcs[0]
+    if svc["host_port"] is None:
+        pytest.skip(f"Next.js service '{svc['name']}' has no published host port")
+    return f"http://localhost:{svc['host_port']}"
+<% } %><% if (surfaces) { %>
+# ---------------------------------------------------------------------------
+# Surface fixtures
+#
+# `surfaces` is the canonical per-surface reach map, derived from the surface
+# registry (docs/surfaces.md / .groundwork/surfaces.json) at scaffold time.
+# Per-surface runner fixtures are generated by test medium:
+#   playwright          -> <slug>_page    (Playwright page scoped to that surface)
+#   subprocess-cli      -> <slug>_runner  (invokes the binary, returns CompletedProcess)
+#   protocol-client     -> <slug>_client  (httpx client bound to the endpoint)
+#   flutter-integration -> <slug>_runner  (drives the app's own integration_test
+#                                          suite as a subprocess via its Nx target)
+#   playwright-electron -> <slug>_runner  (drives the app's own Playwright
+#                                          _electron smoke as a subprocess via its Nx target)
+# A surface whose medium has no generated family (e.g. a scaffold: manual
+# surface on bespoke tooling) is still registered in `surfaces` so tests can name it; the
+# implementation must keep this registration honest when it lands — a health
+# endpoint, ./dev integration, and a reach value tests can use.
+# ---------------------------------------------------------------------------
+
+# Scaffold-time surface specs. reach=None means "discover at runtime": URL
+# mediums resolve the docker-compose service named after the slug.
+_SURFACE_SPECS: dict[str, dict] = {
+<% surfaces.forEach((s) => { %>    "<%= s.slug %>": {"medium": "<%= s.medium %>", "reach": <%- s.reach ? JSON.stringify(s.reach) : 'None' %>},
+<% }) %>}
+
+_URL_MEDIUMS = {"playwright", "protocol-client"}
+<% if (flutterSurfaces.length || electronSurfaces.length) { %>
+# App-harness mediums -> the Nx target that runs the surface's OWN test suite.
+# Their reach is a command (like subprocess-cli), not a URL: the suite, not
+# this conftest, holds the Dart/Electron driver.
+_HARNESS_MEDIUMS = {
+    "flutter-integration": "test-integration",
+    "playwright-electron": "smoke",
+}
+<% } %>
+
+def _surface_reach(slug: str, spec: dict, manifest: list[dict]) -> str | None:
+    """Resolve a surface's reach: the scaffold-time value when set, otherwise
+    the compose service whose name matches the slug (URL mediums) or the
+    scaffolded app's Nx test target (app-harness mediums)."""
+    if spec["reach"] is not None:
+        return spec["reach"]
+<% if (flutterSurfaces.length || electronSurfaces.length) { %>    if spec["medium"] in _HARNESS_MEDIUMS:
+        if (SERVICES_DIR / slug / "project.json").exists():
+            return f"npx nx run {slug}:{_HARNESS_MEDIUMS[spec['medium']]}"
+        return None
+<% } %>    if spec["medium"] not in _URL_MEDIUMS:
+        return None
+    for svc in manifest:
+        if svc["name"] == slug and svc["host_port"] is not None:
+            return f"http://localhost:{svc['host_port']}"
+<% if (graphicalSurfaces.length === 1) { %>    # Single graphical surface: fall back to the first Next.js service — the
+    # same discovery the deprecated frontend_base_url fixture used.
+    if spec["medium"] == "playwright":
+        for svc in manifest:
+            if svc.get("type") == "next" and svc["host_port"] is not None:
+                return f"http://localhost:{svc['host_port']}"
+<% } %>    return None
+
+
+@pytest.fixture(scope="session")
+def surfaces(services_manifest) -> dict[str, dict]:
+    """Surface slug -> {"slug", "medium", "reach"} for every registered surface.
+
+    reach is a base URL for playwright/protocol-client surfaces, a launch
+    command for subprocess-cli surfaces, the test-harness command for
+    flutter-integration/playwright-electron surfaces, and None when the
+    surface is not reachable yet (its runner fixture skips).
+    """
+    return {
+        slug: {
+            "slug": slug,
+            "medium": spec["medium"],
+            "reach": _surface_reach(slug, spec, services_manifest),
+        }
+        for slug, spec in _SURFACE_SPECS.items()
+    }
+<% graphicalSurfaces.forEach((s) => { %>
+
+@pytest.fixture
+def <%= s.ident %>_page(browser, surfaces):
+    """Playwright page for the `<%= s.slug %>` surface. Relative page.goto()
+    paths resolve against the surface's base URL."""
+    base = surfaces["<%= s.slug %>"]["reach"]
+    if base is None:
+        pytest.skip("surface '<%= s.slug %>' has no reachable base URL (is its service in docker-compose.yml?)")
+    context = browser.new_context(base_url=base)
+    page = context.new_page()
+    yield page
+    context.close()
+<% }) %><% cliSurfaces.forEach((s) => { %>
+
+@pytest.fixture
+def <%= s.ident %>_runner(surfaces):
+    """Subprocess runner for the `<%= s.slug %>` CLI surface: run(args, ...) ->
+    subprocess.CompletedProcess. Drive interactive (REPL) flows with pexpect
+    against the same launch command."""
+    command = surfaces["<%= s.slug %>"]["reach"]
+    if command is None:
+        pytest.skip("surface '<%= s.slug %>' has no launch command registered")
+
+    def run(args, input=None, timeout=30, env=None):
+        cmd = [*shlex.split(command), *[str(a) for a in args]]
+        return subprocess.run(
+            cmd,
+            input=input,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            env={**os.environ, **(env or {})},
+        )
+
+    return run
+<% }) %><% protocolSurfaces.forEach((s) => { %>
+
+@pytest.fixture
+async def <%= s.ident %>_client(surfaces):
+    """Async httpx client bound to the `<%= s.slug %>` protocol endpoint."""
+    endpoint = surfaces["<%= s.slug %>"]["reach"]
+    if endpoint is None:
+        pytest.skip("surface '<%= s.slug %>' has no reachable endpoint (is its service in docker-compose.yml?)")
+    async with httpx.AsyncClient(base_url=endpoint, timeout=10.0) as client:
+        yield client
+<% }) %><% if (flutterSurfaces.length || electronSurfaces.length) { %>
+
+def _core_base_url(manifest: list[dict]) -> str | None:
+    """The capability core's gateway URL for app-harness surfaces: the compose
+    service named (or suffixed) `gateway` when one exists, otherwise the first
+    discovered service with a published port — the same booted topology the
+    API tests hit."""
+    candidates = [s for s in manifest if s["host_port"] is not None]
+    for svc in candidates:
+        if svc["name"] == "gateway" or svc["name"].endswith("-gateway"):
+            return _base_url(svc)
+    return _base_url(candidates[0]) if candidates else None
+
+
+def _harness_runner(slug: str, command: str, *, default_args=(), default_env=None):
+    """Build a run() callable that drives a surface's OWN test harness as a
+    subprocess — the only honest integration this conftest can offer a Dart or
+    Electron suite, and what keeps the surface loop thin (prove-once: the
+    harness asserts wiring and rendering; capability behaviour is already
+    proven headless at the core's contract). Running through the app's Nx
+    target reuses its tool/*_exec.sh toolchain guard; a guard exit ("tier
+    skipped") becomes pytest.skip with the guard's reason — skipped with
+    reason, never silently green."""
+
+    def run(extra_args=(), timeout=900, env=None):
+        cmd = [*shlex.split(command), *[str(a) for a in (*default_args, *extra_args)]]
+        proc = subprocess.run(
+            cmd,
+            cwd=WORKSPACE_ROOT,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            env={**os.environ, **(default_env or {}), **(env or {})},
+        )
+        if proc.returncode != 0 and "tier skipped" in (proc.stderr or ""):
+            pytest.skip(
+                f"surface '{slug}' harness unavailable — skipped, not silently "
+                f"green: {proc.stderr.strip()}"
+            )
+        return proc
+
+    return run
+<% } %><% flutterSurfaces.forEach((s) => { %>
+
+@pytest.fixture
+def <%= s.ident %>_runner(surfaces, services_manifest):
+    """Flutter integration_test runner for the `<%= s.slug %>` mobile surface:
+    run(extra_args, ...) -> subprocess.CompletedProcess, driving the app's own
+    integration_test suite as a subprocess via its `test-integration` Nx
+    target. Prove-once: that suite asserts wiring and rendering only —
+    capability behaviour is already proven headless at the core's contract,
+    which is what keeps the expensive emulator loop thin. The core's gateway
+    URL is forwarded via --dart-define=API_BASE_URL so surface tests hit the
+    same booted topology the API tests use. Skips with reason when the Flutter
+    SDK or a device/emulator is unavailable — never silently green. Patrol is
+    the escalation path when a flow crosses the Flutter/OS boundary
+    (permission dialogs, push, system sheets); it is not wired here."""
+    command = surfaces["<%= s.slug %>"]["reach"]
+    if command is None:
+        pytest.skip("surface '<%= s.slug %>' has no test-harness command (is services/<%= s.slug %> scaffolded?)")
+    if shutil.which("flutter") is None:
+        pytest.skip(
+            "flutter SDK not found — '<%= s.slug %>' surface tier skipped, not "
+            "silently green (install the Flutter SDK, then `npx nx run <%= s.slug %>:bootstrap`)"
+        )
+    try:
+        probe = subprocess.run(
+            ["flutter", "devices", "--machine"],
+            capture_output=True,
+            text=True,
+            timeout=120,
+        )
+        devices = json.loads(probe.stdout or "[]")
+    except (OSError, subprocess.SubprocessError, ValueError):
+        devices = []
+    # Only platforms this app project supports count: a host machine usually
+    # reports desktop/web "devices" (macOS, Chrome) that `flutter test
+    # integration_test` cannot target for an android/ios scaffold — counting
+    # them turns the intended skip into a hard no-devices failure.
+    mobile_devices = [
+        d
+        for d in devices
+        if str(d.get("targetPlatform", "")).startswith(("android", "ios"))
+    ]
+    if not mobile_devices:
+        pytest.skip(
+            "no Android/iOS device or emulator available — '<%= s.slug %>' surface tier "
+            "skipped, not silently green (boot a headless Android emulator; "
+            "docs/principles/stack/flutter/testing.md)"
+        )
+    api_base = _core_base_url(services_manifest)
+    # `--` so Nx forwards the define verbatim to tool/flutter_exec.sh.
+    dart_defines = ["--", f"--dart-define=API_BASE_URL={api_base}"] if api_base else []
+    return _harness_runner("<%= s.slug %>", command, default_args=dart_defines)
+<% }) %><% electronSurfaces.forEach((s) => { %>
+
+@pytest.fixture
+def <%= s.ident %>_runner(surfaces, services_manifest):
+    """Playwright _electron smoke runner for the `<%= s.slug %>` desktop surface:
+    run(extra_args, ...) -> subprocess.CompletedProcess, driving the app's own
+    Playwright _electron suite as a subprocess via its `smoke` Nx target.
+    Prove-once: that suite asserts boot, wiring, and rendering only —
+    capability behaviour is already proven headless at the core's contract.
+    tool/electron_exec.sh owns the toolchain question (Electron binary, xvfb
+    on displayless Linux); its guard exits are translated here into
+    pytest.skip with the guard's reason — never silently green. The core's
+    gateway URL is passed via the API_BASE_URL environment variable."""
+    command = surfaces["<%= s.slug %>"]["reach"]
+    if command is None:
+        pytest.skip("surface '<%= s.slug %>' has no test-harness command (is services/<%= s.slug %> scaffolded?)")
+    api_base = _core_base_url(services_manifest)
+    env = {"API_BASE_URL": api_base} if api_base else {}
+    return _harness_runner("<%= s.slug %>", command, default_env=env)
+<% }) %><% if (graphicalSurfaces.length === 1) { %>
+
+@pytest.fixture(scope="session")
+def frontend_base_url(surfaces):
+    """DEPRECATED alias for surfaces["<%= graphicalSurfaces[0].slug %>"]["reach"].
+
+    Generated only while exactly one graphical surface exists; new tests read
+    the `surfaces` fixture instead.
+    """
+    base = surfaces["<%= graphicalSurfaces[0].slug %>"]["reach"]
+    if base is None:
+        pytest.skip("surface '<%= graphicalSurfaces[0].slug %>' has no reachable base URL — frontend_base_url unavailable")
+    return base
+<% } %><% } %>

package/src/generators/system-test-runner/files/tests/pyproject.toml.template

@@ -0,0 +1,20 @@
+[project]
+name = "tests"
+version = "0.1.0"
+description = "System Test Runner for <%= projectPrefix %>"
+requires-python = ">=3.11"
+dependencies = [
+    "pytest>=8.0.0",
+    "pytest-asyncio>=0.23.0",
+    "pytest-timeout>=2.3.1",
+    "httpx>=0.27.0",
+    "tenacity>=8.2.3",
+    "psycopg[binary]>=3.1.18",
+    "PyYAML>=6.0",<% if (includePlaywright) { %>
+    "pytest-playwright>=0.5",<% } %><% if (includePexpect) { %>
+    "pexpect>=4.9",<% } %>
+]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["system", "bets"]

package/src/generators/system-test-runner/files/tests/system/pages/__init__.py.template

@@ -0,0 +1,9 @@
+"""Page objects for graphical-ui system tests.
+
+Model each screen as a subclass of BasePage so selectors and navigation live
+in one place instead of being repeated across tests.
+"""
+
+from .base_page import BasePage
+
+__all__ = ["BasePage"]

package/src/generators/system-test-runner/files/tests/system/pages/base_page.py.template

@@ -0,0 +1,36 @@
+"""Minimal page-object base for graphical-ui system tests.
+
+Subclass per screen and add intent-named methods (e.g. submit_login) on top of
+the shared navigation/assertion helpers here. <% if (!surfaces) { %>The frontend base URL comes from
+the conftest `frontend_base_url` fixture, which discovers the Next.js service
+from the workspace docker-compose.yml.<% } else { %>Each graphical surface's base URL
+comes from the conftest `surfaces` fixture (slug -> reach).<% } %>
+"""
+
+from playwright.sync_api import Page, expect
+
+
+class BasePage:
+    """Holds the Playwright page plus the frontend base URL.
+
+    Helpers return self so page interactions chain:
+<% if (!surfaces) { -%>
+        HomePage(page, frontend_base_url).goto("/").expect_visible("main")
+<% } else { -%>
+        HomePage(page, surfaces["<%= graphicalSurfaces.length ? graphicalSurfaces[0].slug : 'web-app' %>"]["reach"]).goto("/").expect_visible("main")
+<% } -%>
+    """
+
+    def __init__(self, page: Page, base_url: str):
+        self.page = page
+        self.base_url = base_url.rstrip("/")
+
+    def goto(self, path: str = "/") -> "BasePage":
+        """Navigate to a path under the frontend base URL and wait for load."""
+        self.page.goto(f"{self.base_url}{path}", wait_until="load")
+        return self
+
+    def expect_visible(self, selector: str, timeout_ms: float = 10_000) -> "BasePage":
+        """Assert the first element matching the selector is visible."""
+        expect(self.page.locator(selector).first).to_be_visible(timeout=timeout_ms)
+        return self