@databricks-solutions/lakebase-scm-utils 0.3.3 → 0.3.5

package/templates/project/common/consort-test-reporter.ts

@@ -0,0 +1,45 @@
+// Consort per-test "green as it runs" reporter for Playwright.
+//
+// Appends one line to `.consort/test-run.jsonl` as each test tagged `@T<id>` in its title finishes,
+// so the dashboard tails it and lights that test live — even under story cadence (one whole-suite
+// run), where otherwise every test would appear to flip at once. The `file` disambiguates a reused
+// test id across features (F1 and F2 may both carry a T37); the dashboard keys by (scenario_file + id).
+//
+// The shared, framework-agnostic capture writers (emitTestRun / emitTableSnapshot /
+// captureTablesFromRoute / findConsortDir) live in `./consort-capture` — this file is the ONLY one
+// that imports `@playwright/test/reporter`, so the Vitest/client compile never pulls in Playwright.
+// Re-exported here for back-compat with importers that still reach through this module.
+//
+// Pure observability: everything is wrapped so a reporter write can NEVER fail the suite. Tag a test
+// by putting its test-list id in the title, e.g. `test("home table lists stock @T37", …)`.
+import type { FullConfig, Reporter, TestCase, TestResult } from "@playwright/test/reporter";
+import { emitTestRun, captureTablesFromRoute } from "./consort-capture";
+
+export { findConsortDir, emitTestRun, emitTableSnapshot, captureTablesFromRoute, type TestRunStatus } from "./consort-capture";
+
+const TEST_ID = /@(T\d+(?:-[a-z0-9-]+)?)/;
+
+export default class ConsortTestReporter implements Reporter {
+  private baseURL = "";
+
+  // The app's read-only snapshot route lives at the app origin; E2E specs hit the DB THROUGH the
+  // app, so after each spec we ask the app what rows it left (the client never touches the DB).
+  onBegin(config: FullConfig): void {
+    this.baseURL = config.projects?.[0]?.use?.baseURL ?? "";
+  }
+
+  // Emit `running` the moment a tagged spec starts, so the dashboard shows it in-flight (amber)
+  // before it resolves — the dashboard's last-write-wins reduce makes running -> green compose.
+  onTestBegin(test: TestCase): void {
+    const id = TEST_ID.exec(test.title)?.[1];
+    if (id) emitTestRun(id, "running", test.location?.file ?? "", "client");
+  }
+
+  async onTestEnd(test: TestCase, result: TestResult): Promise<void> {
+    const id = TEST_ID.exec(test.title)?.[1];
+    if (!id) return;
+    const status = result.status === "passed" ? "green" : result.status === "skipped" ? "skipped" : "red";
+    emitTestRun(id, status, test.location?.file ?? "", "client");
+    await captureTablesFromRoute(this.baseURL, id, test.location?.file ?? "");
+  }
+}

package/templates/project/common/playwright.config.ts

@@ -37,7 +37,12 @@ export default defineConfig({
   // network jitter against a freshly-provisioned paired branch endpoint.
   retries: process.env.CI ? 1 : 0,
   workers: process.env.CI ? 2 : undefined,
-  reporter: process.env.CI ? [["html", { open: "never" }], ["list"]] : "list",
+  // `./consort-test-reporter.ts` appends each `@T<id>`-tagged test to `.consort/test-run.jsonl` as
+  // it finishes, so the Consort dashboard lights per-test progress live (see that file). It rides
+  // alongside the human reporters and never replaces them.
+  reporter: process.env.CI
+    ? [["html", { open: "never" }], ["list"], ["./consort-test-reporter.ts"]]
+    : [["list"], ["./consort-test-reporter.ts"]],
   use: {
     baseURL,
     trace: "on-first-retry",

package/templates/project/python/app/consort_capture.py

@@ -0,0 +1,130 @@
+"""Per-test RETRIEVED/STORED row capture (dev/test/CI only) for the Consort dashboard.
+
+The dashboard's per-test table view wants the rows each TEST actually touched — not the whole table.
+Every test (pytest / Playwright / Vitest component) exercises the DB THROUGH this app over `/api`, so
+this app is the one place that sees every query. Each test tags its requests with an
+`X-Consort-Test-Id: T<id>` header; a middleware stashes that id in a contextvar for the request, and
+two SQLAlchemy events record the rows that request touched, keyed by the test id:
+
+  * RETRIEVED  — the ORM `load` event fires as each row is materialized into a mapped instance
+                 (covers every `select(Model)` read path).
+  * STORED     — the `after_cursor_execute` event fires for INSERT/UPDATE/DELETE; the written values
+                 live in the statement's bound parameters (the app writes via `insert(...).returning()`
+                 executed as a statement, which `after_flush`/unit-of-work would NOT see).
+
+Each touched row is appended to `.consort/test-interactions.jsonl` as
+`{ts, test_id, table, op: "read"|"write", row}`. Guarded by the PRESENCE of a `.consort/` dir (dev/
+test/CI checkout, not a packaged prod app) exactly like the snapshot route — so this is simply absent
+in a real deploy. Everything is wrapped so a capture write can NEVER fail a request.
+"""
+
+from __future__ import annotations
+
+import contextvars
+import json
+import os
+import re
+import threading
+from datetime import datetime, timezone
+from typing import Any
+
+from sqlalchemy import event
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import DeclarativeBase
+
+# The current request's test id (from the X-Consort-Test-Id header), and the per-request buffer of
+# touched rows. ContextVars propagate into Starlette's sync-endpoint threadpool, so the SQLAlchemy
+# events (which fire on the DB call within the handler) see the id the middleware set.
+_TEST_ID: contextvars.ContextVar[str | None] = contextvars.ContextVar("consort_test_id", default=None)
+_BUFFER: contextvars.ContextVar[list[dict[str, Any]] | None] = contextvars.ContextVar("consort_buffer", default=None)
+
+_WRITE_LOCK = threading.Lock()
+_ROWS_CAP = 200  # per (test, table, op) within one request — keep the log bounded
+_DML = re.compile(r"^\s*(INSERT|UPDATE|DELETE)\b", re.IGNORECASE)
+_TABLE = re.compile(r'(?:INTO|UPDATE|FROM)\s+"?([A-Za-z_][A-Za-z0-9_]*)"?', re.IGNORECASE)
+
+
+def _jsonable(v: Any) -> Any:
+    if isinstance(v, datetime):
+        return v.isoformat()
+    try:
+        json.dumps(v)
+        return v
+    except (TypeError, ValueError):
+        return str(v)
+
+
+def _record(table: str, op: str, row: dict[str, Any]) -> None:
+    """Append a touched row to the current request's buffer (no-op outside a tagged request)."""
+    buf = _BUFFER.get()
+    if buf is None:
+        return
+    # Bound the buffer per (table, op) so a big list read can't explode the log.
+    n = sum(1 for e in buf if e["table"] == table and e["op"] == op)
+    if n >= _ROWS_CAP:
+        return
+    buf.append({"table": table, "op": op, "row": {k: _jsonable(v) for k, v in row.items()}})
+
+
+def _flush(path: str, test_id: str, buf: list[dict[str, Any]]) -> None:
+    if not buf:
+        return
+    ts = datetime.now(timezone.utc).isoformat()
+    lines = "".join(
+        json.dumps({"ts": ts, "test_id": test_id, "table": e["table"], "op": e["op"], "row": e["row"]}) + "\n"
+        for e in buf
+    )
+    try:
+        with _WRITE_LOCK:
+            with open(path, "a", encoding="utf-8") as fh:
+                fh.write(lines)
+    except OSError:
+        pass  # best-effort: a capture write must never fail a request
+
+
+def install(app: Any, engine: Engine, base: type[DeclarativeBase]) -> None:
+    """Wire the middleware + SQLAlchemy events. No-op unless a `.consort/` dir is present."""
+    project_root = os.path.join(os.path.dirname(__file__), os.pardir)
+    consort_dir = os.path.join(project_root, ".consort")
+    if not os.path.isdir(consort_dir):
+        return
+    interactions_path = os.path.join(consort_dir, "test-interactions.jsonl")
+
+    @app.middleware("http")
+    async def _consort_tag_request(request: Any, call_next: Any) -> Any:
+        test_id = request.headers.get("x-consort-test-id")
+        id_token = _TEST_ID.set(test_id)
+        buf_token = _BUFFER.set([] if test_id else None)
+        try:
+            return await call_next(request)
+        finally:
+            buf = _BUFFER.get()
+            if test_id and buf:
+                _flush(interactions_path, test_id, buf)
+            _BUFFER.reset(buf_token)
+            _TEST_ID.reset(id_token)
+
+    @event.listens_for(base, "load", propagate=True)
+    def _consort_on_load(target: Any, _context: Any) -> None:  # RETRIEVED
+        if _TEST_ID.get() is None:
+            return
+        try:
+            table = target.__tablename__
+            row = {c.key: getattr(target, c.key) for c in target.__table__.columns}
+            _record(table, "read", row)
+        except Exception:  # noqa: BLE001 — capture must never break a load
+            pass
+
+    @event.listens_for(engine, "after_cursor_execute")
+    def _consort_on_execute(_conn, _cursor, statement, parameters, _context, _executemany) -> None:  # STORED
+        if _TEST_ID.get() is None or not statement or not _DML.match(statement):
+            return
+        try:
+            m = _TABLE.search(statement)
+            table = m.group(1) if m else "?"
+            params_list = parameters if isinstance(parameters, (list, tuple)) else [parameters]
+            for p in params_list:
+                if isinstance(p, dict) and p:
+                    _record(table, "write", dict(p))
+        except Exception:  # noqa: BLE001
+            pass

package/templates/project/python/app/main.py

@@ -14,6 +14,58 @@ def health():
     return {"status": "ok"}
 
 
+# ── Consort dashboard table-snapshot (dev/test/CI only) ─────────────────────
+# A READ-ONLY peek at the current rows of each public table, for the Consort
+# dashboard's per-test table view. E2E (Playwright) specs mutate the DB THROUGH
+# the app, so the app is the one process already connected to the branch — the
+# E2E reporter calls this after each spec to record the DB state that spec left.
+# Guarded by the PRESENCE of a `.consort/` dir at the project root: present in a
+# dev/test/CI checkout but NOT in a packaged production app, so the route is
+# simply absent in a real deploy — no env flag, no per-server wiring. Read-only,
+# row-capped, identifier-validated.
+_PROJECT_ROOT = os.path.join(os.path.dirname(__file__), os.pardir)
+if os.path.isdir(os.path.join(_PROJECT_ROOT, ".consort")):
+    import re as _re
+
+    from sqlalchemy import text as _sql_text
+
+    from app.database import engine as _snap_engine
+
+    _SNAP_IDENT = _re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+    _SNAP_CAP = 50
+
+    @app.get("/api/_consort/table-snapshot")
+    def _consort_table_snapshot():
+        """Current rows of every public app table (except alembic_version), capped. Read-only."""
+        out: dict[str, dict] = {}
+        with _snap_engine.connect() as conn:
+            names = [
+                r[0]
+                for r in conn.execute(
+                    _sql_text(
+                        "select tablename from pg_tables where schemaname='public' "
+                        "and tablename <> 'alembic_version' order by 1"
+                    )
+                )
+            ]
+            for name in names:
+                if not _SNAP_IDENT.fullmatch(name):
+                    continue
+                rows = conn.execute(_sql_text(f'SELECT * FROM "{name}" ORDER BY 1 DESC LIMIT {_SNAP_CAP}')).mappings().all()
+                total = conn.execute(_sql_text(f'SELECT count(*) FROM "{name}"')).scalar()
+                out[name] = {"rows": [dict(r) for r in rows], "row_count": total}
+        return out
+
+    # Per-test RETRIEVED/STORED row capture: records the rows each test's `/api` requests read/wrote
+    # (keyed by the X-Consort-Test-Id header) to `.consort/test-interactions.jsonl`, for the
+    # dashboard's per-test table view. Middleware + SQLAlchemy ORM `load`/DML events; same dev/test/CI
+    # guard as the snapshot route above. See app/consort_capture.py.
+    from app import consort_capture as _consort_capture
+    from app.database import Base as _consort_base
+
+    _consort_capture.install(app, _snap_engine, _consort_base)
+
+
 # ── Register API routes ABOVE this line ─────────────────────────────────────
 # Everything below serves the built React SPA (client/dist) in production, so a
 # single-process deploy (e.g. a Databricks App) serves both the API and the UI.

package/templates/project/python/pyproject.toml

@@ -33,4 +33,5 @@ dev = [
 # downgrade/upgrade cannot corrupt the shared verify DB for sibling tests.
 markers = [
     "migration: schema-mutating migration-reversibility test; runs on an isolated ephemeral branch",
+    "testid: the test-list id (e.g. @pytest.mark.testid(\"T37\")) this test realizes, for the dashboard's per-test live signal",
 ]

package/templates/project/python/tests/conftest.py

@@ -8,9 +8,19 @@ from app.main import app
 
 
 @pytest.fixture()
-def client():
-    """FastAPI TestClient for making HTTP requests."""
-    return TestClient(app)
+def client(request):
+    """FastAPI TestClient for making HTTP requests.
+
+    Tagged with the test's ``@pytest.mark.testid`` via the ``X-Consort-Test-Id`` header, so the
+    backend's per-test capture (``app/consort_capture.py``) attributes the rows each request
+    RETRIEVES/STORES to this test — exactly as the Vitest fetch shim and the Playwright fixture do.
+    (Direct ``db_session`` setup/asserts bypass the app and are not captured, by design: the capture
+    is what the test did THROUGH the app's ``/api``.)"""
+    c = TestClient(app)
+    test_id = _consort_testid(request.node)
+    if test_id:
+        c.headers["X-Consort-Test-Id"] = test_id
+    return c
 
 
 @pytest.fixture()
@@ -54,3 +64,122 @@ def _restore_migration_head_after_each_test():
         # Everything else (a real schema error) propagates and fails loudly, which
         # is the correct behavior for a genuinely broken restore.
         print(f"WARNING: migration-head restore failed during teardown: {exc}")
+
+
+# --- Per-test "green as it runs" signal for the dashboard -------------------------------------
+# Each test tagged `@pytest.mark.testid("T37")` streams its lifecycle to `.consort/test-run.jsonl`:
+# `running` when it STARTS (pytest_runtest_setup), then `green`/`red`/`skipped` when it FINISHES
+# (pytest_runtest_makereport, the CALL phase). A dashboard tails that file and lights each bubble
+# the moment it changes, so even a single whole-suite run (story cadence) shows per-test progress —
+# pending -> running -> green — not an all-at-once batch flip. The `file` disambiguates a reused id
+# across features (F1 and F2 may both have a T37); the reader keys by (scenario_file + id) and takes
+# the LAST write per key, so running -> green composes. Observability only.
+
+
+def _consort_testid(item):
+    """The test's ``@pytest.mark.testid(...)`` id, or ``None`` when it carries no marker."""
+    return next((m.args[0] for m in item.iter_markers("testid") if m.args), None)
+
+
+def _consort_emit_test_run(test_id, status, item):
+    """Append one per-test line to ``.consort/test-run.jsonl``. Best-effort with a SCOPED except
+    (the only errors this body can raise: file I/O, JSON serialization) — never a blanket
+    ``except Exception``, which the kit's deterministic broad-integrity-except test-smell gate
+    correctly flags even in a conftest. A reporter write must never break the suite."""
+    try:
+        import json
+        from datetime import datetime, timezone
+        from pathlib import Path
+
+        root = Path(__file__).resolve().parent.parent
+        rel = str(Path(str(item.fspath))).replace(f"{root}/", "")
+        out = root / ".consort"
+        out.mkdir(parents=True, exist_ok=True)
+        line = json.dumps(
+            {"ts": datetime.now(timezone.utc).isoformat(), "test_id": test_id, "status": status, "file": rel, "kind": "behavior"}
+        )
+        with open(out / "test-run.jsonl", "a", encoding="utf-8") as fh:
+            fh.write(line + "\n")
+    except (OSError, ValueError, TypeError):
+        pass
+
+
+def _consort_capture_tables(test_id, item):
+    """Append a per-test TABLE snapshot to ``.consort/table-snapshots.jsonl``, keyed by test id: the
+    rows of each public app table AS THIS TEST LEFT THEM, captured on the CALL phase (before teardown
+    prunes). The dashboard shows the table 'as of' the running/selected test — same corpus/one-reader
+    path as the test-status view. Read-only + row-capped. A ``@migration`` test runs on its OWN
+    isolated ephemeral branch and drops/recreates the table, so its snapshot is flagged ``isolated``.
+    Best-effort with a SCOPED except (DB errors + file I/O + JSON) — never a blanket catch, never
+    breaks the suite."""
+    try:
+        import json
+        import re
+        from datetime import datetime, timezone
+        from pathlib import Path
+
+        from sqlalchemy import text
+        from sqlalchemy.exc import SQLAlchemyError
+
+        from app.database import engine
+
+        isolated = any(True for _ in item.iter_markers("migration"))
+        ident = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+        root = Path(__file__).resolve().parent.parent
+        rel = str(Path(str(item.fspath))).replace(f"{root}/", "")
+        out = root / ".consort"
+        out.mkdir(parents=True, exist_ok=True)
+        ts = datetime.now(timezone.utc).isoformat()
+        lines = []
+        try:
+            with engine.connect() as conn:
+                names = [
+                    r[0]
+                    for r in conn.execute(
+                        text(
+                            "select tablename from pg_tables where schemaname='public' "
+                            "and tablename <> 'alembic_version' order by 1"
+                        )
+                    )
+                ]
+                for name in names:
+                    if not ident.fullmatch(name):
+                        continue
+                    rows = [dict(r) for r in conn.execute(text(f'SELECT * FROM "{name}" ORDER BY 1 DESC LIMIT 50')).mappings()]
+                    total = conn.execute(text(f'SELECT count(*) FROM "{name}"')).scalar()
+                    lines.append(
+                        json.dumps(
+                            {"ts": ts, "test_id": test_id, "file": rel, "table": name, "rows": rows, "row_count": total, "isolated": isolated},
+                            default=str,
+                        )
+                    )
+        except SQLAlchemyError:
+            return  # branch unreachable / mid-migration — skip this snapshot, never fail the test
+        if lines:
+            with open(out / "table-snapshots.jsonl", "a", encoding="utf-8") as fh:
+                fh.write("\n".join(lines) + "\n")
+    except (OSError, ValueError, TypeError):
+        pass
+
+
+def pytest_runtest_setup(item):
+    """Emit ``running`` the moment a tagged test starts, so the dashboard shows it in-flight."""
+    test_id = _consort_testid(item)
+    if test_id:
+        _consort_emit_test_run(test_id, "running", item)
+
+
+@pytest.hookimpl(wrapper=True)
+def pytest_runtest_makereport(item, call):
+    """Emit the FINAL status on the CALL phase, and capture the per-test table snapshot (before
+    teardown prunes). A ``makereport`` wrapper (NOT the ``logreport`` hook, whose ``TestReport``
+    carries no test node) so the ``testid`` marker is reachable off ``item``; always returns the
+    real report unchanged so pytest's own reporting is untouched."""
+    report = yield
+    if report.when == "call":
+        test_id = _consort_testid(item)
+        if test_id:
+            status = "green" if report.passed else "red" if report.failed else "skipped"
+            _consort_emit_test_run(test_id, status, item)
+            _consort_capture_tables(test_id, item)
+    return report