@databricks-solutions/lakebase-scm-utils 0.3.8 → 0.3.10

package/scripts/lakebase/snapshot.cli.ts

@@ -0,0 +1,117 @@
+#!/usr/bin/env node
+// CLI for Lakebase SNAPSHOTS: `lakebase-snapshot <create|delete|list> ...`.
+//
+// A snapshot is an immutable point-in-time copy of a branch. Capture the
+// committed schema once, then branch copy-on-write from it per verify cycle
+// (`lakebase-branch create --from-snapshot <id>`) instead of re-running
+// `alembic upgrade head`. Prints JSON on stdout, errors on stderr.
+//   0 = success   1 = runtime error   2 = usage error
+import { createSnapshot, deleteSnapshot, listSnapshots } from "./snapshot.js";
+
+interface Args {
+  subcommand?: string;
+  instance?: string;
+  snapshot?: string;
+  branch?: string;
+  atTime?: string;
+  atLsn?: string;
+  ttl?: string;
+  noExpiry?: boolean;
+  host?: string;
+  pretty?: boolean;
+  help?: boolean;
+}
+
+function parseArgs(argv: string[]): Args {
+  const out: Args = {};
+  let i = 0;
+  if (argv[0] && !argv[0].startsWith("-")) {
+    out.subcommand = argv[0];
+    i = 1;
+  }
+  for (; i < argv.length; i++) {
+    switch (argv[i]) {
+      case "--instance": out.instance = argv[++i]; break;
+      case "--snapshot": out.snapshot = argv[++i]; break;
+      case "--branch": out.branch = argv[++i]; break;
+      case "--at-time": out.atTime = argv[++i]; break;
+      case "--at-lsn": out.atLsn = argv[++i]; break;
+      case "--ttl": out.ttl = argv[++i]; break;
+      case "--no-expiry": out.noExpiry = true; break;
+      case "--host": out.host = argv[++i]; break;
+      case "--pretty": out.pretty = true; break;
+      case "--help": case "-h": out.help = true; break;
+    }
+  }
+  return out;
+}
+
+const USAGE = `usage: lakebase-snapshot <create|delete|list> --instance <project> [flags]
+  create  --snapshot <id> --branch <src> [--at-time <iso> | --at-lsn <lsn>] [--ttl <s> | --no-expiry]
+  delete  --snapshot <id>
+  list
+Then branch from it: lakebase-branch create --branch <v> --from-snapshot <id> --instance <project>`;
+
+function out(result: unknown, pretty: boolean): void {
+  process.stdout.write(JSON.stringify(result, null, pretty ? 2 : undefined) + "\n");
+}
+function need(a: Args, keys: (keyof Args)[]): boolean {
+  const missing = keys.filter((k) => a[k] === undefined || a[k] === "");
+  if (missing.length) {
+    process.stderr.write(`missing ${missing.map((k) => "--" + String(k)).join(", ")}\n${USAGE}\n`);
+    return false;
+  }
+  return true;
+}
+
+async function main(): Promise<number> {
+  const a = parseArgs(process.argv.slice(2));
+  if (a.help || !a.subcommand) {
+    process.stderr.write(USAGE + "\n");
+    return a.help ? 0 : 2;
+  }
+  switch (a.subcommand) {
+    case "create": {
+      if (!need(a, ["instance", "snapshot", "branch"])) return 2;
+      const raw = await createSnapshot({
+        instance: a.instance!,
+        snapshot: a.snapshot!,
+        sourceBranch: a.branch!,
+        sourceBranchTime: a.atTime,
+        sourceBranchLsn: a.atLsn,
+        ttl: a.ttl,
+        noExpiry: a.noExpiry,
+        host: a.host,
+      });
+      out({ created: a.snapshot, raw: raw.trim() || undefined }, !!a.pretty);
+      return 0;
+    }
+    case "delete": {
+      if (!need(a, ["instance", "snapshot"])) return 2;
+      await deleteSnapshot({ instance: a.instance!, snapshot: a.snapshot!, host: a.host });
+      out({ deleted: a.snapshot }, !!a.pretty);
+      return 0;
+    }
+    case "list": {
+      if (!need(a, ["instance"])) return 2;
+      const raw = await listSnapshots({ instance: a.instance!, host: a.host });
+      try {
+        out(JSON.parse(raw), !!a.pretty);
+      } catch {
+        process.stdout.write(raw);
+      }
+      return 0;
+    }
+    default:
+      process.stderr.write(`unknown subcommand '${a.subcommand}'\n${USAGE}\n`);
+      return 2;
+  }
+}
+
+main().then(
+  (code) => process.exit(code),
+  (err) => {
+    process.stderr.write(String(err instanceof Error ? err.message : err) + "\n");
+    process.exit(1);
+  },
+);

package/scripts/lakebase/snapshot.ts

@@ -0,0 +1,85 @@
+// Lakebase SNAPSHOT primitives — the fast verify-branch substrate.
+//
+// A snapshot is an immutable point-in-time copy of a branch's data. Capture the
+// committed schema ONCE (create), then branch copy-on-write from it per verify
+// cycle (createBranch({ fromSnapshot })) instead of forking + `alembic upgrade
+// head` every time. Because the snapshot is a fixed point, the per-cycle branch
+// never goes stale. "Restore" is not an in-place op in Lakebase — you restore by
+// branching from the snapshot (see createBranch's fromSnapshot path).
+//
+// API (verified): POST .../snapshots?snapshot_id=<id> with spec.source_branch
+// (+ optional source_branch_time | source_branch_lsn for PITR, + one of
+// no_expiry | ttl | expire_time). Mirrors the one-databricks-CLI-wrapper rule.
+import { runDatabricks } from "./databricks-cli.js";
+import { projectPath } from "./branch-utils.js";
+import { KIT_TIMEOUTS } from "./kit-config.js";
+
+/** Full branch resource path from a bare leaf or an already-full path. */
+function branchResource(instance: string, branch: string): string {
+  return branch.includes("/") ? branch : `${projectPath(instance)}/branches/${branch}`;
+}
+
+/** Full snapshot resource path from a bare id or an already-full path. */
+function snapshotResource(instance: string, snapshot: string): string {
+  return snapshot.includes("/") ? snapshot : `${projectPath(instance)}/snapshots/${snapshot}`;
+}
+
+export interface SnapshotSpec {
+  source_branch: string;
+  source_branch_time?: string;
+  source_branch_lsn?: string;
+  no_expiry?: boolean;
+  ttl?: string;
+}
+
+export interface CreateSnapshotArgs {
+  instance: string;
+  /** Client-chosen snapshot id (becomes the resource leaf). */
+  snapshot: string;
+  /** Branch to snapshot — bare leaf (e.g. "staging") or full path. */
+  sourceBranch: string;
+  /** PITR: snapshot the branch AS OF this ISO timestamp (source_branch_time). */
+  sourceBranchTime?: string;
+  /** PITR: snapshot AS OF this LSN (source_branch_lsn). Exclusive with sourceBranchTime. */
+  sourceBranchLsn?: string;
+  /** TTL string ("604800s"). Mutually exclusive with noExpiry. */
+  ttl?: string;
+  /** Never auto-expire. Default true when ttl is unset (the API requires one). */
+  noExpiry?: boolean;
+  host?: string;
+}
+
+/** Build the create-snapshot `spec` (pure — unit-testable without the CLI). */
+export function buildSnapshotSpec(args: CreateSnapshotArgs): SnapshotSpec {
+  if (args.sourceBranchTime && args.sourceBranchLsn) {
+    throw new Error("sourceBranchTime and sourceBranchLsn are mutually exclusive — pass at most one point-in-time.");
+  }
+  if (args.ttl && args.noExpiry === true) {
+    throw new Error(`Cannot set both ttl ("${args.ttl}") and noExpiry: true — they are mutually exclusive.`);
+  }
+  const spec: SnapshotSpec = { source_branch: branchResource(args.instance, args.sourceBranch) };
+  if (args.sourceBranchTime) spec.source_branch_time = args.sourceBranchTime;
+  else if (args.sourceBranchLsn) spec.source_branch_lsn = args.sourceBranchLsn;
+  if (args.ttl) spec.ttl = args.ttl;
+  else if (args.noExpiry ?? true) spec.no_expiry = true;
+  return spec;
+}
+
+/** Create an immutable snapshot of a branch (waits for the operation to complete). */
+export async function createSnapshot(args: CreateSnapshotArgs): Promise<string> {
+  const spec = buildSnapshotSpec(args);
+  return runDatabricks(
+    ["postgres", "create-snapshot", projectPath(args.instance), args.snapshot, "--json", JSON.stringify({ spec })],
+    { host: args.host, timeout: KIT_TIMEOUTS.cliCreateBranch },
+  );
+}
+
+export async function deleteSnapshot(args: { instance: string; snapshot: string; host?: string }): Promise<string> {
+  return runDatabricks(["postgres", "delete-snapshot", snapshotResource(args.instance, args.snapshot)], {
+    host: args.host,
+  });
+}
+
+export async function listSnapshots(args: { instance: string; host?: string }): Promise<string> {
+  return runDatabricks(["postgres", "list-snapshots", projectPath(args.instance), "-o", "json"], { host: args.host });
+}

package/templates/project/common/scripts/run-tests.sh

@@ -155,6 +155,27 @@ else
   exit 1
 fi
 
+# NOTE: a per-cycle client AC-scope (vitest -t / playwright --grep from SFTDD_TEST_IDS) was tried and
+# REVERTED (live-proven harmful, 2026-10-24): scoping the per-cycle client pass to the current AC's
+# @T specs stops the FULL client suite from running per cycle, which is what detects SUPERSESSION — a
+# prior test asserting an element the current story removed (e.g. S2 split the combined
+# `tracking-not-tracked` cell into batch/serial). Without that in-cycle RED the driver never refactors
+# the superseded test, so it surfaces LATER as a deploy-verify HALT instead of a smooth in-cycle
+# refactor. The client Vitest is cheap (jsdom) and the expensive Playwright E2E is already cost-gated
+# by cycle layer below, so the full per-cycle client run stays — it is load-bearing, not waste.
+#
+# EXCEPTION — #2 parallel-lane scope (gated): in a SCOPED-LANE drive (CONSORT_SCOPE_AC set, i.e. a
+# parallel per-AC lane / the contract story's serial-first AC), the cross-story supersession is owned by
+# the DETERMINISTIC hoist sweep, NOT by this reactive full-client-run. So there the client pass is scoped
+# to THIS AC's @T specs (it must not re-run the cross-story suite + re-trigger supersession wave-by-wave
+# on the lanes' critical path). FLEET drives (no CONSORT_SCOPE_AC) are UNCHANGED — full client suite,
+# supersession detection intact. The gate is what makes the reverted scope safe to re-introduce here.
+SFTDD_CLIENT_GREP=""
+if [ -n "${CONSORT_SCOPE_AC:-}" ] && [ -n "${SFTDD_TEST_IDS:-}" ]; then
+  SFTDD_CLIENT_GREP="@($(echo "$SFTDD_TEST_IDS" | tr -s ' ' '|' | sed 's/^|//;s/|$//'))"
+  echo "Scoped-lane client pass (CONSORT_SCOPE_AC=$CONSORT_SCOPE_AC): client tests scoped to $SFTDD_CLIENT_GREP"
+fi
+
 # React SPA client unit tests (Vitest + Testing Library). Only on a full run
 # (no positional path arg, so a per-cycle backend-layer invocation does not drag
 # in the client suite), and only when a client/ workspace is present. This is the
@@ -192,7 +213,11 @@ if [ "$#" -eq 0 ] && [ -f "$REPO_ROOT/client/package.json" ]; then
   echo "Running client strict typecheck (tsc --noEmit, matching CI's build)..."
   ( cd "$REPO_ROOT/client" && npm run typecheck --if-present )
   echo "Running client unit tests (Vitest)..."
-  ( cd "$REPO_ROOT/client" && npm test )
+  if [ -n "$SFTDD_CLIENT_GREP" ]; then
+    ( cd "$REPO_ROOT/client" && npm test -- -t "$SFTDD_CLIENT_GREP" )
+  else
+    ( cd "$REPO_ROOT/client" && npm test )
+  fi
 fi
 
 # React SPA client E2E (Playwright). The client's end-to-end suite (client/tests/e2e/*.spec.ts) MUST run
@@ -273,7 +298,23 @@ if [ "$#" -eq 0 ] && [ -d "$REPO_ROOT/client/tests/e2e" ] \
     # (old `!CI` and new `!!BASE_URL` alike) so the verify never reuses a stale
     # server bound to an unmigrated branch (issue #197's false-500 misdiagnosis).
     ensure_local_e2e_ports
-    ( cd "$REPO_ROOT/client" && ./node_modules/.bin/playwright install chromium && CI=1 npm run test:e2e )
+    e2e_rc=0
+    # #2 scoped-lane: scope the E2E to THIS AC's @T specs (gated on CONSORT_SCOPE_AC; fleet runs full).
+    if [ -n "$SFTDD_CLIENT_GREP" ]; then
+      ( cd "$REPO_ROOT/client" && ./node_modules/.bin/playwright install chromium && CI=1 npm run test:e2e -- --grep "$SFTDD_CLIENT_GREP" ) || e2e_rc=$?
+    else
+      ( cd "$REPO_ROOT/client" && ./node_modules/.bin/playwright install chromium && CI=1 npm run test:e2e ) || e2e_rc=$?
+    fi
+    # #7 within-cycle reap: Playwright's webServer SHOULD self-tear-down, but a killed/detached run
+    # leaves a uvicorn (+ vite) on the E2E ports that otherwise lingers until the NEXT cycle's
+    # scan-reaper — so a heavy-E2E cycle's repeated runs pile up. Reap them here, right after the run.
+    # NEVER when BASE_URL is set (the deploy gate owns that server, same guard as ensure_local_e2e_ports).
+    if [ -z "${BASE_URL:-}" ] && command -v lsof >/dev/null 2>&1; then
+      for p in "${E2E_BACKEND_PORT:-}" "${E2E_CLIENT_PORT:-}"; do
+        [ -n "$p" ] && lsof -ti tcp:"$p" 2>/dev/null | xargs kill 2>/dev/null || true
+      done
+    fi
+    [ "$e2e_rc" -eq 0 ] || exit "$e2e_rc"
   fi
 fi
 

package/templates/project/common/tests/e2e/conftest.py

@@ -25,6 +25,7 @@ import os
 import subprocess
 import tempfile
 import time
+import urllib.error
 import urllib.request
 
 import pytest
@@ -41,7 +42,10 @@ def _is_up(url: str) -> bool:
             return resp.status < 500
     except urllib.error.HTTPError as exc:  # noqa: PERF203 - a 4xx still means it's up
         return exc.code < 500
-    except Exception:
+    except (urllib.error.URLError, OSError):
+        # NOT a broad-integrity swallow: a readiness poll returns False on the server
+        # not-yet-listening (connection refused / DNS / socket timeout — all URLError/OSError),
+        # never masks a DB-integrity violation. Narrow so the smell gate does not false-positive.
         return False
 
 

package/templates/project/python/tests/conftest.py

@@ -34,7 +34,7 @@ def db_session():
 
 
 @pytest.fixture(autouse=True)
-def _restore_migration_head_after_each_test():
+def _restore_migration_head_after_each_test(request):
     """Within-run test isolation for migration up/down fixtures.
 
     Some tests drive Alembic up/down against the shared verify database (a
@@ -42,10 +42,18 @@ def _restore_migration_head_after_each_test():
     `upgrade head`). Run sequentially against ONE database, those fixtures can
     leave it half-migrated , or, if a downgrade overshoots, below a table a later
     test needs , so the next test fails with `relation ... does not exist`. This
-    autouse teardown brings the DB back to `head` after EVERY test, so each test
-    starts from the fully-migrated schema regardless of what the previous one did.
-    """
+    teardown brings the DB back to `head` after a schema-mutating test.
+
+    SCOPED to ``@pytest.mark.migration`` tests only (perf): ONLY a migration test
+    downgrades/alters the schema , and the ``migration-marker-presence`` smell
+    GUARANTEES every schema-mutating test carries that marker , so a NON-migration
+    test cannot have changed the schema and needs no restore. Restoring after every
+    test ran a full alembic env-load + DB round-trip (~1-2s) after each of the many
+    non-migration tests to 'restore' a schema they never touched , the dominant cost
+    of the non-migration pass. Skipping it there is safe by construction."""
     yield
+    if request.node.get_closest_marker("migration") is None:
+        return  # a non-migration test cannot have altered the schema (marker smell enforces it)
     try:
         from pathlib import Path
 
@@ -251,6 +259,54 @@ def _consort_capture_tables(test_id, item):
         pass
 
 
+def pytest_collection_modifyitems(config, items):
+    """Build-lane SCOPE (per-cycle waste elimination): when ``SFTDD_TEST_IDS`` is set (a space/comma
+    list of test ids), run ONLY the tests whose ``@pytest.mark.testid`` is in that set and DESELECT the
+    rest. The build's per-cycle GREEN verify sets it to the CURRENT STORY's ids, so a cycle re-runs
+    only its own story instead of every feature's suite (intra-story regressions still caught); the
+    story-close REFACTOR + feature-deploy gates leave it UNSET and run the FULL cross-story suite (that
+    is where a cross-story regression is caught, before merge). UNTAGGED items (no testid — shared
+    fixtures / infra) are always KEPT, never deselected. Unset/empty => no-op (full collection)."""
+    import os
+    import re
+
+    # DEFER superseded cross-story tests off the serial-first AC's verify (SFTDD_DEFER_FILES, set
+    # by the deferred serial-first AC of a parallel-lane fan-out). These prior-story tests fail only
+    # because THIS AC's drop migration superseded them; the coordinator's supersession sweep retargets
+    # them ONCE right after this AC, and the final feature-deploy verify (no defer) re-runs them
+    # retargeted. Deselected by FILE because the migration pass runs them regardless of the testid
+    # scope below, and they are typically cross-story / untagged (so always-KEPT by that scope).
+    defer_raw = (os.environ.get("SFTDD_DEFER_FILES") or "").strip()
+    if defer_raw:
+        defer_files = {f for f in re.split(r"[,\s]+", defer_raw) if f.endswith(".py")}
+        if defer_files:
+            kept, dropped = [], []
+            for item in items:
+                nodefile = item.nodeid.split("::", 1)[0].replace(os.sep, "/")
+                fspath = str(getattr(item, "fspath", "") or "").replace(os.sep, "/")
+                if any(nodefile == df or nodefile.endswith("/" + df) or fspath.endswith("/" + df) for df in defer_files):
+                    dropped.append(item)
+                else:
+                    kept.append(item)
+            if dropped:
+                config.hook.pytest_deselected(items=dropped)
+                items[:] = kept
+
+    raw = (os.environ.get("SFTDD_TEST_IDS") or "").strip()
+    if not raw:
+        return
+    wanted = {t for t in re.split(r"[,\s]+", raw) if t}
+    if not wanted:
+        return
+    selected, deselected = [], []
+    for item in items:
+        tid = _consort_testid(item)
+        (selected if (tid is None or tid in wanted) else deselected).append(item)
+    if deselected:
+        config.hook.pytest_deselected(items=deselected)
+        items[:] = selected
+
+
 def pytest_runtest_setup(item):
     """Emit ``running`` the moment a tagged test starts, so the dashboard shows it in-flight."""
     test_id = _consort_testid(item)