@databricks-solutions/lakebase-scm-utils 0.3.9 → 0.3.10

package/scripts/lakebase/snapshot.cli.ts

@@ -0,0 +1,117 @@
+#!/usr/bin/env node
+// CLI for Lakebase SNAPSHOTS: `lakebase-snapshot <create|delete|list> ...`.
+//
+// A snapshot is an immutable point-in-time copy of a branch. Capture the
+// committed schema once, then branch copy-on-write from it per verify cycle
+// (`lakebase-branch create --from-snapshot <id>`) instead of re-running
+// `alembic upgrade head`. Prints JSON on stdout, errors on stderr.
+//   0 = success   1 = runtime error   2 = usage error
+import { createSnapshot, deleteSnapshot, listSnapshots } from "./snapshot.js";
+
+interface Args {
+  subcommand?: string;
+  instance?: string;
+  snapshot?: string;
+  branch?: string;
+  atTime?: string;
+  atLsn?: string;
+  ttl?: string;
+  noExpiry?: boolean;
+  host?: string;
+  pretty?: boolean;
+  help?: boolean;
+}
+
+function parseArgs(argv: string[]): Args {
+  const out: Args = {};
+  let i = 0;
+  if (argv[0] && !argv[0].startsWith("-")) {
+    out.subcommand = argv[0];
+    i = 1;
+  }
+  for (; i < argv.length; i++) {
+    switch (argv[i]) {
+      case "--instance": out.instance = argv[++i]; break;
+      case "--snapshot": out.snapshot = argv[++i]; break;
+      case "--branch": out.branch = argv[++i]; break;
+      case "--at-time": out.atTime = argv[++i]; break;
+      case "--at-lsn": out.atLsn = argv[++i]; break;
+      case "--ttl": out.ttl = argv[++i]; break;
+      case "--no-expiry": out.noExpiry = true; break;
+      case "--host": out.host = argv[++i]; break;
+      case "--pretty": out.pretty = true; break;
+      case "--help": case "-h": out.help = true; break;
+    }
+  }
+  return out;
+}
+
+const USAGE = `usage: lakebase-snapshot <create|delete|list> --instance <project> [flags]
+  create  --snapshot <id> --branch <src> [--at-time <iso> | --at-lsn <lsn>] [--ttl <s> | --no-expiry]
+  delete  --snapshot <id>
+  list
+Then branch from it: lakebase-branch create --branch <v> --from-snapshot <id> --instance <project>`;
+
+function out(result: unknown, pretty: boolean): void {
+  process.stdout.write(JSON.stringify(result, null, pretty ? 2 : undefined) + "\n");
+}
+function need(a: Args, keys: (keyof Args)[]): boolean {
+  const missing = keys.filter((k) => a[k] === undefined || a[k] === "");
+  if (missing.length) {
+    process.stderr.write(`missing ${missing.map((k) => "--" + String(k)).join(", ")}\n${USAGE}\n`);
+    return false;
+  }
+  return true;
+}
+
+async function main(): Promise<number> {
+  const a = parseArgs(process.argv.slice(2));
+  if (a.help || !a.subcommand) {
+    process.stderr.write(USAGE + "\n");
+    return a.help ? 0 : 2;
+  }
+  switch (a.subcommand) {
+    case "create": {
+      if (!need(a, ["instance", "snapshot", "branch"])) return 2;
+      const raw = await createSnapshot({
+        instance: a.instance!,
+        snapshot: a.snapshot!,
+        sourceBranch: a.branch!,
+        sourceBranchTime: a.atTime,
+        sourceBranchLsn: a.atLsn,
+        ttl: a.ttl,
+        noExpiry: a.noExpiry,
+        host: a.host,
+      });
+      out({ created: a.snapshot, raw: raw.trim() || undefined }, !!a.pretty);
+      return 0;
+    }
+    case "delete": {
+      if (!need(a, ["instance", "snapshot"])) return 2;
+      await deleteSnapshot({ instance: a.instance!, snapshot: a.snapshot!, host: a.host });
+      out({ deleted: a.snapshot }, !!a.pretty);
+      return 0;
+    }
+    case "list": {
+      if (!need(a, ["instance"])) return 2;
+      const raw = await listSnapshots({ instance: a.instance!, host: a.host });
+      try {
+        out(JSON.parse(raw), !!a.pretty);
+      } catch {
+        process.stdout.write(raw);
+      }
+      return 0;
+    }
+    default:
+      process.stderr.write(`unknown subcommand '${a.subcommand}'\n${USAGE}\n`);
+      return 2;
+  }
+}
+
+main().then(
+  (code) => process.exit(code),
+  (err) => {
+    process.stderr.write(String(err instanceof Error ? err.message : err) + "\n");
+    process.exit(1);
+  },
+);

package/scripts/lakebase/snapshot.ts

@@ -0,0 +1,85 @@
+// Lakebase SNAPSHOT primitives — the fast verify-branch substrate.
+//
+// A snapshot is an immutable point-in-time copy of a branch's data. Capture the
+// committed schema ONCE (create), then branch copy-on-write from it per verify
+// cycle (createBranch({ fromSnapshot })) instead of forking + `alembic upgrade
+// head` every time. Because the snapshot is a fixed point, the per-cycle branch
+// never goes stale. "Restore" is not an in-place op in Lakebase — you restore by
+// branching from the snapshot (see createBranch's fromSnapshot path).
+//
+// API (verified): POST .../snapshots?snapshot_id=<id> with spec.source_branch
+// (+ optional source_branch_time | source_branch_lsn for PITR, + one of
+// no_expiry | ttl | expire_time). Mirrors the one-databricks-CLI-wrapper rule.
+import { runDatabricks } from "./databricks-cli.js";
+import { projectPath } from "./branch-utils.js";
+import { KIT_TIMEOUTS } from "./kit-config.js";
+
+/** Full branch resource path from a bare leaf or an already-full path. */
+function branchResource(instance: string, branch: string): string {
+  return branch.includes("/") ? branch : `${projectPath(instance)}/branches/${branch}`;
+}
+
+/** Full snapshot resource path from a bare id or an already-full path. */
+function snapshotResource(instance: string, snapshot: string): string {
+  return snapshot.includes("/") ? snapshot : `${projectPath(instance)}/snapshots/${snapshot}`;
+}
+
+export interface SnapshotSpec {
+  source_branch: string;
+  source_branch_time?: string;
+  source_branch_lsn?: string;
+  no_expiry?: boolean;
+  ttl?: string;
+}
+
+export interface CreateSnapshotArgs {
+  instance: string;
+  /** Client-chosen snapshot id (becomes the resource leaf). */
+  snapshot: string;
+  /** Branch to snapshot — bare leaf (e.g. "staging") or full path. */
+  sourceBranch: string;
+  /** PITR: snapshot the branch AS OF this ISO timestamp (source_branch_time). */
+  sourceBranchTime?: string;
+  /** PITR: snapshot AS OF this LSN (source_branch_lsn). Exclusive with sourceBranchTime. */
+  sourceBranchLsn?: string;
+  /** TTL string ("604800s"). Mutually exclusive with noExpiry. */
+  ttl?: string;
+  /** Never auto-expire. Default true when ttl is unset (the API requires one). */
+  noExpiry?: boolean;
+  host?: string;
+}
+
+/** Build the create-snapshot `spec` (pure — unit-testable without the CLI). */
+export function buildSnapshotSpec(args: CreateSnapshotArgs): SnapshotSpec {
+  if (args.sourceBranchTime && args.sourceBranchLsn) {
+    throw new Error("sourceBranchTime and sourceBranchLsn are mutually exclusive — pass at most one point-in-time.");
+  }
+  if (args.ttl && args.noExpiry === true) {
+    throw new Error(`Cannot set both ttl ("${args.ttl}") and noExpiry: true — they are mutually exclusive.`);
+  }
+  const spec: SnapshotSpec = { source_branch: branchResource(args.instance, args.sourceBranch) };
+  if (args.sourceBranchTime) spec.source_branch_time = args.sourceBranchTime;
+  else if (args.sourceBranchLsn) spec.source_branch_lsn = args.sourceBranchLsn;
+  if (args.ttl) spec.ttl = args.ttl;
+  else if (args.noExpiry ?? true) spec.no_expiry = true;
+  return spec;
+}
+
+/** Create an immutable snapshot of a branch (waits for the operation to complete). */
+export async function createSnapshot(args: CreateSnapshotArgs): Promise<string> {
+  const spec = buildSnapshotSpec(args);
+  return runDatabricks(
+    ["postgres", "create-snapshot", projectPath(args.instance), args.snapshot, "--json", JSON.stringify({ spec })],
+    { host: args.host, timeout: KIT_TIMEOUTS.cliCreateBranch },
+  );
+}
+
+export async function deleteSnapshot(args: { instance: string; snapshot: string; host?: string }): Promise<string> {
+  return runDatabricks(["postgres", "delete-snapshot", snapshotResource(args.instance, args.snapshot)], {
+    host: args.host,
+  });
+}
+
+export async function listSnapshots(args: { instance: string; host?: string }): Promise<string> {
+  return runDatabricks(["postgres", "list-snapshots", projectPath(args.instance), "-o", "json"], { host: args.host });
+}

package/templates/project/common/scripts/run-tests.sh

@@ -155,6 +155,27 @@ else
   exit 1
 fi
 
+# NOTE: a per-cycle client AC-scope (vitest -t / playwright --grep from SFTDD_TEST_IDS) was tried and
+# REVERTED (live-proven harmful, 2026-10-24): scoping the per-cycle client pass to the current AC's
+# @T specs stops the FULL client suite from running per cycle, which is what detects SUPERSESSION — a
+# prior test asserting an element the current story removed (e.g. S2 split the combined
+# `tracking-not-tracked` cell into batch/serial). Without that in-cycle RED the driver never refactors
+# the superseded test, so it surfaces LATER as a deploy-verify HALT instead of a smooth in-cycle
+# refactor. The client Vitest is cheap (jsdom) and the expensive Playwright E2E is already cost-gated
+# by cycle layer below, so the full per-cycle client run stays — it is load-bearing, not waste.
+#
+# EXCEPTION — #2 parallel-lane scope (gated): in a SCOPED-LANE drive (CONSORT_SCOPE_AC set, i.e. a
+# parallel per-AC lane / the contract story's serial-first AC), the cross-story supersession is owned by
+# the DETERMINISTIC hoist sweep, NOT by this reactive full-client-run. So there the client pass is scoped
+# to THIS AC's @T specs (it must not re-run the cross-story suite + re-trigger supersession wave-by-wave
+# on the lanes' critical path). FLEET drives (no CONSORT_SCOPE_AC) are UNCHANGED — full client suite,
+# supersession detection intact. The gate is what makes the reverted scope safe to re-introduce here.
+SFTDD_CLIENT_GREP=""
+if [ -n "${CONSORT_SCOPE_AC:-}" ] && [ -n "${SFTDD_TEST_IDS:-}" ]; then
+  SFTDD_CLIENT_GREP="@($(echo "$SFTDD_TEST_IDS" | tr -s ' ' '|' | sed 's/^|//;s/|$//'))"
+  echo "Scoped-lane client pass (CONSORT_SCOPE_AC=$CONSORT_SCOPE_AC): client tests scoped to $SFTDD_CLIENT_GREP"
+fi
+
 # React SPA client unit tests (Vitest + Testing Library). Only on a full run
 # (no positional path arg, so a per-cycle backend-layer invocation does not drag
 # in the client suite), and only when a client/ workspace is present. This is the
@@ -192,7 +213,11 @@ if [ "$#" -eq 0 ] && [ -f "$REPO_ROOT/client/package.json" ]; then
   echo "Running client strict typecheck (tsc --noEmit, matching CI's build)..."
   ( cd "$REPO_ROOT/client" && npm run typecheck --if-present )
   echo "Running client unit tests (Vitest)..."
-  ( cd "$REPO_ROOT/client" && npm test )
+  if [ -n "$SFTDD_CLIENT_GREP" ]; then
+    ( cd "$REPO_ROOT/client" && npm test -- -t "$SFTDD_CLIENT_GREP" )
+  else
+    ( cd "$REPO_ROOT/client" && npm test )
+  fi
 fi
 
 # React SPA client E2E (Playwright). The client's end-to-end suite (client/tests/e2e/*.spec.ts) MUST run
@@ -273,7 +298,23 @@ if [ "$#" -eq 0 ] && [ -d "$REPO_ROOT/client/tests/e2e" ] \
     # (old `!CI` and new `!!BASE_URL` alike) so the verify never reuses a stale
     # server bound to an unmigrated branch (issue #197's false-500 misdiagnosis).
     ensure_local_e2e_ports
-    ( cd "$REPO_ROOT/client" && ./node_modules/.bin/playwright install chromium && CI=1 npm run test:e2e )
+    e2e_rc=0
+    # #2 scoped-lane: scope the E2E to THIS AC's @T specs (gated on CONSORT_SCOPE_AC; fleet runs full).
+    if [ -n "$SFTDD_CLIENT_GREP" ]; then
+      ( cd "$REPO_ROOT/client" && ./node_modules/.bin/playwright install chromium && CI=1 npm run test:e2e -- --grep "$SFTDD_CLIENT_GREP" ) || e2e_rc=$?
+    else
+      ( cd "$REPO_ROOT/client" && ./node_modules/.bin/playwright install chromium && CI=1 npm run test:e2e ) || e2e_rc=$?
+    fi
+    # #7 within-cycle reap: Playwright's webServer SHOULD self-tear-down, but a killed/detached run
+    # leaves a uvicorn (+ vite) on the E2E ports that otherwise lingers until the NEXT cycle's
+    # scan-reaper — so a heavy-E2E cycle's repeated runs pile up. Reap them here, right after the run.
+    # NEVER when BASE_URL is set (the deploy gate owns that server, same guard as ensure_local_e2e_ports).
+    if [ -z "${BASE_URL:-}" ] && command -v lsof >/dev/null 2>&1; then
+      for p in "${E2E_BACKEND_PORT:-}" "${E2E_CLIENT_PORT:-}"; do
+        [ -n "$p" ] && lsof -ti tcp:"$p" 2>/dev/null | xargs kill 2>/dev/null || true
+      done
+    fi
+    [ "$e2e_rc" -eq 0 ] || exit "$e2e_rc"
   fi
 fi
 

package/templates/project/python/tests/conftest.py

@@ -270,6 +270,28 @@ def pytest_collection_modifyitems(config, items):
     import os
     import re
 
+    # DEFER superseded cross-story tests off the serial-first AC's verify (SFTDD_DEFER_FILES, set
+    # by the deferred serial-first AC of a parallel-lane fan-out). These prior-story tests fail only
+    # because THIS AC's drop migration superseded them; the coordinator's supersession sweep retargets
+    # them ONCE right after this AC, and the final feature-deploy verify (no defer) re-runs them
+    # retargeted. Deselected by FILE because the migration pass runs them regardless of the testid
+    # scope below, and they are typically cross-story / untagged (so always-KEPT by that scope).
+    defer_raw = (os.environ.get("SFTDD_DEFER_FILES") or "").strip()
+    if defer_raw:
+        defer_files = {f for f in re.split(r"[,\s]+", defer_raw) if f.endswith(".py")}
+        if defer_files:
+            kept, dropped = [], []
+            for item in items:
+                nodefile = item.nodeid.split("::", 1)[0].replace(os.sep, "/")
+                fspath = str(getattr(item, "fspath", "") or "").replace(os.sep, "/")
+                if any(nodefile == df or nodefile.endswith("/" + df) or fspath.endswith("/" + df) for df in defer_files):
+                    dropped.append(item)
+                else:
+                    kept.append(item)
+            if dropped:
+                config.hook.pytest_deselected(items=dropped)
+                items[:] = kept
+
     raw = (os.environ.get("SFTDD_TEST_IDS") or "").strip()
     if not raw:
         return