@databricks-solutions/lakebase-scm-utils 0.2.46 → 0.2.48

package/dist/scripts/util/index.cjs

@@ -1466,12 +1466,14 @@ var util_exports = {};
 __export(util_exports, {
   LAKEBASE_BRANCH_NAME_MAX: () => LAKEBASE_BRANCH_NAME_MAX,
   PROXY_ENV_KEYS: () => PROXY_ENV_KEYS,
+  REQUIRED_CI_SECRETS: () => REQUIRED_CI_SECRETS,
   copyDirSubstituted: () => copyDirSubstituted,
   delay: () => delay,
   exec: () => exec2,
   extractZipToDir: () => extractZipToDir,
   formatOwnerRepo: () => formatOwnerRepo,
   isCliEntry: () => isCliEntry,
+  missingCiSecrets: () => missingCiSecrets,
   parseOwnerRepo: () => parseOwnerRepo,
   patchPomForLakebase: () => patchPomForLakebase,
   pollUntil: () => pollUntil,
@@ -1659,8 +1661,8 @@ var PKG_NAME = "@databricks-solutions/lakebase-scm-utils";
 var cached;
 function substrateSelfVersion() {
   if (cached !== void 0) return cached;
-  if ("0.2.46".length > 0) {
-    cached = "0.2.46";
+  if ("0.2.48".length > 0) {
+    cached = "0.2.48";
     return cached;
   }
   cached = "unknown";
@@ -8580,6 +8582,16 @@ async function setRepoSecrets(ownerRepo, secrets) {
     await setRepoSecret(ownerRepo, name, value);
   }
 }
+async function listSecretNames(ownerRepo) {
+  try {
+    const { owner, repo } = parseOwnerRepo(ownerRepo);
+    const octokit = await getOctokit();
+    const { data } = await octokit.rest.actions.listRepoSecrets({ owner, repo });
+    return data.secrets.map((s) => s.name);
+  } catch {
+    return [];
+  }
+}
 
 // scripts/git/remote.ts
 init_cjs_shims();
@@ -8619,6 +8631,11 @@ async function getOwnerRepo(cwd) {
 }
 
 // scripts/util/ci-secrets.ts
+var REQUIRED_CI_SECRETS = ["DATABRICKS_HOST", "LAKEBASE_PROJECT_ID", "DATABRICKS_TOKEN"];
+async function missingCiSecrets(ownerRepo) {
+  const present = new Set(await listSecretNames(ownerRepo));
+  return REQUIRED_CI_SECRETS.filter((n) => !present.has(n));
+}
 async function syncCiSecrets(args) {
   const lifetime = args.lifetimeSeconds ?? 86400;
   const comment = args.comment ?? "GitHub Actions CI";
@@ -8757,12 +8774,14 @@ async function pollUntilDefined(probe, opts) {
 0 && (module.exports = {
   LAKEBASE_BRANCH_NAME_MAX,
   PROXY_ENV_KEYS,
+  REQUIRED_CI_SECRETS,
   copyDirSubstituted,
   delay,
   exec,
   extractZipToDir,
   formatOwnerRepo,
   isCliEntry,
+  missingCiSecrets,
   parseOwnerRepo,
   patchPomForLakebase,
   pollUntil,

package/dist/scripts/util/index.d.cts

@@ -33,6 +33,14 @@ declare function copyDirSubstituted(srcDir: string, destDir: string, args?: Copy
 declare const LAKEBASE_BRANCH_NAME_MAX = 63;
 declare function sanitizeBranchName(gitBranch: string): string;
 
+/** The repo Actions secrets CI needs to provision a per-PR Lakebase branch. Missing ANY of
+ *  these leaves CI unable to produce a DATABASE_URL, so the test step aborts — the failure
+ *  that otherwise only surfaces at promotion, an entire feature after creation. */
+declare const REQUIRED_CI_SECRETS: readonly ["DATABRICKS_HOST", "LAKEBASE_PROJECT_ID", "DATABRICKS_TOKEN"];
+/** Which required CI secrets are NOT present on the repo (empty = fully provisioned). Used to
+ *  VERIFY a sync landed — a `databricks tokens create` or `gh secret set` that fails must be
+ *  caught here, not swallowed, so CI-auth is never silently absent. */
+declare function missingCiSecrets(ownerRepo: string): Promise<string[]>;
 interface SyncCiSecretsArgs {
     /** Project root (used to resolve ownerRepo from `git remote` when not given,
      *  and as the cwd for the `databricks tokens create` call). */
@@ -161,4 +169,4 @@ declare function pollUntilDefined<T>(probe: (ctx: {
     elapsedMs: number;
 }) => Promise<T | undefined>, opts: Omit<PollUntilArgs<T>, "probe">): Promise<PollUntilResult<T>>;
 
-export { type CopyDirSubstitutedArgs, LAKEBASE_BRANCH_NAME_MAX, type OwnerRepo, PROXY_ENV_KEYS, type PollProbeDone, type PollProbePending, type PollProbeResult, type PollUntilArgs, type PollUntilDoneResult, type PollUntilResult, type PollUntilTimeoutResult, type SyncCiSecretsArgs, copyDirSubstituted, delay, extractZipToDir, formatOwnerRepo, isCliEntry, parseOwnerRepo, patchPomForLakebase, pollUntil, pollUntilDefined, proxyEnvSubset, sanitizeArtifactId, sanitizeBranchName, syncCiSecrets, withProxyEnv };
+export { type CopyDirSubstitutedArgs, LAKEBASE_BRANCH_NAME_MAX, type OwnerRepo, PROXY_ENV_KEYS, type PollProbeDone, type PollProbePending, type PollProbeResult, type PollUntilArgs, type PollUntilDoneResult, type PollUntilResult, type PollUntilTimeoutResult, REQUIRED_CI_SECRETS, type SyncCiSecretsArgs, copyDirSubstituted, delay, extractZipToDir, formatOwnerRepo, isCliEntry, missingCiSecrets, parseOwnerRepo, patchPomForLakebase, pollUntil, pollUntilDefined, proxyEnvSubset, sanitizeArtifactId, sanitizeBranchName, syncCiSecrets, withProxyEnv };

package/dist/scripts/util/index.d.ts

@@ -33,6 +33,14 @@ declare function copyDirSubstituted(srcDir: string, destDir: string, args?: Copy
 declare const LAKEBASE_BRANCH_NAME_MAX = 63;
 declare function sanitizeBranchName(gitBranch: string): string;
 
+/** The repo Actions secrets CI needs to provision a per-PR Lakebase branch. Missing ANY of
+ *  these leaves CI unable to produce a DATABASE_URL, so the test step aborts — the failure
+ *  that otherwise only surfaces at promotion, an entire feature after creation. */
+declare const REQUIRED_CI_SECRETS: readonly ["DATABRICKS_HOST", "LAKEBASE_PROJECT_ID", "DATABRICKS_TOKEN"];
+/** Which required CI secrets are NOT present on the repo (empty = fully provisioned). Used to
+ *  VERIFY a sync landed — a `databricks tokens create` or `gh secret set` that fails must be
+ *  caught here, not swallowed, so CI-auth is never silently absent. */
+declare function missingCiSecrets(ownerRepo: string): Promise<string[]>;
 interface SyncCiSecretsArgs {
     /** Project root (used to resolve ownerRepo from `git remote` when not given,
      *  and as the cwd for the `databricks tokens create` call). */
@@ -161,4 +169,4 @@ declare function pollUntilDefined<T>(probe: (ctx: {
     elapsedMs: number;
 }) => Promise<T | undefined>, opts: Omit<PollUntilArgs<T>, "probe">): Promise<PollUntilResult<T>>;
 
-export { type CopyDirSubstitutedArgs, LAKEBASE_BRANCH_NAME_MAX, type OwnerRepo, PROXY_ENV_KEYS, type PollProbeDone, type PollProbePending, type PollProbeResult, type PollUntilArgs, type PollUntilDoneResult, type PollUntilResult, type PollUntilTimeoutResult, type SyncCiSecretsArgs, copyDirSubstituted, delay, extractZipToDir, formatOwnerRepo, isCliEntry, parseOwnerRepo, patchPomForLakebase, pollUntil, pollUntilDefined, proxyEnvSubset, sanitizeArtifactId, sanitizeBranchName, syncCiSecrets, withProxyEnv };
+export { type CopyDirSubstitutedArgs, LAKEBASE_BRANCH_NAME_MAX, type OwnerRepo, PROXY_ENV_KEYS, type PollProbeDone, type PollProbePending, type PollProbeResult, type PollUntilArgs, type PollUntilDoneResult, type PollUntilResult, type PollUntilTimeoutResult, REQUIRED_CI_SECRETS, type SyncCiSecretsArgs, copyDirSubstituted, delay, extractZipToDir, formatOwnerRepo, isCliEntry, missingCiSecrets, parseOwnerRepo, patchPomForLakebase, pollUntil, pollUntilDefined, proxyEnvSubset, sanitizeArtifactId, sanitizeBranchName, syncCiSecrets, withProxyEnv };

package/dist/scripts/util/index.js

@@ -1630,8 +1630,8 @@ var PKG_NAME = "@databricks-solutions/lakebase-scm-utils";
 var cached;
 function substrateSelfVersion() {
   if (cached !== void 0) return cached;
-  if ("0.2.46".length > 0) {
-    cached = "0.2.46";
+  if ("0.2.48".length > 0) {
+    cached = "0.2.48";
     return cached;
   }
   cached = "unknown";
@@ -8551,6 +8551,16 @@ async function setRepoSecrets(ownerRepo, secrets) {
     await setRepoSecret(ownerRepo, name, value);
   }
 }
+async function listSecretNames(ownerRepo) {
+  try {
+    const { owner, repo } = parseOwnerRepo(ownerRepo);
+    const octokit = await getOctokit();
+    const { data } = await octokit.rest.actions.listRepoSecrets({ owner, repo });
+    return data.secrets.map((s) => s.name);
+  } catch {
+    return [];
+  }
+}
 
 // scripts/git/remote.ts
 init_esm_shims();
@@ -8590,6 +8600,11 @@ async function getOwnerRepo(cwd) {
 }
 
 // scripts/util/ci-secrets.ts
+var REQUIRED_CI_SECRETS = ["DATABRICKS_HOST", "LAKEBASE_PROJECT_ID", "DATABRICKS_TOKEN"];
+async function missingCiSecrets(ownerRepo) {
+  const present = new Set(await listSecretNames(ownerRepo));
+  return REQUIRED_CI_SECRETS.filter((n) => !present.has(n));
+}
 async function syncCiSecrets(args) {
   const lifetime = args.lifetimeSeconds ?? 86400;
   const comment = args.comment ?? "GitHub Actions CI";
@@ -8727,12 +8742,14 @@ async function pollUntilDefined(probe, opts) {
 export {
   LAKEBASE_BRANCH_NAME_MAX,
   PROXY_ENV_KEYS,
+  REQUIRED_CI_SECRETS,
   copyDirSubstituted,
   delay,
   exec2 as exec,
   extractZipToDir,
   formatOwnerRepo,
   isCliEntry,
+  missingCiSecrets,
   parseOwnerRepo,
   patchPomForLakebase,
   pollUntil,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@databricks-solutions/lakebase-scm-utils",
-  "version": "0.2.46",
+  "version": "0.2.48",
   "description": "Lakebase SCM + substrate engine: database branching, paired-branch SCM workflows, connection/credential minting, schema migration, project scaffold + deploy primitives, and the git/github utility layer. Extracted from consort for portable, standalone consumption by lakebase-scm-extension and the SFTDD orchestration.",
   "type": "module",
   "private": false,
@@ -105,7 +105,8 @@
     "lakebase-scm-prepare-pr": "./dist/scripts/lakebase/scm-prepare-pr.cli.js",
     "lakebase-scm-recover-orphans": "./dist/scripts/lakebase/scm-recover-orphans.cli.js",
     "lakebase-scm-state": "./dist/scripts/lakebase/scm-state.cli.js",
-    "lakebase-scm-wait-ci": "./dist/scripts/lakebase/scm-wait-ci.cli.js"
+    "lakebase-scm-wait-ci": "./dist/scripts/lakebase/scm-wait-ci.cli.js",
+    "lakebase-sync-ci-secrets": "./dist/scripts/lakebase/sync-ci-secrets.cli.js"
   },
   "scripts": {
     "build": "tsup",

package/scripts/github/pr.ts

@@ -67,6 +67,12 @@ export interface PullRequestInfo {
   isDraft: boolean;
   ciStatus: "pending" | "success" | "failure";
   checks: PullRequestCheck[];
+  /** GitHub's mergeability triple: true/false/null (null = still computing). */
+  mergeable?: boolean | null;
+  /** GitHub's `mergeable_state`: `clean`/`unstable`/`behind`/`blocked`/`dirty`/`unknown`.
+   *  `dirty` = merge conflict with the base — a `pull_request` CI run can't be dispatched
+   *  (there is no mergeable ref), so a waiter must surface this, not poll until timeout. */
+  mergeableState?: string;
   headBranch: string;
   baseBranch: string;
   body?: string;
@@ -184,6 +190,8 @@ export async function getPullRequest(
       isDraft: pr.draft || false,
       ciStatus,
       checks,
+      mergeable: pr.mergeable,
+      mergeableState: pr.mergeable_state,
       headBranch: pr.head?.ref || headBranch,
       baseBranch: pr.base?.ref || "",
       body: pr.body || undefined,

package/scripts/lakebase/create-project.ts

@@ -30,7 +30,7 @@ import { createLongRunningBranch } from "./long-running-branch.js";
 import { enableE2eForProject } from "./enable-e2e.js";
 import { enableInfraForProject } from "./enable-infra.js";
 import { setupRunner } from "./runner-setup.js";
-import { syncCiSecrets } from "../util/ci-secrets.js";
+import { syncCiSecrets, missingCiSecrets } from "../util/ci-secrets.js";
 import { delay } from "../util/delay.js";
 import {
   initWorkflowState,
@@ -437,10 +437,21 @@ export async function createProject(
         lifetimeSeconds: 86_400,
         ownerRepo: fullRepoName,
       });
+      // VERIFY the secrets actually landed — syncCiSecrets fail-softs a dropped PAT, and a
+      // swallowed failure here only surfaces at the first promotion CI run (no DATABASE_URL),
+      // an entire feature later. Name the exact gaps + the one-command repair so it is actionable
+      // NOW, not a buried warning.
+      const missing = await missingCiSecrets(fullRepoName);
+      if (missing.length > 0) {
+        const m = `CI auth incomplete — repo secret(s) MISSING: ${missing.join(", ")}. CI cannot provision a per-PR Lakebase branch until fixed. Repair with: lakebase-sync-ci-secrets (run in the project).`;
+        warnings.push(m);
+        report(`WARNING: ${m}`);
+      }
     } catch (err) {
       const msg = err instanceof Error ? err.message : String(err);
-      warnings.push(`CI auth setup failed: ${msg}`);
-      report(`Warning: CI auth setup failed (${msg})`);
+      const m = `CI auth setup failed: ${msg}. CI will fail (no DATABASE_URL) until repaired — run: lakebase-sync-ci-secrets (in the project).`;
+      warnings.push(m);
+      report(`WARNING: ${m}`);
     }
   }
 

package/scripts/lakebase/scm-merge.ts

@@ -12,6 +12,7 @@
 // stable surface for that signal.
 
 import {
+  getPullRequest,
   listWorkflowRuns,
   mergePairedPullRequest,
   type WorkflowRunSummary,
@@ -68,6 +69,7 @@ export class ScmMergeError extends Error {
       | "no-pr-url"
       | "bad-pr-url"
       | "merge-failed"
+      | "promote-conflict"
       | "migrate-failed"
       | "migrate-timeout"
       | "migrate-auth",
@@ -166,6 +168,13 @@ export interface MergeArgs {
    * implementation ({@link defaultRefreshPromoteWorkflows}); injected in tests.
    */
   refreshWorkflows?: () => Promise<{ refreshed: boolean; detail?: string }>;
+  /** Pre-merge reconcile of a pipeline.json-ledger promotion conflict to the run side (keeps it
+   *  tracked). Called ONLY when the PR is CONFLICTING. Defaults to {@link reconcilePromoteRunState};
+   *  injected in tests. */
+  reconcileRunState?: (baseBranch: string) => Promise<{ reconciled: boolean; detail?: string }>;
+  /** Fetch the PR's mergeability (to gate the reconcile). Defaults to the real getPullRequest;
+   *  injected in tests so the gate is exercised without the network. */
+  fetchPrState?: (ownerRepo: string, headBranch: string) => Promise<{ mergeableState?: string } | undefined>;
 }
 
 export interface MergeResult {
@@ -248,6 +257,71 @@ function shaMigratePredicate(
  * working-tree changes. Callers invoke it BEST-EFFORT — a failure here must never
  * block the merge (the local migrate fallback still keeps git + schema in sync).
  */
+/** The ONLY paths a promotion merge may auto-resolve to the RUN (feature/head) side: the per-story
+ *  pipeline.json ledger (per-feature, and the legacy top-level pointer). It is the feature branch's
+ *  own durable bookkeeping — the tier has no authoritative copy — so on a promotion conflict the
+ *  head side always wins. Everything else is real divergence and must surface, never auto-resolve. */
+const PROMOTE_RUNSTATE_CONFLICT = /(^|\/)\.consort\/(features\/[^/]+\/)?pipeline\.json$/;
+
+/**
+ * Clear a promotion-merge conflict that is ONLY the pipeline.json ledger, so GitHub can dispatch CI
+ * and merge — WITHOUT untracking the ledger (it's durable run-state that must persist). Merges the
+ * base into the PR head and resolves the ledger path(s) to the head (run) side, then pushes. This is
+ * the merge-layer fix for the "pipeline.json conflict blocks promotion" defect, replacing the wrong
+ * earlier approach of gitignoring/untracking it. Behavior:
+ *   - a dirty tree under `.consort/` only is checkpointed first (the run's own bookkeeping);
+ *   - a dirty tree OUTSIDE `.consort/` aborts (never auto-commit app code);
+ *   - a conflict on any NON-ledger path aborts the merge + surfaces (real divergence isn't resolved);
+ *   - a clean merge (no conflict) is a no-op (reconciled:false).
+ * Only call this when the PR is actually CONFLICTING — a clean promote needs no merge commit.
+ */
+export async function reconcilePromoteRunState(
+  projectDir: string,
+  baseBranch: string,
+): Promise<{ reconciled: boolean; detail?: string }> {
+  const o = { cwd: projectDir, timeout: 30_000 };
+  // Working tree must be clean to merge. Checkpoint run-state (.consort/) if that's all that's dirty;
+  // refuse on anything else so we never auto-commit app code during a promotion.
+  const dirty = (await exec("git status --porcelain", o).catch(() => "")).split("\n").map((l) => l.slice(3).trim()).filter(Boolean);
+  const nonRunStateDirty = dirty.filter((p) => !p.startsWith(".consort/"));
+  if (nonRunStateDirty.length > 0) {
+    throw new ScmMergeError(
+      `working tree has uncommitted non-run-state changes (${nonRunStateDirty.slice(0, 5).join(", ")}${nonRunStateDirty.length > 5 ? ", …" : ""}); commit or stash before promoting.`,
+      "promote-conflict",
+    );
+  }
+  if (dirty.length > 0) {
+    await exec(`git add ${shellEscape(".consort")}`, o);
+    await exec(`git commit -m ${shellEscape("chore(run): checkpoint run state before promotion")}`, o);
+  }
+  await exec(`git fetch origin ${shellEscape(baseBranch)}`, o).catch(() => {});
+  let conflicted = false;
+  try {
+    await exec(`git merge --no-ff -m ${shellEscape(`merge ${baseBranch} for promotion`)} ${shellEscape(`origin/${baseBranch}`)}`, o);
+  } catch {
+    conflicted = true;
+  }
+  if (!conflicted) return { reconciled: false }; // clean merge (or already up to date) — nothing to resolve
+  const u = (await exec("git diff --name-only --diff-filter=U", o).catch(() => "")).split("\n").map((s) => s.trim()).filter(Boolean);
+  const other = u.filter((p) => !PROMOTE_RUNSTATE_CONFLICT.test(p));
+  if (u.length === 0 || other.length > 0) {
+    await exec("git merge --abort", o).catch(() => {});
+    throw new ScmMergeError(
+      other.length > 0
+        ? `promotion conflicts on non-ledger file(s): ${other.join(", ")}. Resolve the real divergence manually, then re-run the promote.`
+        : `promotion merge failed with no resolvable conflicts; resolve manually and re-run.`,
+      "promote-conflict",
+    );
+  }
+  for (const p of u) {
+    await exec(`git checkout --ours -- ${shellEscape(p)}`, o); // keep the run (feature) ledger
+    await exec(`git add -- ${shellEscape(p)}`, o);
+  }
+  await exec(`git commit -m ${shellEscape(`merge ${baseBranch} for promotion (pipeline.json ledger kept at run state)`)}`, o);
+  await pushCurrentBranchForPr({ cwd: projectDir });
+  return { reconciled: true, detail: `reconciled ${u.length} ledger path(s) to the run side + pushed` };
+}
+
 export async function defaultRefreshPromoteWorkflows(
   projectDir: string,
 ): Promise<{ refreshed: boolean; detail?: string }> {
@@ -376,6 +450,23 @@ export async function mergeFeature(args: MergeArgs): Promise<MergeResult> {
     }
   }
 
+  // ─── Pre-merge pipeline.json-ledger reconcile ───
+  // The per-feature pipeline.json ledger is TRACKED (durable run-state) and can diverge between the
+  // feature branch and its tier, so the promotion PR lands CONFLICTING — GitHub won't dispatch CI or
+  // merge. When that happens, reconcile the ledger to the run (head) side at the MERGE layer and push,
+  // so the PR goes mergeable WITHOUT untracking the ledger. Only runs when the PR is actually dirty;
+  // a non-ledger conflict throws (real divergence surfaces). This is the correct replacement for the
+  // withdrawn gitignore/untrack approach.
+  {
+    const fetchState = args.fetchPrState ?? ((o, h) => getPullRequest(o, h));
+    const prState = await fetchState(ownerRepo, current.branch).catch(() => undefined);
+    if (prState?.mergeableState === "dirty") {
+      const reconcile = args.reconcileRunState ?? ((base: string) => reconcilePromoteRunState(args.projectDir, base));
+      const r = await reconcile(gitBase); // throws ScmMergeError on a non-ledger conflict (surfaces)
+      if (r.reconciled) preflightNotes.push(`Reconciled a pipeline.json-ledger promotion conflict to the run side${r.detail ? ` (${r.detail})` : ""}.`);
+    }
+  }
+
   let paired: Awaited<ReturnType<typeof mergePairedPullRequest>>;
   try {
     paired = await mergePairedPullRequest({

package/scripts/lakebase/scm-wait-ci.ts

@@ -24,6 +24,7 @@ export class ScmWaitCiError extends Error {
       | "bad-precondition"
       | "no-github-remote"
       | "ci-failed"
+      | "pr-conflicting"
       | "timeout"
       | "pr-not-found",
   ) {
@@ -123,6 +124,18 @@ export async function waitForCi(args: WaitCiArgs): Promise<WaitCiResult> {
       if (lastPr.ciStatus === "success") {
         return { done: true, value: lastPr };
       }
+      // A CONFLICTING PR (merge conflict with base) has no mergeable ref, so GitHub never
+      // dispatches the `pull_request` workflow — CI stays at 0 runs forever. Detect it and
+      // surface at once (resolve + re-push re-triggers), rather than spinning to the timeout
+      // on a `pending` that can never advance. Guard on `mergeableState==="dirty"` AND no
+      // checks yet, so a transient null/unknown while GitHub computes mergeability doesn't
+      // false-trip, and a PR that already has runs is judged by ciStatus as before.
+      if (lastPr.mergeableState === "dirty" && lastPr.checks.length === 0) {
+        throw new ScmWaitCiError(
+          `PR ${lastPr.url} is CONFLICTING with ${lastPr.baseBranch} — GitHub will not dispatch CI (0 runs) until the conflict is resolved. Reconcile the branch and re-push; the synchronize event will trigger CI.`,
+          "pr-conflicting",
+        );
+      }
       if (lastPr.ciStatus === "failure") {
         const failed = lastPr.checks
           .filter((c) => /(FAILURE|TIMED_OUT|CANCELLED|ACTION_REQUIRED)/i.test(c.conclusion))

package/scripts/lakebase/sync-ci-secrets.cli.ts

@@ -0,0 +1,122 @@
+#!/usr/bin/env node
+// CLI: (re)provision the repo's CI auth secrets — the standalone repair path.
+//
+// create-project runs syncCiSecrets ONCE, fail-soft (a failed PAT mint or `gh secret set` is
+// swallowed as a warning), and there was no way to retry without recreating the whole project.
+// So a repo could sit with ZERO CI secrets until a promotion CI run failed an entire feature
+// later (empty DATABRICKS_* → no DATABASE_URL → the test step aborts). This bin mints a fresh
+// CI PAT and sets DATABRICKS_HOST / LAKEBASE_PROJECT_ID / DATABRICKS_TOKEN, then VERIFIES all
+// three landed — exiting non-zero (naming the gaps) instead of leaving CI silently broken.
+
+import * as path from "node:path";
+import { isCliEntry } from "../util/cli-entry.js";
+import { readEnvVar } from "./env-file.js";
+import { getOwnerRepo } from "../git/remote.js";
+import { syncCiSecrets, missingCiSecrets, REQUIRED_CI_SECRETS } from "../util/ci-secrets.js";
+
+interface ParsedArgs {
+  projectDir?: string;
+  host?: string;
+  projectId?: string;
+  repo?: string;
+  lifetimeSec?: number;
+  comment?: string;
+  help?: boolean;
+}
+
+function parseArgs(argv: string[]): ParsedArgs {
+  const out: ParsedArgs = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    switch (a) {
+      case "--project-dir":
+      case "--cwd": out.projectDir = argv[++i]; break;
+      case "--host": out.host = argv[++i]; break;
+      case "--project-id": out.projectId = argv[++i]; break;
+      case "--repo": out.repo = argv[++i]; break;
+      case "--lifetime-sec": out.lifetimeSec = Number.parseInt(argv[++i], 10); break;
+      case "--comment": out.comment = argv[++i]; break;
+      case "--help": case "-h": out.help = true; break;
+    }
+  }
+  return out;
+}
+
+const HELP = `lakebase-sync-ci-secrets
+
+(Re)provision the GitHub repo's CI auth: mint a fresh Databricks CI PAT and set
+DATABRICKS_HOST, LAKEBASE_PROJECT_ID, and DATABRICKS_TOKEN as repo Actions secrets,
+then verify all three are present. The standalone repair for the create-time
+"CI auth setup failed" warning — run it from the project, no recreate needed.
+
+Usage:
+  lakebase-sync-ci-secrets [flags]
+
+Flags:
+  --project-dir <dir>   Project root (default: cwd); resolves host/project-id from its .env
+  --host <url>          DATABRICKS_HOST (default: .env DATABRICKS_HOST)
+  --project-id <id>     LAKEBASE_PROJECT_ID (default: .env LAKEBASE_PROJECT_ID)
+  --repo <owner/name>   Target repo (default: origin remote)
+  --lifetime-sec <n>    CI PAT lifetime (default: 86400 = 24h)
+  --comment <text>      Token comment (default: "GitHub Actions CI")
+  -h, --help            Show this help
+
+Exit codes:
+  0 = all three secrets present after sync
+  2 = could not resolve host / project-id / repo
+  3 = sync ran but a required secret is still missing (e.g. PAT mint failed)
+`;
+
+export async function runSyncCiSecretsCli(argv: string[]): Promise<number> {
+  const args = parseArgs(argv);
+  if (args.help) {
+    process.stdout.write(`${HELP}\n`);
+    return 0;
+  }
+  const projectDir = path.resolve(args.projectDir ?? process.cwd());
+  const envPath = path.join(projectDir, ".env");
+  const host = args.host ?? readEnvVar(envPath, "DATABRICKS_HOST");
+  const projectId = args.projectId ?? readEnvVar(envPath, "LAKEBASE_PROJECT_ID");
+  const ownerRepo = args.repo ?? (await getOwnerRepo(projectDir));
+
+  if (!ownerRepo) {
+    process.stderr.write("lakebase-sync-ci-secrets: no GitHub repo (pass --repo or run inside a repo with an origin remote).\n");
+    return 2;
+  }
+  if (!host || !projectId) {
+    process.stderr.write(
+      `lakebase-sync-ci-secrets: missing ${!host ? "DATABRICKS_HOST" : ""}${!host && !projectId ? " and " : ""}${!projectId ? "LAKEBASE_PROJECT_ID" : ""}` +
+        ` — not in ${envPath}; pass --host / --project-id.\n`,
+    );
+    return 2;
+  }
+
+  process.stdout.write(`Provisioning CI auth for ${ownerRepo} (host ${host}, project ${projectId})…\n`);
+  await syncCiSecrets({
+    projectDir,
+    databricksHost: host,
+    lakebaseProjectId: projectId,
+    ownerRepo,
+    ...(args.comment ? { comment: args.comment } : {}),
+    ...(args.lifetimeSec ? { lifetimeSeconds: args.lifetimeSec } : {}),
+  });
+
+  // VERIFY — the whole point of the repair. A silently-dropped PAT (the create-time failure
+  // mode) leaves DATABRICKS_TOKEN absent; catch it here and exit non-zero rather than report success.
+  const missing = await missingCiSecrets(ownerRepo);
+  if (missing.length > 0) {
+    process.stderr.write(
+      `lakebase-sync-ci-secrets: sync ran but these secrets are still MISSING on ${ownerRepo}: ${missing.join(", ")}.\n` +
+        (missing.includes("DATABRICKS_TOKEN")
+          ? `  DATABRICKS_TOKEN missing usually means 'databricks tokens create' failed — check your auth for ${host} (e.g. 'databricks auth login --host ${host}'), then re-run.\n`
+          : ""),
+    );
+    return 3;
+  }
+  process.stdout.write(`CI auth ready: ${REQUIRED_CI_SECRETS.join(", ")} set on ${ownerRepo}.\n`);
+  return 0;
+}
+
+if (isCliEntry(import.meta.url)) {
+  void runSyncCiSecretsCli(process.argv.slice(2)).then((c) => process.exit(c));
+}

package/scripts/util/ci-secrets.ts

@@ -7,9 +7,22 @@
 // be set), the values have to come from the create-project caller's scope.
 
 import { runDatabricks } from "../lakebase/databricks-cli.js";
-import { setRepoSecrets } from "../github/secrets.js";
+import { setRepoSecrets, listSecretNames } from "../github/secrets.js";
 import { getOwnerRepo } from "../git/remote.js";
 
+/** The repo Actions secrets CI needs to provision a per-PR Lakebase branch. Missing ANY of
+ *  these leaves CI unable to produce a DATABASE_URL, so the test step aborts — the failure
+ *  that otherwise only surfaces at promotion, an entire feature after creation. */
+export const REQUIRED_CI_SECRETS = ["DATABRICKS_HOST", "LAKEBASE_PROJECT_ID", "DATABRICKS_TOKEN"] as const;
+
+/** Which required CI secrets are NOT present on the repo (empty = fully provisioned). Used to
+ *  VERIFY a sync landed — a `databricks tokens create` or `gh secret set` that fails must be
+ *  caught here, not swallowed, so CI-auth is never silently absent. */
+export async function missingCiSecrets(ownerRepo: string): Promise<string[]> {
+  const present = new Set(await listSecretNames(ownerRepo));
+  return REQUIRED_CI_SECRETS.filter((n) => !present.has(n));
+}
+
 export interface SyncCiSecretsArgs {
   /** Project root (used to resolve ownerRepo from `git remote` when not given,
    *  and as the cwd for the `databricks tokens create` call). */

package/templates/project/common/.gitignore.base

@@ -30,6 +30,10 @@ application-local.properties
 .consort/cycles/
 .consort/experiments/
 .consort/pipeline.json
+# NOTE: the PER-FEATURE ledger .consort/features/<id>/pipeline.json is deliberately NOT ignored —
+# it is the durable per-story pipeline state (persisted on the feature branch so it survives across
+# sessions). Ignoring it lost the ledger. A promotion-merge conflict on it is resolved at the MERGE
+# layer (scm-merge reconciles it to the run side), never by untracking it.
 # The turn recorder's per-run observability: its internal file-hash map
 # (.recorder-state.json, used for delta computation), the routing-decision
 # diagnostic stream (routing-decisions.jsonl), and the per-turn record dirs

package/templates/project/common/scripts/lk

@@ -297,6 +297,14 @@ if [ "$first" = "--warm" ] || [ "$first" = "--rewarm" ]; then
 fi
 
 bin="$first"; shift
+# A help request (`-h`/`--help`) must NEVER trigger a one-time cold-cache install:
+# printing usage should have no side effects. Treat it like LK_NO_INSTALL so a cold
+# cache SHORT-CIRCUITS with a clear "warm first" note instead of silently kicking off a
+# multi-minute download (the "my --help refreshed the cache?!" surprise). When the cache
+# is already warm, help still prints instantly, exactly as before.
+help_req=0
+for _a in "$@"; do case "$_a" in -h|--help) help_req=1; break ;; esac; done
+[ "$help_req" -eq 1 ] && export LK_NO_INSTALL=1
 # Deprecation: lakebase-sftdd-* / lakebase-tdd-* are legacy aliases from the
 # pre-Consort skill names. They still run (routed to the kit below), but are
 # scheduled for removal in consort v0.4.0 , prefer the consort-* / lakebase-*
@@ -306,8 +314,15 @@ case "$bin" in
     echo "lk: [deprecated] '${bin}' is a legacy sftdd/tdd-era alias; use the consort-* / lakebase-* name (removed in consort v0.4.0). Still running." >&2 ;;
 esac
 slug="$(pkg_for_bin "$bin")"
-resolve_pkg "$slug" run; rc=$?
-[ "$rc" -eq 97 ] && exit 97
+# Capture the status with `|| rc=$?` so `set -e` does NOT abort on resolve_pkg's
+# non-zero returns (notably 97 = cold cache under LK_NO_INSTALL / a help request) —
+# a bare `resolve_pkg …; rc=$?` exits before rc is ever read.
+rc=0; resolve_pkg "$slug" run || rc=$?
+if [ "$rc" -eq 97 ]; then
+  # Cold cache + a help request: do NOT install just to print usage.
+  [ "$help_req" -eq 1 ] && { echo "lk: not downloading the toolkit just to show '--help'. The ${slug} cache for this ref is cold; run 'lk --warm' to install it (one-time), then retry '${bin} --help'. A help request never triggers a download." >&2; exit 0; }
+  exit 97
+fi
 [ "$rc" -ne 0 ] && exit "$rc"
 rel="$(bin_rel "$pkg_dir" "$bin")"
 
@@ -317,8 +332,11 @@ rel="$(bin_rel "$pkg_dir" "$bin")"
 # route to the kit up front and never reach here, so an unknown kit bin still fails
 # fast with no substrate round-trip.
 if [ -z "$rel" ] && [ "$slug" = "scm-utils" ] && project_uses_kit; then
-  resolve_pkg kit run; krc=$?
-  [ "$krc" -eq 97 ] && exit 97
+  krc=0; resolve_pkg kit run || krc=$?
+  if [ "$krc" -eq 97 ]; then
+    [ "$help_req" -eq 1 ] && { echo "lk: not downloading the toolkit just to show '--help'. The kit cache for this ref is cold; run 'lk --warm' to install it (one-time), then retry '${bin} --help'. A help request never triggers a download." >&2; exit 0; }
+    exit 97
+  fi
   if [ "$krc" -eq 0 ]; then
     rel="$(bin_rel "$pkg_dir" "$bin")"
     [ -n "$rel" ] && slug="kit"