@databricks-solutions/lakebase-scm-utils 0.3.9 → 0.3.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{pr-e-E4aP4j.d.cts → pr-D-eayWyJ.d.cts} +59 -1
- package/dist/{pr-e-E4aP4j.d.ts → pr-D-eayWyJ.d.ts} +59 -1
- package/dist/scripts/git/index.cjs +30 -14
- package/dist/scripts/git/index.d.cts +32 -1
- package/dist/scripts/git/index.d.ts +32 -1
- package/dist/scripts/git/index.js +28 -14
- package/dist/scripts/github/index.d.cts +2 -60
- package/dist/scripts/github/index.d.ts +2 -60
- package/dist/scripts/index.cjs +301 -98
- package/dist/scripts/index.d.cts +4 -4
- package/dist/scripts/index.d.ts +4 -4
- package/dist/scripts/index.js +296 -98
- package/dist/scripts/lakebase/branch.cli.cjs +55 -10
- package/dist/scripts/lakebase/branch.cli.js +55 -10
- package/dist/scripts/lakebase/ci-resolve-branch.cli.cjs +59 -24
- package/dist/scripts/lakebase/ci-resolve-branch.cli.js +59 -24
- package/dist/scripts/lakebase/collapse-heads.cli.cjs +16 -14
- package/dist/scripts/lakebase/collapse-heads.cli.js +16 -14
- package/dist/scripts/lakebase/cut-backup.cli.cjs +43 -10
- package/dist/scripts/lakebase/cut-backup.cli.js +43 -10
- package/dist/scripts/lakebase/cut-tier.cli.cjs +43 -10
- package/dist/scripts/lakebase/cut-tier.cli.js +43 -10
- package/dist/scripts/lakebase/detect-language.cli.cjs +16 -14
- package/dist/scripts/lakebase/detect-language.cli.js +16 -14
- package/dist/scripts/lakebase/doctor.cli.cjs +16 -14
- package/dist/scripts/lakebase/doctor.cli.js +16 -14
- package/dist/scripts/lakebase/get-connection.cli.cjs +16 -14
- package/dist/scripts/lakebase/get-connection.cli.js +16 -14
- package/dist/scripts/lakebase/index.cjs +313 -99
- package/dist/scripts/lakebase/index.d.cts +97 -2
- package/dist/scripts/lakebase/index.d.ts +97 -2
- package/dist/scripts/lakebase/index.js +310 -99
- package/dist/scripts/lakebase/infra-runner.cli.cjs +16 -14
- package/dist/scripts/lakebase/infra-runner.cli.js +16 -14
- package/dist/scripts/lakebase/new-migration.cli.cjs +16 -14
- package/dist/scripts/lakebase/new-migration.cli.js +16 -14
- package/dist/scripts/lakebase/schema-diff.cli.cjs +16 -14
- package/dist/scripts/lakebase/schema-diff.cli.js +16 -14
- package/dist/scripts/lakebase/schema-migrate.cli.cjs +16 -14
- package/dist/scripts/lakebase/schema-migrate.cli.js +16 -14
- package/dist/scripts/lakebase/scm-claim-feature.cli.cjs +59 -24
- package/dist/scripts/lakebase/scm-claim-feature.cli.js +59 -24
- package/dist/scripts/lakebase/scm-doctor.cli.cjs +59 -24
- package/dist/scripts/lakebase/scm-doctor.cli.js +59 -24
- package/dist/scripts/lakebase/scm-merge.cli.cjs +16 -14
- package/dist/scripts/lakebase/scm-merge.cli.js +16 -14
- package/dist/scripts/lakebase/scm-reconcile-tier.cli.cjs +16 -14
- package/dist/scripts/lakebase/scm-reconcile-tier.cli.js +16 -14
- package/dist/scripts/lakebase/scm-recover-orphans.cli.cjs +43 -10
- package/dist/scripts/lakebase/scm-recover-orphans.cli.js +43 -10
- package/dist/scripts/lakebase/snapshot.cli.cjs +418 -0
- package/dist/scripts/lakebase/snapshot.cli.d.cts +1 -0
- package/dist/scripts/lakebase/snapshot.cli.d.ts +1 -0
- package/dist/scripts/lakebase/snapshot.cli.js +395 -0
- package/dist/scripts/util/index.cjs +2 -2
- package/dist/scripts/util/index.js +2 -2
- package/package.json +2 -1
- package/scripts/git/commit-push.ts +60 -14
- package/scripts/lakebase/branch-create.ts +93 -14
- package/scripts/lakebase/branch.cli.ts +15 -0
- package/scripts/lakebase/create-preflight.ts +213 -1
- package/scripts/lakebase/create-project.ts +98 -30
- package/scripts/lakebase/get-connection.ts +29 -12
- package/scripts/lakebase/snapshot.cli.ts +117 -0
- package/scripts/lakebase/snapshot.ts +85 -0
- package/templates/project/common/scripts/run-tests.sh +43 -2
- package/templates/project/python/tests/conftest.py +22 -0
- package/templates/project/common/tests/e2e/__pycache__/conftest.cpython-314.pyc +0 -0
- package/templates/project/python/tests/__pycache__/conftest.cpython-314.pyc +0 -0
|
@@ -39,6 +39,9 @@ interface ParsedArgs {
|
|
|
39
39
|
instance?: string;
|
|
40
40
|
branch?: string;
|
|
41
41
|
parentBranch?: string;
|
|
42
|
+
fromSnapshot?: string;
|
|
43
|
+
atTime?: string;
|
|
44
|
+
atLsn?: string;
|
|
42
45
|
ttl?: string;
|
|
43
46
|
noExpiry?: boolean;
|
|
44
47
|
cwd?: string;
|
|
@@ -85,6 +88,15 @@ function parseArgs(argv: string[]): ParsedArgs {
|
|
|
85
88
|
case "--parent-branch":
|
|
86
89
|
out.parentBranch = argv[++i];
|
|
87
90
|
break;
|
|
91
|
+
case "--from-snapshot":
|
|
92
|
+
out.fromSnapshot = argv[++i];
|
|
93
|
+
break;
|
|
94
|
+
case "--at-time":
|
|
95
|
+
out.atTime = argv[++i];
|
|
96
|
+
break;
|
|
97
|
+
case "--at-lsn":
|
|
98
|
+
out.atLsn = argv[++i];
|
|
99
|
+
break;
|
|
88
100
|
case "--ttl":
|
|
89
101
|
out.ttl = argv[++i];
|
|
90
102
|
break;
|
|
@@ -259,6 +271,9 @@ async function main(): Promise<number> {
|
|
|
259
271
|
host: args.host,
|
|
260
272
|
branch: args.branch!,
|
|
261
273
|
parentBranch: args.parentBranch,
|
|
274
|
+
fromSnapshot: args.fromSnapshot,
|
|
275
|
+
sourceBranchTime: args.atTime,
|
|
276
|
+
sourceBranchLsn: args.atLsn,
|
|
262
277
|
ttl: args.ttl,
|
|
263
278
|
noExpiry: args.noExpiry,
|
|
264
279
|
});
|
|
@@ -19,7 +19,7 @@ import * as fs from "node:fs";
|
|
|
19
19
|
import * as path from "node:path";
|
|
20
20
|
import { deleteLakebaseProject } from "./lakebase-project.js";
|
|
21
21
|
import { runDatabricks } from "./databricks-cli.js";
|
|
22
|
-
import { checkGitHubScopes } from "../github/repo.js";
|
|
22
|
+
import { checkGitHubScopes, getCurrentUser, repoExists, type GitHubScopeStatus } from "../github/repo.js";
|
|
23
23
|
|
|
24
24
|
export interface PreflightResult {
|
|
25
25
|
ok: boolean;
|
|
@@ -145,6 +145,134 @@ export async function checkGitHubWorkflowScope(
|
|
|
145
145
|
}
|
|
146
146
|
}
|
|
147
147
|
|
|
148
|
+
export interface GitHubPreflightInput {
|
|
149
|
+
/** Target repo owner (user or org). */
|
|
150
|
+
owner: string;
|
|
151
|
+
/** Bare repo name (the project name). */
|
|
152
|
+
repoName: string;
|
|
153
|
+
/** Scopes whose absence BLOCKS the happy path (create + push). Default ["repo","workflow"]. */
|
|
154
|
+
hardScopes?: readonly string[];
|
|
155
|
+
/** Scopes whose absence only WARNS (e.g. delete_repo, needed for rollback repo-delete). Default ["delete_repo"]. */
|
|
156
|
+
advisoryScopes?: readonly string[];
|
|
157
|
+
/** Injected probes (default to the real GitHub reads). */
|
|
158
|
+
currentUser?: () => Promise<string>;
|
|
159
|
+
scopes?: (required: readonly string[]) => Promise<GitHubScopeStatus>;
|
|
160
|
+
repoExists?: (fullName: string) => Promise<boolean>;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export interface GitHubPreflightResult extends PreflightResult {
|
|
164
|
+
/** Non-blocking notes (org/owner mismatch, missing advisory scope, invisible scopes). */
|
|
165
|
+
warnings: string[];
|
|
166
|
+
/** The resolved authenticated login, when B1 succeeded. */
|
|
167
|
+
activeUser?: string;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* One up-front GitHub preflight run BEFORE any repo / Lakebase project is created,
|
|
172
|
+
* so a missing requisite fails in seconds with a fix instead of mid-create (where
|
|
173
|
+
* it orphans partially-created artifacts). Supersedes the workflow-only check:
|
|
174
|
+
*
|
|
175
|
+
* B1 active account getCurrentUser() must resolve (a valid token) — else a
|
|
176
|
+
* wrong/absent token otherwise 403s LATE at the push.
|
|
177
|
+
* B2 scope set checkGitHubScopes([...hard, ...advisory]): a missing HARD
|
|
178
|
+
* scope (repo/workflow) fails; a missing ADVISORY scope
|
|
179
|
+
* (delete_repo) warns (rollback repo-delete unavailable);
|
|
180
|
+
* granted null (fine-grained PAT / App) can't be asserted.
|
|
181
|
+
* B3 name availability repoExists(owner/name) → fail before anything exists.
|
|
182
|
+
* B4 owner note login != owner ⇒ creating under an org/other account —
|
|
183
|
+
* surfaced (createRepo enforces the real org-create right).
|
|
184
|
+
*
|
|
185
|
+
* CONSERVATIVE: hard-fails ONLY on the clear blockers (invalid token, missing
|
|
186
|
+
* happy-path scope, name taken); every other uncertainty is a warning, never a
|
|
187
|
+
* false block (same ethos as checkGitHubWorkflowScope). All probes injected, so
|
|
188
|
+
* this is unit-testable with no network.
|
|
189
|
+
*/
|
|
190
|
+
export async function checkGitHubPreflight(input: GitHubPreflightInput): Promise<GitHubPreflightResult> {
|
|
191
|
+
const warnings: string[] = [];
|
|
192
|
+
const hard = input.hardScopes ?? ["repo", "workflow"];
|
|
193
|
+
const advisory = input.advisoryScopes ?? ["delete_repo"];
|
|
194
|
+
const currentUserProbe = input.currentUser ?? getCurrentUser;
|
|
195
|
+
const scopesProbe = input.scopes ?? ((req: readonly string[]) => checkGitHubScopes(req));
|
|
196
|
+
const repoExistsProbe = input.repoExists ?? repoExists;
|
|
197
|
+
|
|
198
|
+
// B1: active account — a valid token. Hard blocker; nothing downstream works without it.
|
|
199
|
+
let activeUser: string;
|
|
200
|
+
try {
|
|
201
|
+
activeUser = await currentUserProbe();
|
|
202
|
+
} catch (err) {
|
|
203
|
+
return {
|
|
204
|
+
ok: false,
|
|
205
|
+
warnings,
|
|
206
|
+
reason:
|
|
207
|
+
`GitHub authentication failed: ${err instanceof Error ? err.message : String(err)}. ` +
|
|
208
|
+
"Sign in with `gh auth login`, or set GITHUB_TOKEN to a valid token, then re-run.",
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// B4 (note): a login different from the owner means an org (or another account).
|
|
213
|
+
// Not a hard block — createRepo surfaces the real org-create permission — but
|
|
214
|
+
// surface it so a wrong active account is visible BEFORE the push's late 403.
|
|
215
|
+
if (activeUser.toLowerCase() !== input.owner.toLowerCase()) {
|
|
216
|
+
warnings.push(
|
|
217
|
+
`Signed in to GitHub as "${activeUser}" but creating under "${input.owner}" (an org or another account); ` +
|
|
218
|
+
`ensure "${activeUser}" can create repositories there.`,
|
|
219
|
+
);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// B2: the full required scope set — not just `workflow`.
|
|
223
|
+
try {
|
|
224
|
+
const { granted, missing } = await scopesProbe([...hard, ...advisory]);
|
|
225
|
+
if (granted !== null) {
|
|
226
|
+
const missingHard = missing.filter((s) => hard.includes(s));
|
|
227
|
+
const missingAdvisory = missing.filter((s) => advisory.includes(s));
|
|
228
|
+
if (missingHard.length > 0) {
|
|
229
|
+
return {
|
|
230
|
+
ok: false,
|
|
231
|
+
warnings,
|
|
232
|
+
activeUser,
|
|
233
|
+
reason:
|
|
234
|
+
`GitHub token missing required scope(s): ${missingHard.join(", ")} ` +
|
|
235
|
+
`(grants [${granted.join(", ") || "none"}]). Fix: gh auth refresh -h github.com -s ${[...hard].join(",")}`,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
if (missingAdvisory.length > 0) {
|
|
239
|
+
warnings.push(
|
|
240
|
+
`GitHub token missing the ${missingAdvisory.join(", ")} scope — automatic rollback of the repo on a failed ` +
|
|
241
|
+
"create is unavailable (you would delete it by hand). Grant it with: gh auth refresh -h github.com -s delete_repo",
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
} else {
|
|
245
|
+
warnings.push(
|
|
246
|
+
"GitHub token scopes are not visible (fine-grained PAT or App token); cannot pre-verify repo/workflow access , " +
|
|
247
|
+
"the create will surface any gap when it pushes.",
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
} catch {
|
|
251
|
+
warnings.push("Could not read GitHub token scopes (probe error); skipping the scope pre-check.");
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// B3: name availability — fail before anything is created.
|
|
255
|
+
try {
|
|
256
|
+
if (await repoExistsProbe(`${input.owner}/${input.repoName}`)) {
|
|
257
|
+
return {
|
|
258
|
+
ok: false,
|
|
259
|
+
warnings,
|
|
260
|
+
activeUser,
|
|
261
|
+
reason:
|
|
262
|
+
`GitHub repo "${input.owner}/${input.repoName}" already exists. ` +
|
|
263
|
+
"Pick a different project name, or delete the existing repo first.",
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
} catch {
|
|
267
|
+
warnings.push(
|
|
268
|
+
`Could not check whether "${input.owner}/${input.repoName}" already exists (probe error); ` +
|
|
269
|
+
"the create will fail clearly if it does.",
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
return { ok: true, warnings, activeUser };
|
|
274
|
+
}
|
|
275
|
+
|
|
148
276
|
/** The actionable prereq message for a GitHub token missing the `workflow` scope. */
|
|
149
277
|
export function githubWorkflowScopePrereqMessage(reason?: string): string {
|
|
150
278
|
return (
|
|
@@ -258,3 +386,87 @@ export async function withLakebaseRollback<T>(opts: RollbackOptions, fn: () => P
|
|
|
258
386
|
throw wrapped;
|
|
259
387
|
}
|
|
260
388
|
}
|
|
389
|
+
|
|
390
|
+
export interface RollbackStep {
|
|
391
|
+
label: string;
|
|
392
|
+
undo: () => Promise<void> | void;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
export interface RollbackLedger {
|
|
396
|
+
/** Record a teardown for a resource the moment it is created. */
|
|
397
|
+
register(label: string, undo: () => Promise<void> | void): void;
|
|
398
|
+
/** The steps, in registration order (teardown runs them REVERSED). */
|
|
399
|
+
steps(): readonly RollbackStep[];
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** A created-resources ledger for the full create flow (A3). */
|
|
403
|
+
export function createRollbackLedger(): RollbackLedger {
|
|
404
|
+
const steps: RollbackStep[] = [];
|
|
405
|
+
return {
|
|
406
|
+
register: (label, undo) => {
|
|
407
|
+
steps.push({ label, undo });
|
|
408
|
+
},
|
|
409
|
+
steps: () => steps,
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
export interface CreateRollbackOptions {
|
|
414
|
+
report?: (step: string, detail?: string) => void;
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* A3: run `fn`; on a throw, tear down EVERY resource the ledger recorded, in
|
|
419
|
+
* REVERSE order (last-created first), then rethrow with a summary of what was
|
|
420
|
+
* and wasn't cleaned up.
|
|
421
|
+
*
|
|
422
|
+
* Unlike {@link withLakebaseRollback} (Lakebase project only), this undoes the
|
|
423
|
+
* GitHub repo, the self-hosted runner, the local dir, AND the Lakebase project
|
|
424
|
+
* together — so a genuine provisioning fault leaves a clean "name is free" state
|
|
425
|
+
* instead of orphaning three of the four artifacts (the exact 4-artifact manual
|
|
426
|
+
* cleanup the cold-download incident caused). A step's own failure is captured +
|
|
427
|
+
* reported, never thrown, so one stuck teardown can't block the others; a
|
|
428
|
+
* "not found" teardown counts as already-gone. The caller registers a step the
|
|
429
|
+
* instant each resource is created, so a failure anywhere inside `fn` rolls back
|
|
430
|
+
* exactly what existed at that point.
|
|
431
|
+
*/
|
|
432
|
+
export async function withCreateRollback<T>(
|
|
433
|
+
ledger: RollbackLedger,
|
|
434
|
+
opts: CreateRollbackOptions,
|
|
435
|
+
fn: () => Promise<T>,
|
|
436
|
+
): Promise<T> {
|
|
437
|
+
try {
|
|
438
|
+
return await fn();
|
|
439
|
+
} catch (err) {
|
|
440
|
+
const report = opts.report ?? (() => {});
|
|
441
|
+
const steps = [...ledger.steps()].reverse();
|
|
442
|
+
const rolledBack: string[] = [];
|
|
443
|
+
const failed: string[] = [];
|
|
444
|
+
if (steps.length > 0) report(`Create failed; rolling back ${steps.length} created resource(s)...`);
|
|
445
|
+
for (const s of steps) {
|
|
446
|
+
try {
|
|
447
|
+
report(` rolling back: ${s.label}`);
|
|
448
|
+
await s.undo();
|
|
449
|
+
rolledBack.push(s.label);
|
|
450
|
+
} catch (e) {
|
|
451
|
+
const m = e instanceof Error ? e.message : String(e);
|
|
452
|
+
if (/not.?found/i.test(m)) {
|
|
453
|
+
rolledBack.push(s.label);
|
|
454
|
+
continue;
|
|
455
|
+
}
|
|
456
|
+
failed.push(`${s.label} (${m})`);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
const base = err instanceof Error ? err.message : String(err);
|
|
460
|
+
const parts: string[] = [];
|
|
461
|
+
if (rolledBack.length) parts.push(`rolled back: ${rolledBack.join(", ")}`);
|
|
462
|
+
if (failed.length) {
|
|
463
|
+
parts.push(
|
|
464
|
+
`COULD NOT roll back: ${failed.join("; ")} , clean these up by hand before retrying the same name`,
|
|
465
|
+
);
|
|
466
|
+
}
|
|
467
|
+
const suffix = parts.length ? ` (${parts.join("; ")})` : "";
|
|
468
|
+
const wrapped = err instanceof Error ? err : new Error(base);
|
|
469
|
+
wrapped.message = `${base}${suffix}`;
|
|
470
|
+
throw wrapped;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
@@ -9,20 +9,23 @@ import * as fs from "node:fs";
|
|
|
9
9
|
import * as path from "node:path";
|
|
10
10
|
import { writeEnvFile } from "./env-file.js";
|
|
11
11
|
import { verifyProject, verifyHooks, verifyWorkflows } from "./project-verify.js";
|
|
12
|
-
import { createRepo, getRepoFullName, getCurrentUser } from "../github/repo.js";
|
|
12
|
+
import { createRepo, getRepoFullName, getCurrentUser, deleteRepo } from "../github/repo.js";
|
|
13
13
|
import { cloneRepo } from "../git/clone.js";
|
|
14
14
|
import { gitInit } from "../git/init.js";
|
|
15
|
-
import {
|
|
15
|
+
import { commitInitial, pushInitial, WorkflowScopeError } from "../git/commit-push.js";
|
|
16
16
|
import {
|
|
17
17
|
createLakebaseProject,
|
|
18
18
|
getDefaultBranchId,
|
|
19
|
+
deleteLakebaseProject,
|
|
19
20
|
} from "./lakebase-project.js";
|
|
20
21
|
import {
|
|
21
22
|
checkDatabricksAuth,
|
|
22
23
|
databricksAuthPrereqMessage,
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
checkGitHubPreflight,
|
|
25
|
+
warmAndVerifyKit,
|
|
26
|
+
kitWarmWarning,
|
|
27
|
+
createRollbackLedger,
|
|
28
|
+
withCreateRollback,
|
|
26
29
|
validateCreateInputs,
|
|
27
30
|
dirIsEmpty,
|
|
28
31
|
} from "./create-preflight.js";
|
|
@@ -32,7 +35,7 @@ import { createLongRunningBranch } from "./long-running-branch.js";
|
|
|
32
35
|
import { enableE2eForProject } from "./enable-e2e.js";
|
|
33
36
|
import { ensureClientLockfile } from "./client-lockfile.js";
|
|
34
37
|
import { enableInfraForProject } from "./enable-infra.js";
|
|
35
|
-
import { setupRunner } from "./runner-setup.js";
|
|
38
|
+
import { setupRunner, removeRunner } from "./runner-setup.js";
|
|
36
39
|
import { syncCiSecrets, missingCiSecrets } from "../util/ci-secrets.js";
|
|
37
40
|
import { delay } from "../util/delay.js";
|
|
38
41
|
import {
|
|
@@ -291,19 +294,33 @@ export async function createProject(
|
|
|
291
294
|
throw new Error(databricksAuthPrereqMessage(host, auth.reason));
|
|
292
295
|
}
|
|
293
296
|
|
|
294
|
-
// ── GitHub
|
|
295
|
-
//
|
|
296
|
-
//
|
|
297
|
-
//
|
|
298
|
-
//
|
|
297
|
+
// ── GitHub account + requisites precondition (B) ──────────────
|
|
298
|
+
// ONE up-front GitHub preflight, BEFORE any repo / Lakebase project exists, so a
|
|
299
|
+
// missing requisite fails in seconds with a fix instead of mid-create (where it
|
|
300
|
+
// orphans partial artifacts): the active account resolves (else a wrong/absent
|
|
301
|
+
// token 403s LATE at the push), the full required scope set is present (repo +
|
|
302
|
+
// workflow — not just workflow), and the repo name is free. Conservative: notes
|
|
303
|
+
// (owner/org mismatch, invisible/advisory scopes) are surfaced but never block.
|
|
299
304
|
if (useGithub) {
|
|
300
|
-
report("Checking GitHub token
|
|
301
|
-
const
|
|
302
|
-
|
|
303
|
-
|
|
305
|
+
report("Checking GitHub account + token requisites...");
|
|
306
|
+
const gh = await checkGitHubPreflight({ owner: input.githubOwner!, repoName: input.projectName });
|
|
307
|
+
for (const w of gh.warnings) {
|
|
308
|
+
warnings.push(w);
|
|
309
|
+
report(`Note: ${w}`);
|
|
304
310
|
}
|
|
311
|
+
if (!gh.ok) throw new Error(gh.reason);
|
|
305
312
|
}
|
|
306
313
|
|
|
314
|
+
// The created-resources ledger (A3): each resource registers its teardown the
|
|
315
|
+
// moment it is created, and a genuine provisioning throw inside the wrapper below
|
|
316
|
+
// rolls back ALL of them in reverse — the GitHub repo, the runner, the local dir,
|
|
317
|
+
// AND the Lakebase project together — leaving a clean "name is free" state (not
|
|
318
|
+
// the old 4-artifact orphan where only the Lakebase project was undone). A
|
|
319
|
+
// push-only failure is caught locally below and is NOT a provisioning throw, so
|
|
320
|
+
// it never triggers a rollback.
|
|
321
|
+
const rollback = createRollbackLedger();
|
|
322
|
+
return await withCreateRollback(rollback, { report }, async (): Promise<CreateProjectResult> => {
|
|
323
|
+
|
|
307
324
|
// ── Step 1+2: GitHub repo + clone, OR local-only setup ────────
|
|
308
325
|
if (useGithub) {
|
|
309
326
|
report("Creating GitHub repository...", fullRepoName);
|
|
@@ -311,6 +328,7 @@ export async function createProject(
|
|
|
311
328
|
private: input.privateRepo !== false,
|
|
312
329
|
description: `Lakebase project: ${input.projectName}`,
|
|
313
330
|
});
|
|
331
|
+
rollback.register(`GitHub repo ${fullRepoName}`, () => deleteRepo(fullRepoName));
|
|
314
332
|
|
|
315
333
|
report("Waiting for GitHub repo to be visible...", fullRepoName);
|
|
316
334
|
const probeDelays = [1000, 2000, 3000, 5000, 8000];
|
|
@@ -349,25 +367,23 @@ export async function createProject(
|
|
|
349
367
|
repoUrl: `https://github.com/${fullRepoName}.git`,
|
|
350
368
|
parentDir: input.parentDir,
|
|
351
369
|
});
|
|
370
|
+
rollback.register(`local project dir ${projectDir}`, () => fs.rmSync(projectDir, { recursive: true, force: true }));
|
|
352
371
|
} else {
|
|
353
372
|
report("Creating local project directory...", projectDir);
|
|
354
373
|
// A pre-existing EMPTY dir is accepted (validated up front); mkdir with
|
|
355
374
|
// recursive is a no-op on it. A non-empty dir was already rejected.
|
|
356
375
|
fs.mkdirSync(projectDir, { recursive: true });
|
|
357
376
|
await gitInit(projectDir);
|
|
377
|
+
rollback.register(`local project dir ${projectDir}`, () => fs.rmSync(projectDir, { recursive: true, force: true }));
|
|
358
378
|
}
|
|
359
379
|
|
|
360
380
|
// ── Step 3: Lakebase project ──────────────────────────────────
|
|
361
381
|
report("Creating Lakebase database (provisioning Postgres)...", lakebaseProjectId);
|
|
362
382
|
await createLakebaseProject({ projectId: lakebaseProjectId, host });
|
|
363
|
-
|
|
364
|
-
//
|
|
365
|
-
//
|
|
366
|
-
|
|
367
|
-
// with rollback context.
|
|
368
|
-
return await withLakebaseRollback(
|
|
369
|
-
{ projectId: lakebaseProjectId, host, report },
|
|
370
|
-
async (): Promise<CreateProjectResult> => {
|
|
383
|
+
// From here on the Lakebase project EXISTS; register its teardown so a later
|
|
384
|
+
// provisioning throw rolls it back (W9) alongside the repo + dir above — a retry
|
|
385
|
+
// with the same name would otherwise collide with the reserved/soft-deleted slug.
|
|
386
|
+
rollback.register(`Lakebase project ${lakebaseProjectId}`, () => deleteLakebaseProject({ projectId: lakebaseProjectId, host }));
|
|
371
387
|
|
|
372
388
|
// ── Step 4: Default branch lookup (non-fatal if not ready yet) ─
|
|
373
389
|
report("Resolving database endpoint...");
|
|
@@ -481,6 +497,11 @@ export async function createProject(
|
|
|
481
497
|
projectName: input.projectName,
|
|
482
498
|
report: (m) => report(m),
|
|
483
499
|
});
|
|
500
|
+
// Registered only after a SUCCESSFUL setup, so a provisioning throw later
|
|
501
|
+
// tears the runner down (stop + deregister + rm dir) alongside the rest.
|
|
502
|
+
rollback.register(`self-hosted runner ${input.projectName}`, () =>
|
|
503
|
+
removeRunner({ fullRepoName, projectName: input.projectName }),
|
|
504
|
+
);
|
|
484
505
|
} catch (err) {
|
|
485
506
|
const msg = err instanceof Error ? err.message : String(err);
|
|
486
507
|
warnings.push(`Runner setup failed: ${msg}`);
|
|
@@ -594,6 +615,25 @@ export async function createProject(
|
|
|
594
615
|
);
|
|
595
616
|
}
|
|
596
617
|
|
|
618
|
+
// ── Step 7g: pre-warm the toolkit so the first push doesn't cold-download mid-hook (A1) ──
|
|
619
|
+
// The scaffolded project's pre-push hook runs `./scripts/lk`, which triggers the
|
|
620
|
+
// one-time cold download of the Consort toolkit (hundreds of npm fetches). If that
|
|
621
|
+
// very first `lk` is the PUSH's hook, the heavy download races the push and the push
|
|
622
|
+
// fails (the cold-download incident). Warming HERE — after the scaffold laid down
|
|
623
|
+
// scripts/lk, before the push — makes the hook find a populated cache. Best-effort
|
|
624
|
+
// with ONE retry: if warming still fails we do NOT block create (the push then
|
|
625
|
+
// cold-downloads itself, exactly as before, and the retriable push below absorbs it).
|
|
626
|
+
if (enableSftdd && useGithub) {
|
|
627
|
+
report("Warming the Consort toolkit (so the first push doesn't cold-download mid-hook)...");
|
|
628
|
+
let warm = warmAndVerifyKit(projectDir);
|
|
629
|
+
if (!warm.ok) warm = warmAndVerifyKit(projectDir);
|
|
630
|
+
if (!warm.ok) {
|
|
631
|
+
const m = kitWarmWarning(projectDir, warm.reason);
|
|
632
|
+
warnings.push(m);
|
|
633
|
+
report(`Note: ${m}`);
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
|
|
597
637
|
// ── Step 8: Initial commit (+ push when GitHub configured) ────
|
|
598
638
|
const langLabels: Record<string, string> = {
|
|
599
639
|
java: "Java/Spring Boot",
|
|
@@ -622,11 +662,29 @@ export async function createProject(
|
|
|
622
662
|
}
|
|
623
663
|
|
|
624
664
|
report("Creating initial commit...");
|
|
625
|
-
await
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
665
|
+
await commitInitial(projectDir, `Initial project scaffold (${langLabel} + Lakebase)`);
|
|
666
|
+
// The commit is now LOCAL on disk. A push failure after this point is RETRIABLE,
|
|
667
|
+
// NOT a provisioning fault (A2): repo + Lakebase + runner + scaffold all succeeded,
|
|
668
|
+
// so tearing them down for a transient first-push cold download is wrong. Push with
|
|
669
|
+
// ONE retry (the toolkit is cached after the warm above / the first hook run); if it
|
|
670
|
+
// STILL fails, leave every resource intact and print the one-line resume — do NOT
|
|
671
|
+
// throw (which would fire the rollback ledger). A real `workflow`-scope rejection is
|
|
672
|
+
// not transient and IS surfaced as a hard error.
|
|
673
|
+
let pushLanded = !useGithub; // local-only: nothing to push → treat as landed
|
|
674
|
+
if (useGithub) {
|
|
675
|
+
try {
|
|
676
|
+
await pushInitial({ projectDir, attempts: 2 });
|
|
677
|
+
pushLanded = true;
|
|
678
|
+
} catch (err) {
|
|
679
|
+
if (err instanceof WorkflowScopeError) throw err;
|
|
680
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
681
|
+
const m =
|
|
682
|
+
`Provisioning is complete, but the initial push did not land after a retry: ${msg}. ` +
|
|
683
|
+
`The project on disk is fine , finish it with: cd ${projectDir} && git push -u origin main`;
|
|
684
|
+
warnings.push(m);
|
|
685
|
+
report(`WARNING: ${m}`);
|
|
686
|
+
}
|
|
687
|
+
}
|
|
630
688
|
|
|
631
689
|
// ── Step 8b: Long-running tier setup (architectural choice) ───
|
|
632
690
|
// Tier semantics (features are NOT tiers, they are short-lived branches):
|
|
@@ -644,7 +702,17 @@ export async function createProject(
|
|
|
644
702
|
// origin to already have the parent ref (e.g. main, staging).
|
|
645
703
|
// tiers 2/3 with --no-github is rejected up front by validateCreateInputs, so
|
|
646
704
|
// by here useGithub is guaranteed true for any tiers>1 request.
|
|
647
|
-
|
|
705
|
+
// Skip cleanly when the initial push did NOT land (A2): createLongRunningBranch
|
|
706
|
+
// forks off origin/main, which does not exist yet — cutting a tier now would fail.
|
|
707
|
+
// Surface the deferral so the user cuts the tiers after finishing the push.
|
|
708
|
+
if ((tiers === 2 || tiers === 3) && !pushLanded) {
|
|
709
|
+
const m =
|
|
710
|
+
`INCOMPLETE TIERS: you requested ${tiers} tiers, but the initial push hasn't landed, so the ` +
|
|
711
|
+
`tier branches can't fork off origin yet. After you finish the push (above), cut them with ` +
|
|
712
|
+
`\`./scripts/lk lakebase-cut-tier --name staging --fork-from main\`${tiers === 3 ? " and \`--name dev --fork-from staging\`" : ""}.`;
|
|
713
|
+
warnings.push(m);
|
|
714
|
+
report(`WARNING: ${m}`);
|
|
715
|
+
} else if (tiers === 2 || tiers === 3) {
|
|
648
716
|
report(`Cutting staging tier (tiers=${tiers}) via createLongRunningBranch...`);
|
|
649
717
|
try {
|
|
650
718
|
await createLongRunningBranch({
|
|
@@ -712,7 +780,7 @@ export async function createProject(
|
|
|
712
780
|
warnings,
|
|
713
781
|
};
|
|
714
782
|
|
|
715
|
-
} // end
|
|
783
|
+
} // end withCreateRollback closure
|
|
716
784
|
);
|
|
717
785
|
}
|
|
718
786
|
|
|
@@ -136,7 +136,6 @@ export async function getConnection(args: ConnectionArgs): Promise<DsnResult | P
|
|
|
136
136
|
|
|
137
137
|
// output === "pool"
|
|
138
138
|
const host = await resolveEndpointHost(args.instance, branchId);
|
|
139
|
-
const email = await resolveCurrentUser();
|
|
140
139
|
// Label the POOLED path too , it is the PRIMARY one (schema-diff, reconcile-tier). Passing
|
|
141
140
|
// application_name in createLakebasePool's config does NOT work: it builds its own pg
|
|
142
141
|
// PoolConfig and drops the field (verified live , the pooled connection came back with an
|
|
@@ -145,17 +144,35 @@ export async function getConnection(args: ConnectionArgs): Promise<DsnResult | P
|
|
|
145
144
|
// call time (not import) so CONSORT_VERSION , which Consort exports at its entry point , is
|
|
146
145
|
// already in the env when connectionApplicationName() reads it.
|
|
147
146
|
process.env.PGAPPNAME = connectionApplicationName();
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
:
|
|
158
|
-
|
|
147
|
+
|
|
148
|
+
// AppKit OBO: an explicit workspaceClient is a pre-authenticated caller , keep the
|
|
149
|
+
// OAuth/token-rotation path with it verbatim.
|
|
150
|
+
if (args.workspaceClient !== undefined) {
|
|
151
|
+
const email = await resolveCurrentUser();
|
|
152
|
+
return createLakebasePool({
|
|
153
|
+
endpoint: endpointPath,
|
|
154
|
+
host,
|
|
155
|
+
database,
|
|
156
|
+
user: email,
|
|
157
|
+
workspaceClient: args.workspaceClient as never,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// CLI/kit path (no workspaceClient): DO NOT let createLakebasePool self-authenticate
|
|
162
|
+
// via the SDK default chain. That runs `databricks auth token --host <workspace-host>`
|
|
163
|
+
// WITHOUT a profile, so when several ~/.databrickscfg profiles match that host (e.g.
|
|
164
|
+
// DEFAULT + an adb-* + logfood all pointing at the same workspace) the CLI refuses
|
|
165
|
+
// ("Use --profile to specify which profile to use") and the pool connection fails
|
|
166
|
+
// ("default auth: databricks-cli: cannot get access token"). That silently broke
|
|
167
|
+
// schema-diff at deploy-verify (the diff came back empty, failing the schema-diff
|
|
168
|
+
// test) whenever the host was ambiguous. Instead MINT the credential through the
|
|
169
|
+
// kit's profile-threaded CLI wrapper (mintCredential -> runDatabricksSync -> the
|
|
170
|
+
// resolved `--profile`, the same unambiguous path the DSN output uses) and pass it as
|
|
171
|
+
// `password`: LakebasePoolConfig skips OAuth entirely when a password is supplied. The
|
|
172
|
+
// kit's pool consumers (schema-diff, reconcile-tier) are short batch reads, well inside
|
|
173
|
+
// the credential TTL, so forgoing token rotation here is safe.
|
|
174
|
+
const { token, email } = await mintCredential(endpointPath);
|
|
175
|
+
return createLakebasePool({ host, database, user: email, password: token });
|
|
159
176
|
}
|
|
160
177
|
|
|
161
178
|
/**
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// CLI for Lakebase SNAPSHOTS: `lakebase-snapshot <create|delete|list> ...`.
|
|
3
|
+
//
|
|
4
|
+
// A snapshot is an immutable point-in-time copy of a branch. Capture the
|
|
5
|
+
// committed schema once, then branch copy-on-write from it per verify cycle
|
|
6
|
+
// (`lakebase-branch create --from-snapshot <id>`) instead of re-running
|
|
7
|
+
// `alembic upgrade head`. Prints JSON on stdout, errors on stderr.
|
|
8
|
+
// 0 = success 1 = runtime error 2 = usage error
|
|
9
|
+
import { createSnapshot, deleteSnapshot, listSnapshots } from "./snapshot.js";
|
|
10
|
+
|
|
11
|
+
interface Args {
|
|
12
|
+
subcommand?: string;
|
|
13
|
+
instance?: string;
|
|
14
|
+
snapshot?: string;
|
|
15
|
+
branch?: string;
|
|
16
|
+
atTime?: string;
|
|
17
|
+
atLsn?: string;
|
|
18
|
+
ttl?: string;
|
|
19
|
+
noExpiry?: boolean;
|
|
20
|
+
host?: string;
|
|
21
|
+
pretty?: boolean;
|
|
22
|
+
help?: boolean;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function parseArgs(argv: string[]): Args {
|
|
26
|
+
const out: Args = {};
|
|
27
|
+
let i = 0;
|
|
28
|
+
if (argv[0] && !argv[0].startsWith("-")) {
|
|
29
|
+
out.subcommand = argv[0];
|
|
30
|
+
i = 1;
|
|
31
|
+
}
|
|
32
|
+
for (; i < argv.length; i++) {
|
|
33
|
+
switch (argv[i]) {
|
|
34
|
+
case "--instance": out.instance = argv[++i]; break;
|
|
35
|
+
case "--snapshot": out.snapshot = argv[++i]; break;
|
|
36
|
+
case "--branch": out.branch = argv[++i]; break;
|
|
37
|
+
case "--at-time": out.atTime = argv[++i]; break;
|
|
38
|
+
case "--at-lsn": out.atLsn = argv[++i]; break;
|
|
39
|
+
case "--ttl": out.ttl = argv[++i]; break;
|
|
40
|
+
case "--no-expiry": out.noExpiry = true; break;
|
|
41
|
+
case "--host": out.host = argv[++i]; break;
|
|
42
|
+
case "--pretty": out.pretty = true; break;
|
|
43
|
+
case "--help": case "-h": out.help = true; break;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const USAGE = `usage: lakebase-snapshot <create|delete|list> --instance <project> [flags]
|
|
50
|
+
create --snapshot <id> --branch <src> [--at-time <iso> | --at-lsn <lsn>] [--ttl <s> | --no-expiry]
|
|
51
|
+
delete --snapshot <id>
|
|
52
|
+
list
|
|
53
|
+
Then branch from it: lakebase-branch create --branch <v> --from-snapshot <id> --instance <project>`;
|
|
54
|
+
|
|
55
|
+
function out(result: unknown, pretty: boolean): void {
|
|
56
|
+
process.stdout.write(JSON.stringify(result, null, pretty ? 2 : undefined) + "\n");
|
|
57
|
+
}
|
|
58
|
+
function need(a: Args, keys: (keyof Args)[]): boolean {
|
|
59
|
+
const missing = keys.filter((k) => a[k] === undefined || a[k] === "");
|
|
60
|
+
if (missing.length) {
|
|
61
|
+
process.stderr.write(`missing ${missing.map((k) => "--" + String(k)).join(", ")}\n${USAGE}\n`);
|
|
62
|
+
return false;
|
|
63
|
+
}
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function main(): Promise<number> {
|
|
68
|
+
const a = parseArgs(process.argv.slice(2));
|
|
69
|
+
if (a.help || !a.subcommand) {
|
|
70
|
+
process.stderr.write(USAGE + "\n");
|
|
71
|
+
return a.help ? 0 : 2;
|
|
72
|
+
}
|
|
73
|
+
switch (a.subcommand) {
|
|
74
|
+
case "create": {
|
|
75
|
+
if (!need(a, ["instance", "snapshot", "branch"])) return 2;
|
|
76
|
+
const raw = await createSnapshot({
|
|
77
|
+
instance: a.instance!,
|
|
78
|
+
snapshot: a.snapshot!,
|
|
79
|
+
sourceBranch: a.branch!,
|
|
80
|
+
sourceBranchTime: a.atTime,
|
|
81
|
+
sourceBranchLsn: a.atLsn,
|
|
82
|
+
ttl: a.ttl,
|
|
83
|
+
noExpiry: a.noExpiry,
|
|
84
|
+
host: a.host,
|
|
85
|
+
});
|
|
86
|
+
out({ created: a.snapshot, raw: raw.trim() || undefined }, !!a.pretty);
|
|
87
|
+
return 0;
|
|
88
|
+
}
|
|
89
|
+
case "delete": {
|
|
90
|
+
if (!need(a, ["instance", "snapshot"])) return 2;
|
|
91
|
+
await deleteSnapshot({ instance: a.instance!, snapshot: a.snapshot!, host: a.host });
|
|
92
|
+
out({ deleted: a.snapshot }, !!a.pretty);
|
|
93
|
+
return 0;
|
|
94
|
+
}
|
|
95
|
+
case "list": {
|
|
96
|
+
if (!need(a, ["instance"])) return 2;
|
|
97
|
+
const raw = await listSnapshots({ instance: a.instance!, host: a.host });
|
|
98
|
+
try {
|
|
99
|
+
out(JSON.parse(raw), !!a.pretty);
|
|
100
|
+
} catch {
|
|
101
|
+
process.stdout.write(raw);
|
|
102
|
+
}
|
|
103
|
+
return 0;
|
|
104
|
+
}
|
|
105
|
+
default:
|
|
106
|
+
process.stderr.write(`unknown subcommand '${a.subcommand}'\n${USAGE}\n`);
|
|
107
|
+
return 2;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
main().then(
|
|
112
|
+
(code) => process.exit(code),
|
|
113
|
+
(err) => {
|
|
114
|
+
process.stderr.write(String(err instanceof Error ? err.message : err) + "\n");
|
|
115
|
+
process.exit(1);
|
|
116
|
+
},
|
|
117
|
+
);
|