@databricks-solutions/lakebase-scm-utils 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/scripts/index.cjs +59 -24
- package/dist/scripts/index.js +59 -24
- package/dist/scripts/lakebase/branch.cli.cjs +55 -10
- package/dist/scripts/lakebase/branch.cli.js +55 -10
- package/dist/scripts/lakebase/ci-resolve-branch.cli.cjs +59 -24
- package/dist/scripts/lakebase/ci-resolve-branch.cli.js +59 -24
- package/dist/scripts/lakebase/collapse-heads.cli.cjs +16 -14
- package/dist/scripts/lakebase/collapse-heads.cli.js +16 -14
- package/dist/scripts/lakebase/cut-backup.cli.cjs +43 -10
- package/dist/scripts/lakebase/cut-backup.cli.js +43 -10
- package/dist/scripts/lakebase/cut-tier.cli.cjs +43 -10
- package/dist/scripts/lakebase/cut-tier.cli.js +43 -10
- package/dist/scripts/lakebase/detect-language.cli.cjs +16 -14
- package/dist/scripts/lakebase/detect-language.cli.js +16 -14
- package/dist/scripts/lakebase/doctor.cli.cjs +16 -14
- package/dist/scripts/lakebase/doctor.cli.js +16 -14
- package/dist/scripts/lakebase/get-connection.cli.cjs +16 -14
- package/dist/scripts/lakebase/get-connection.cli.js +16 -14
- package/dist/scripts/lakebase/index.cjs +59 -24
- package/dist/scripts/lakebase/index.d.cts +23 -0
- package/dist/scripts/lakebase/index.d.ts +23 -0
- package/dist/scripts/lakebase/index.js +59 -24
- package/dist/scripts/lakebase/infra-runner.cli.cjs +16 -14
- package/dist/scripts/lakebase/infra-runner.cli.js +16 -14
- package/dist/scripts/lakebase/new-migration.cli.cjs +16 -14
- package/dist/scripts/lakebase/new-migration.cli.js +16 -14
- package/dist/scripts/lakebase/schema-diff.cli.cjs +16 -14
- package/dist/scripts/lakebase/schema-diff.cli.js +16 -14
- package/dist/scripts/lakebase/schema-migrate.cli.cjs +16 -14
- package/dist/scripts/lakebase/schema-migrate.cli.js +16 -14
- package/dist/scripts/lakebase/scm-claim-feature.cli.cjs +59 -24
- package/dist/scripts/lakebase/scm-claim-feature.cli.js +59 -24
- package/dist/scripts/lakebase/scm-doctor.cli.cjs +59 -24
- package/dist/scripts/lakebase/scm-doctor.cli.js +59 -24
- package/dist/scripts/lakebase/scm-merge.cli.cjs +16 -14
- package/dist/scripts/lakebase/scm-merge.cli.js +16 -14
- package/dist/scripts/lakebase/scm-reconcile-tier.cli.cjs +16 -14
- package/dist/scripts/lakebase/scm-reconcile-tier.cli.js +16 -14
- package/dist/scripts/lakebase/scm-recover-orphans.cli.cjs +43 -10
- package/dist/scripts/lakebase/scm-recover-orphans.cli.js +43 -10
- package/dist/scripts/lakebase/snapshot.cli.cjs +418 -0
- package/dist/scripts/lakebase/snapshot.cli.d.cts +1 -0
- package/dist/scripts/lakebase/snapshot.cli.d.ts +1 -0
- package/dist/scripts/lakebase/snapshot.cli.js +395 -0
- package/dist/scripts/util/index.cjs +2 -2
- package/dist/scripts/util/index.js +2 -2
- package/package.json +2 -1
- package/scripts/lakebase/branch-create.ts +93 -14
- package/scripts/lakebase/branch.cli.ts +15 -0
- package/scripts/lakebase/get-connection.ts +29 -12
- package/scripts/lakebase/snapshot.cli.ts +117 -0
- package/scripts/lakebase/snapshot.ts +85 -0
- package/templates/project/common/scripts/run-tests.sh +43 -2
- package/templates/project/common/tests/e2e/conftest.py +5 -1
- package/templates/project/python/tests/conftest.py +60 -4
- package/templates/project/python/tests/__pycache__/conftest.cpython-314.pyc +0 -0
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// scripts/lakebase/databricks-cli.ts
|
|
4
|
+
import { execFile, execFileSync as execFileSync2 } from "child_process";
|
|
5
|
+
import { promisify } from "util";
|
|
6
|
+
import { join as join2 } from "path";
|
|
7
|
+
|
|
8
|
+
// scripts/lakebase/kit-config.ts
|
|
9
|
+
function intFromEnv(name, fallback) {
|
|
10
|
+
const raw = process.env[name];
|
|
11
|
+
if (!raw) return fallback;
|
|
12
|
+
const parsed = Number.parseInt(raw, 10);
|
|
13
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
|
|
14
|
+
return parsed;
|
|
15
|
+
}
|
|
16
|
+
var DAY_MS = 24 * 60 * 60 * 1e3;
|
|
17
|
+
var KIT_TIMEOUTS = {
|
|
18
|
+
cliDefault: intFromEnv("LAKEBASE_KIT_TIMEOUT_CLI_DEFAULT_MS", 3e4),
|
|
19
|
+
cliCreateProject: intFromEnv("LAKEBASE_KIT_TIMEOUT_CLI_CREATE_PROJECT_MS", 18e4),
|
|
20
|
+
cliCreateBranch: intFromEnv("LAKEBASE_KIT_TIMEOUT_CLI_CREATE_BRANCH_MS", 6e4),
|
|
21
|
+
cliCreateEndpoint: intFromEnv("LAKEBASE_KIT_TIMEOUT_CLI_CREATE_ENDPOINT_MS", 6e4),
|
|
22
|
+
readyWait: intFromEnv("LAKEBASE_KIT_TIMEOUT_READY_WAIT_MS", 12e4),
|
|
23
|
+
readyPoll: intFromEnv("LAKEBASE_KIT_TIMEOUT_READY_POLL_MS", 5e3),
|
|
24
|
+
pgConnect: intFromEnv("LAKEBASE_KIT_TIMEOUT_PG_CONNECT_MS", 1e4),
|
|
25
|
+
pgStatement: intFromEnv("LAKEBASE_KIT_TIMEOUT_PG_STATEMENT_MS", 15e3),
|
|
26
|
+
gitDefault: intFromEnv("LAKEBASE_KIT_TIMEOUT_GIT_DEFAULT_MS", 5e3),
|
|
27
|
+
gitCheckout: intFromEnv("LAKEBASE_KIT_TIMEOUT_GIT_CHECKOUT_MS", 1e4),
|
|
28
|
+
gitNetwork: intFromEnv("LAKEBASE_KIT_TIMEOUT_GIT_NETWORK_MS", 15e3),
|
|
29
|
+
gitPush: intFromEnv("LAKEBASE_KIT_TIMEOUT_GIT_PUSH_MS", 3e4),
|
|
30
|
+
cliLong: intFromEnv("LAKEBASE_KIT_TIMEOUT_CLI_LONG_MS", 6e4),
|
|
31
|
+
cmdShort: intFromEnv("LAKEBASE_KIT_TIMEOUT_CMD_SHORT_MS", 5e3),
|
|
32
|
+
initializrCacheTtl: intFromEnv("LAKEBASE_KIT_INITIALIZR_CACHE_TTL_MS", 10 * 60 * 1e3),
|
|
33
|
+
featureBranchTtlMs: intFromEnv("LAKEBASE_KIT_FEATURE_BRANCH_TTL_MS", 30 * DAY_MS),
|
|
34
|
+
testBranchTtlMs: intFromEnv("LAKEBASE_KIT_TEST_BRANCH_TTL_MS", 14 * DAY_MS),
|
|
35
|
+
uatBranchTtlMs: intFromEnv("LAKEBASE_KIT_UAT_BRANCH_TTL_MS", 14 * DAY_MS),
|
|
36
|
+
perfBranchTtlMs: intFromEnv("LAKEBASE_KIT_PERF_BRANCH_TTL_MS", 7 * DAY_MS)
|
|
37
|
+
};
|
|
38
|
+
function urlFromEnv(name, fallback) {
|
|
39
|
+
const raw = process.env[name];
|
|
40
|
+
if (!raw) return fallback;
|
|
41
|
+
return raw.replace(/\/+$/, "");
|
|
42
|
+
}
|
|
43
|
+
var KIT_REGISTRIES = {
|
|
44
|
+
mavenCentral: urlFromEnv("LAKEBASE_KIT_REGISTRY_MAVEN_CENTRAL", "https://repo1.maven.org/maven2"),
|
|
45
|
+
springInitializr: urlFromEnv("LAKEBASE_KIT_REGISTRY_SPRING_INITIALIZR", "https://start.spring.io")
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
// scripts/lakebase/databricks-profile.ts
|
|
49
|
+
import * as fs from "fs";
|
|
50
|
+
import { execFileSync } from "child_process";
|
|
51
|
+
|
|
52
|
+
// scripts/util/exec.ts
|
|
53
|
+
import * as cp from "child_process";
|
|
54
|
+
|
|
55
|
+
// scripts/lakebase/databricks-profile.ts
|
|
56
|
+
function normalizeHost(host) {
|
|
57
|
+
return host.trim().replace(/\/+$/, "").toLowerCase();
|
|
58
|
+
}
|
|
59
|
+
function selectProfileForHost(profilesJson, host) {
|
|
60
|
+
const target = normalizeHost(host);
|
|
61
|
+
if (!target) return void 0;
|
|
62
|
+
const start = profilesJson.indexOf("{");
|
|
63
|
+
if (start < 0) return void 0;
|
|
64
|
+
let parsed;
|
|
65
|
+
try {
|
|
66
|
+
parsed = JSON.parse(profilesJson.slice(start));
|
|
67
|
+
} catch {
|
|
68
|
+
return void 0;
|
|
69
|
+
}
|
|
70
|
+
const profiles = parsed.profiles;
|
|
71
|
+
if (!Array.isArray(profiles)) return void 0;
|
|
72
|
+
const names = profiles.filter((p) => {
|
|
73
|
+
if (!p || typeof p !== "object") return false;
|
|
74
|
+
const rec = p;
|
|
75
|
+
return typeof rec.name === "string" && typeof rec.host === "string" && rec.valid === true && normalizeHost(rec.host) === target;
|
|
76
|
+
}).map((p) => p.name);
|
|
77
|
+
const distinct = Array.from(new Set(names));
|
|
78
|
+
return distinct.length === 1 ? distinct[0] : void 0;
|
|
79
|
+
}
|
|
80
|
+
function resolveProfileForHostSync(host, timeoutMs = KIT_TIMEOUTS.cliDefault) {
|
|
81
|
+
if (!normalizeHost(host)) return void 0;
|
|
82
|
+
let out2;
|
|
83
|
+
try {
|
|
84
|
+
out2 = execFileSync("databricks", ["auth", "profiles", "-o", "json"], {
|
|
85
|
+
encoding: "utf8",
|
|
86
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
87
|
+
timeout: timeoutMs
|
|
88
|
+
});
|
|
89
|
+
} catch {
|
|
90
|
+
return void 0;
|
|
91
|
+
}
|
|
92
|
+
return selectProfileForHost(out2, host);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// scripts/lakebase/env-file.ts
|
|
96
|
+
import * as fs2 from "fs";
|
|
97
|
+
import * as path from "path";
|
|
98
|
+
function readEnvVar(envPath, key) {
|
|
99
|
+
if (!fs2.existsSync(envPath)) return void 0;
|
|
100
|
+
let value;
|
|
101
|
+
for (const line of fs2.readFileSync(envPath, "utf-8").split("\n")) {
|
|
102
|
+
const trimmed = line.trimStart();
|
|
103
|
+
if (trimmed.startsWith("#") || !trimmed.startsWith(`${key}=`)) continue;
|
|
104
|
+
value = trimmed.slice(key.length + 1).trim().replace(/^["']|["']$/g, "");
|
|
105
|
+
}
|
|
106
|
+
return value && value.length > 0 ? value : void 0;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// scripts/lakebase/databricks-cli.ts
|
|
110
|
+
var execFileP = promisify(execFile);
|
|
111
|
+
var DatabricksCliError = class extends Error {
|
|
112
|
+
constructor(message, profile, stderr) {
|
|
113
|
+
super(message);
|
|
114
|
+
this.profile = profile;
|
|
115
|
+
this.stderr = stderr;
|
|
116
|
+
this.name = "DatabricksCliError";
|
|
117
|
+
}
|
|
118
|
+
profile;
|
|
119
|
+
stderr;
|
|
120
|
+
};
|
|
121
|
+
var DatabricksAuthError = class extends DatabricksCliError {
|
|
122
|
+
constructor(profile, detail) {
|
|
123
|
+
const login = `databricks auth login${profile ? ` --profile ${profile}` : ""}`;
|
|
124
|
+
super(
|
|
125
|
+
`Databricks authentication failed${profile ? ` for profile "${profile}"` : ""}: the cached token is missing or expired. Re-authenticate, then re-run:
|
|
126
|
+
${login}
|
|
127
|
+
${detail}`,
|
|
128
|
+
profile,
|
|
129
|
+
detail
|
|
130
|
+
);
|
|
131
|
+
this.name = "DatabricksAuthError";
|
|
132
|
+
}
|
|
133
|
+
};
|
|
134
|
+
var profileByHost = /* @__PURE__ */ new Map();
|
|
135
|
+
var profileByEnvFile = /* @__PURE__ */ new Map();
|
|
136
|
+
var hostByEnvFile = /* @__PURE__ */ new Map();
|
|
137
|
+
function envFileHost(cwd) {
|
|
138
|
+
if (hostByEnvFile.has(cwd)) return hostByEnvFile.get(cwd);
|
|
139
|
+
const v = readEnvVar(join2(cwd, ".env"), "DATABRICKS_HOST");
|
|
140
|
+
hostByEnvFile.set(cwd, v);
|
|
141
|
+
return v;
|
|
142
|
+
}
|
|
143
|
+
function effectiveHost(opts) {
|
|
144
|
+
const base = opts.env ?? process.env;
|
|
145
|
+
const cwd = opts.cwd ?? process.cwd();
|
|
146
|
+
const h = opts.host ?? base.DATABRICKS_HOST ?? envFileHost(cwd);
|
|
147
|
+
return h?.trim() || void 0;
|
|
148
|
+
}
|
|
149
|
+
function isAuthFailure(text) {
|
|
150
|
+
return /refresh token is invalid|auth login|could not be retrieved because|not authenticated|no valid.*(credential|token)|invalid.*(access token|credential)|\b401\b|unauthorized/i.test(
|
|
151
|
+
text
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
function resolveProfile(opts) {
|
|
155
|
+
const base = opts.env ?? process.env;
|
|
156
|
+
if (opts.profile) return opts.profile;
|
|
157
|
+
const envProfile = base.DATABRICKS_CONFIG_PROFILE?.trim();
|
|
158
|
+
if (envProfile) return envProfile;
|
|
159
|
+
const cwd = opts.cwd ?? process.cwd();
|
|
160
|
+
let fromEnvFile;
|
|
161
|
+
if (profileByEnvFile.has(cwd)) {
|
|
162
|
+
fromEnvFile = profileByEnvFile.get(cwd);
|
|
163
|
+
} else {
|
|
164
|
+
fromEnvFile = readEnvVar(join2(cwd, ".env"), "DATABRICKS_CONFIG_PROFILE");
|
|
165
|
+
profileByEnvFile.set(cwd, fromEnvFile);
|
|
166
|
+
}
|
|
167
|
+
if (fromEnvFile) return fromEnvFile;
|
|
168
|
+
const host = effectiveHost(opts);
|
|
169
|
+
if (!host) return void 0;
|
|
170
|
+
if (profileByHost.has(host)) return profileByHost.get(host);
|
|
171
|
+
const resolved = resolveProfileForHostSync(host, opts.timeout);
|
|
172
|
+
profileByHost.set(host, resolved);
|
|
173
|
+
return resolved;
|
|
174
|
+
}
|
|
175
|
+
function buildInvocation(args, opts) {
|
|
176
|
+
const base = opts.env ?? process.env;
|
|
177
|
+
const trimmedHost = effectiveHost(opts)?.replace(/\/+$/, "");
|
|
178
|
+
const env = { ...base };
|
|
179
|
+
if (trimmedHost) env.DATABRICKS_HOST = trimmedHost;
|
|
180
|
+
delete env.DATABRICKS_WORKSPACE_ID;
|
|
181
|
+
const profile = resolveProfile(opts);
|
|
182
|
+
const argv = profile && !opts.noProfile && !args.includes("--profile") ? [...args, "--profile", profile] : args;
|
|
183
|
+
return { argv, env, profile };
|
|
184
|
+
}
|
|
185
|
+
function classifyDatabricksError(err, argv, profile) {
|
|
186
|
+
const e = err;
|
|
187
|
+
const asText = (v) => typeof v === "string" ? v : Buffer.isBuffer(v) ? v.toString("utf8") : "";
|
|
188
|
+
const stderr = asText(e.stderr).trim();
|
|
189
|
+
const stdout = asText(e.stdout).trim();
|
|
190
|
+
const haystack = `${e.message ?? ""}
|
|
191
|
+
${stderr}
|
|
192
|
+
${stdout}`;
|
|
193
|
+
if (isAuthFailure(haystack)) {
|
|
194
|
+
return new DatabricksAuthError(profile, stderr || stdout || (e.message ?? ""));
|
|
195
|
+
}
|
|
196
|
+
const killed = e.killed === true;
|
|
197
|
+
const signal = e.signal ?? void 0;
|
|
198
|
+
const detail = stderr ? `
|
|
199
|
+
stderr: ${stderr}` : stdout ? `
|
|
200
|
+
stdout: ${stdout}` : killed || signal ? `
|
|
201
|
+
(no output; the CLI was killed${signal ? ` by ${signal}` : ""}, likely a TIMEOUT; raise the budget via the matching LAKEBASE_KIT_TIMEOUT_* env var)` : e.code !== void 0 ? `
|
|
202
|
+
(no stderr/stdout; exit ${e.code})` : "";
|
|
203
|
+
return new DatabricksCliError(
|
|
204
|
+
`databricks ${argv.join(" ")} failed: ${e.message}${detail}`,
|
|
205
|
+
profile,
|
|
206
|
+
stderr || stdout
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
async function runDatabricks(args, opts = {}) {
|
|
210
|
+
const { argv, env, profile } = buildInvocation(args, opts);
|
|
211
|
+
const timeout = opts.timeout ?? KIT_TIMEOUTS.cliDefault;
|
|
212
|
+
try {
|
|
213
|
+
if (opts.input !== void 0) {
|
|
214
|
+
return await execDatabricksWithStdin(argv, opts.input, env, timeout);
|
|
215
|
+
}
|
|
216
|
+
const { stdout } = await execFileP("databricks", argv, { env, timeout });
|
|
217
|
+
return stdout.toString();
|
|
218
|
+
} catch (err) {
|
|
219
|
+
throw classifyDatabricksError(err, argv, profile);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
function execDatabricksWithStdin(argv, input, env, timeout) {
|
|
223
|
+
return new Promise((resolve, reject) => {
|
|
224
|
+
const child = execFile("databricks", argv, { env, timeout }, (err, stdout, stderr) => {
|
|
225
|
+
if (err) {
|
|
226
|
+
err.stdout = String(stdout ?? "");
|
|
227
|
+
err.stderr = String(stderr ?? "");
|
|
228
|
+
reject(err);
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
resolve(String(stdout ?? ""));
|
|
232
|
+
});
|
|
233
|
+
child.stdin?.on("error", () => {
|
|
234
|
+
});
|
|
235
|
+
child.stdin?.end(input);
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// scripts/lakebase/branch-utils.ts
|
|
240
|
+
function projectPath(instance) {
|
|
241
|
+
return `projects/${instance}`;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// scripts/lakebase/snapshot.ts
|
|
245
|
+
function branchResource(instance, branch) {
|
|
246
|
+
return branch.includes("/") ? branch : `${projectPath(instance)}/branches/${branch}`;
|
|
247
|
+
}
|
|
248
|
+
function snapshotResource(instance, snapshot) {
|
|
249
|
+
return snapshot.includes("/") ? snapshot : `${projectPath(instance)}/snapshots/${snapshot}`;
|
|
250
|
+
}
|
|
251
|
+
function buildSnapshotSpec(args) {
|
|
252
|
+
if (args.sourceBranchTime && args.sourceBranchLsn) {
|
|
253
|
+
throw new Error("sourceBranchTime and sourceBranchLsn are mutually exclusive \u2014 pass at most one point-in-time.");
|
|
254
|
+
}
|
|
255
|
+
if (args.ttl && args.noExpiry === true) {
|
|
256
|
+
throw new Error(`Cannot set both ttl ("${args.ttl}") and noExpiry: true \u2014 they are mutually exclusive.`);
|
|
257
|
+
}
|
|
258
|
+
const spec = { source_branch: branchResource(args.instance, args.sourceBranch) };
|
|
259
|
+
if (args.sourceBranchTime) spec.source_branch_time = args.sourceBranchTime;
|
|
260
|
+
else if (args.sourceBranchLsn) spec.source_branch_lsn = args.sourceBranchLsn;
|
|
261
|
+
if (args.ttl) spec.ttl = args.ttl;
|
|
262
|
+
else if (args.noExpiry ?? true) spec.no_expiry = true;
|
|
263
|
+
return spec;
|
|
264
|
+
}
|
|
265
|
+
async function createSnapshot(args) {
|
|
266
|
+
const spec = buildSnapshotSpec(args);
|
|
267
|
+
return runDatabricks(
|
|
268
|
+
["postgres", "create-snapshot", projectPath(args.instance), args.snapshot, "--json", JSON.stringify({ spec })],
|
|
269
|
+
{ host: args.host, timeout: KIT_TIMEOUTS.cliCreateBranch }
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
async function deleteSnapshot(args) {
|
|
273
|
+
return runDatabricks(["postgres", "delete-snapshot", snapshotResource(args.instance, args.snapshot)], {
|
|
274
|
+
host: args.host
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
async function listSnapshots(args) {
|
|
278
|
+
return runDatabricks(["postgres", "list-snapshots", projectPath(args.instance), "-o", "json"], { host: args.host });
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// scripts/lakebase/snapshot.cli.ts
|
|
282
|
+
function parseArgs(argv) {
|
|
283
|
+
const out2 = {};
|
|
284
|
+
let i = 0;
|
|
285
|
+
if (argv[0] && !argv[0].startsWith("-")) {
|
|
286
|
+
out2.subcommand = argv[0];
|
|
287
|
+
i = 1;
|
|
288
|
+
}
|
|
289
|
+
for (; i < argv.length; i++) {
|
|
290
|
+
switch (argv[i]) {
|
|
291
|
+
case "--instance":
|
|
292
|
+
out2.instance = argv[++i];
|
|
293
|
+
break;
|
|
294
|
+
case "--snapshot":
|
|
295
|
+
out2.snapshot = argv[++i];
|
|
296
|
+
break;
|
|
297
|
+
case "--branch":
|
|
298
|
+
out2.branch = argv[++i];
|
|
299
|
+
break;
|
|
300
|
+
case "--at-time":
|
|
301
|
+
out2.atTime = argv[++i];
|
|
302
|
+
break;
|
|
303
|
+
case "--at-lsn":
|
|
304
|
+
out2.atLsn = argv[++i];
|
|
305
|
+
break;
|
|
306
|
+
case "--ttl":
|
|
307
|
+
out2.ttl = argv[++i];
|
|
308
|
+
break;
|
|
309
|
+
case "--no-expiry":
|
|
310
|
+
out2.noExpiry = true;
|
|
311
|
+
break;
|
|
312
|
+
case "--host":
|
|
313
|
+
out2.host = argv[++i];
|
|
314
|
+
break;
|
|
315
|
+
case "--pretty":
|
|
316
|
+
out2.pretty = true;
|
|
317
|
+
break;
|
|
318
|
+
case "--help":
|
|
319
|
+
case "-h":
|
|
320
|
+
out2.help = true;
|
|
321
|
+
break;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
return out2;
|
|
325
|
+
}
|
|
326
|
+
var USAGE = `usage: lakebase-snapshot <create|delete|list> --instance <project> [flags]
|
|
327
|
+
create --snapshot <id> --branch <src> [--at-time <iso> | --at-lsn <lsn>] [--ttl <s> | --no-expiry]
|
|
328
|
+
delete --snapshot <id>
|
|
329
|
+
list
|
|
330
|
+
Then branch from it: lakebase-branch create --branch <v> --from-snapshot <id> --instance <project>`;
|
|
331
|
+
function out(result, pretty) {
|
|
332
|
+
process.stdout.write(JSON.stringify(result, null, pretty ? 2 : void 0) + "\n");
|
|
333
|
+
}
|
|
334
|
+
function need(a, keys) {
|
|
335
|
+
const missing = keys.filter((k) => a[k] === void 0 || a[k] === "");
|
|
336
|
+
if (missing.length) {
|
|
337
|
+
process.stderr.write(`missing ${missing.map((k) => "--" + String(k)).join(", ")}
|
|
338
|
+
${USAGE}
|
|
339
|
+
`);
|
|
340
|
+
return false;
|
|
341
|
+
}
|
|
342
|
+
return true;
|
|
343
|
+
}
|
|
344
|
+
async function main() {
|
|
345
|
+
const a = parseArgs(process.argv.slice(2));
|
|
346
|
+
if (a.help || !a.subcommand) {
|
|
347
|
+
process.stderr.write(USAGE + "\n");
|
|
348
|
+
return a.help ? 0 : 2;
|
|
349
|
+
}
|
|
350
|
+
switch (a.subcommand) {
|
|
351
|
+
case "create": {
|
|
352
|
+
if (!need(a, ["instance", "snapshot", "branch"])) return 2;
|
|
353
|
+
const raw = await createSnapshot({
|
|
354
|
+
instance: a.instance,
|
|
355
|
+
snapshot: a.snapshot,
|
|
356
|
+
sourceBranch: a.branch,
|
|
357
|
+
sourceBranchTime: a.atTime,
|
|
358
|
+
sourceBranchLsn: a.atLsn,
|
|
359
|
+
ttl: a.ttl,
|
|
360
|
+
noExpiry: a.noExpiry,
|
|
361
|
+
host: a.host
|
|
362
|
+
});
|
|
363
|
+
out({ created: a.snapshot, raw: raw.trim() || void 0 }, !!a.pretty);
|
|
364
|
+
return 0;
|
|
365
|
+
}
|
|
366
|
+
case "delete": {
|
|
367
|
+
if (!need(a, ["instance", "snapshot"])) return 2;
|
|
368
|
+
await deleteSnapshot({ instance: a.instance, snapshot: a.snapshot, host: a.host });
|
|
369
|
+
out({ deleted: a.snapshot }, !!a.pretty);
|
|
370
|
+
return 0;
|
|
371
|
+
}
|
|
372
|
+
case "list": {
|
|
373
|
+
if (!need(a, ["instance"])) return 2;
|
|
374
|
+
const raw = await listSnapshots({ instance: a.instance, host: a.host });
|
|
375
|
+
try {
|
|
376
|
+
out(JSON.parse(raw), !!a.pretty);
|
|
377
|
+
} catch {
|
|
378
|
+
process.stdout.write(raw);
|
|
379
|
+
}
|
|
380
|
+
return 0;
|
|
381
|
+
}
|
|
382
|
+
default:
|
|
383
|
+
process.stderr.write(`unknown subcommand '${a.subcommand}'
|
|
384
|
+
${USAGE}
|
|
385
|
+
`);
|
|
386
|
+
return 2;
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
main().then(
|
|
390
|
+
(code) => process.exit(code),
|
|
391
|
+
(err) => {
|
|
392
|
+
process.stderr.write(String(err instanceof Error ? err.message : err) + "\n");
|
|
393
|
+
process.exit(1);
|
|
394
|
+
}
|
|
395
|
+
);
|
|
@@ -1667,8 +1667,8 @@ var PKG_NAME = "@databricks-solutions/lakebase-scm-utils";
|
|
|
1667
1667
|
var cached;
|
|
1668
1668
|
function substrateSelfVersion() {
|
|
1669
1669
|
if (cached !== void 0) return cached;
|
|
1670
|
-
if ("0.3.
|
|
1671
|
-
cached = "0.3.
|
|
1670
|
+
if ("0.3.10".length > 0) {
|
|
1671
|
+
cached = "0.3.10";
|
|
1672
1672
|
return cached;
|
|
1673
1673
|
}
|
|
1674
1674
|
cached = "unknown";
|
|
@@ -1630,8 +1630,8 @@ var PKG_NAME = "@databricks-solutions/lakebase-scm-utils";
|
|
|
1630
1630
|
var cached;
|
|
1631
1631
|
function substrateSelfVersion() {
|
|
1632
1632
|
if (cached !== void 0) return cached;
|
|
1633
|
-
if ("0.3.
|
|
1634
|
-
cached = "0.3.
|
|
1633
|
+
if ("0.3.10".length > 0) {
|
|
1634
|
+
cached = "0.3.10";
|
|
1635
1635
|
return cached;
|
|
1636
1636
|
}
|
|
1637
1637
|
cached = "unknown";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@databricks-solutions/lakebase-scm-utils",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.10",
|
|
4
4
|
"description": "Lakebase SCM + substrate engine: database branching, paired-branch SCM workflows, connection/credential minting, schema migration, project scaffold + deploy primitives, and the git/github utility layer. Extracted from consort for portable, standalone consumption by lakebase-scm-extension and the SFTDD orchestration.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"private": false,
|
|
@@ -107,6 +107,7 @@
|
|
|
107
107
|
"lakebase-scm-recover-orphans": "./dist/scripts/lakebase/scm-recover-orphans.cli.js",
|
|
108
108
|
"lakebase-scm-state": "./dist/scripts/lakebase/scm-state.cli.js",
|
|
109
109
|
"lakebase-scm-wait-ci": "./dist/scripts/lakebase/scm-wait-ci.cli.js",
|
|
110
|
+
"lakebase-snapshot": "./dist/scripts/lakebase/snapshot.cli.js",
|
|
110
111
|
"lakebase-sync-ci-secrets": "./dist/scripts/lakebase/sync-ci-secrets.cli.js"
|
|
111
112
|
},
|
|
112
113
|
"scripts": {
|
|
@@ -87,6 +87,57 @@ export interface CreateBranchArgs extends BranchLookupOpts {
|
|
|
87
87
|
* caller's expectation. Default: false (fallback enabled).
|
|
88
88
|
*/
|
|
89
89
|
strictParent?: boolean;
|
|
90
|
+
/**
|
|
91
|
+
* Branch from an immutable SNAPSHOT instead of a source branch (the kit's
|
|
92
|
+
* fast verify-branch provisioning: snapshot the committed schema once, then
|
|
93
|
+
* branch copy-on-write from it per cycle — no `alembic upgrade head` rebuild,
|
|
94
|
+
* and a FIXED point so the fork never goes stale). Accepts a bare snapshot id
|
|
95
|
+
* or a full `projects/<p>/snapshots/<id>` path. Maps to `spec.source_snapshot`,
|
|
96
|
+
* which the Lakebase API treats as mutually exclusive with source_branch /
|
|
97
|
+
* source_branch_time / source_branch_lsn — so when set, the parentBranch
|
|
98
|
+
* resolution below is skipped entirely.
|
|
99
|
+
*/
|
|
100
|
+
fromSnapshot?: string;
|
|
101
|
+
/**
|
|
102
|
+
* Point-in-time for a branch-from-branch create: an ISO-8601 timestamp.
|
|
103
|
+
* Maps to `spec.source_branch_time` — branch the source branch AS OF this
|
|
104
|
+
* time (PITR). Mutually exclusive with sourceBranchLsn and with fromSnapshot.
|
|
105
|
+
*/
|
|
106
|
+
sourceBranchTime?: string;
|
|
107
|
+
/**
|
|
108
|
+
* Point-in-time for a branch-from-branch create: an LSN (e.g. "0/16B3748").
|
|
109
|
+
* Maps to `spec.source_branch_lsn`. Mutually exclusive with sourceBranchTime
|
|
110
|
+
* and with fromSnapshot.
|
|
111
|
+
*/
|
|
112
|
+
sourceBranchLsn?: string;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** The create-branch request `spec` — one source group (source_branch [+time|+lsn]
|
|
116
|
+
* OR source_snapshot) plus exactly one expiration (no_expiry | ttl). */
|
|
117
|
+
interface BranchSpec {
|
|
118
|
+
source_branch?: string;
|
|
119
|
+
source_snapshot?: string;
|
|
120
|
+
source_branch_time?: string;
|
|
121
|
+
source_branch_lsn?: string;
|
|
122
|
+
no_expiry?: boolean;
|
|
123
|
+
ttl?: string;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Resolve a snapshot arg (bare id or full path) to the full snapshot resource path. */
|
|
127
|
+
function snapshotPath(instance: string, snapshot: string): string {
|
|
128
|
+
return snapshot.includes("/") ? snapshot : `${projectPath(instance)}/snapshots/${snapshot}`;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Apply the expiration policy (ttl wins; else default no_expiry: true) onto a spec,
|
|
132
|
+
* refusing the inconsistent ttl + noExpiry:true combination. */
|
|
133
|
+
function applyExpiration(spec: BranchSpec, ttl: string | undefined, noExpiry: boolean | undefined): void {
|
|
134
|
+
if (ttl && noExpiry === true) {
|
|
135
|
+
throw new LakebaseBranchError(
|
|
136
|
+
`Cannot set both ttl ("${ttl}") and noExpiry: true on the same branch – they are mutually exclusive. Pass one or the other.`,
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
if (ttl) spec.ttl = ttl;
|
|
140
|
+
else if (noExpiry ?? true) spec.no_expiry = true;
|
|
90
141
|
}
|
|
91
142
|
|
|
92
143
|
/**
|
|
@@ -104,6 +155,38 @@ export async function createBranch(args: CreateBranchArgs): Promise<LakebaseBran
|
|
|
104
155
|
const sanitized = sanitizeBranchName(args.branch);
|
|
105
156
|
const lookup: BranchLookupOpts = { instance: args.instance, host: args.host };
|
|
106
157
|
|
|
158
|
+
// SNAPSHOT SOURCE (fast verify-branch provisioning): branch copy-on-write from an
|
|
159
|
+
// immutable snapshot. source_snapshot is mutually exclusive with the source_branch
|
|
160
|
+
// family, so this path skips parentBranch resolution entirely. Idempotent: a branch
|
|
161
|
+
// that already exists under this name is returned as-is (the ephemeral verify branches
|
|
162
|
+
// this serves are single-use; we don't re-assert the snapshot lineage of a survivor).
|
|
163
|
+
if (args.fromSnapshot) {
|
|
164
|
+
if (args.parentBranch || args.sourceBranchTime || args.sourceBranchLsn) {
|
|
165
|
+
throw new LakebaseBranchError(
|
|
166
|
+
`fromSnapshot is mutually exclusive with parentBranch / sourceBranchTime / sourceBranchLsn ` +
|
|
167
|
+
`(the Lakebase API rejects source_snapshot combined with any source_branch field).`,
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
const existing = await getBranchByName(sanitized, lookup);
|
|
171
|
+
if (existing) return existing;
|
|
172
|
+
const spec: BranchSpec = { source_snapshot: snapshotPath(args.instance, args.fromSnapshot) };
|
|
173
|
+
applyExpiration(spec, args.ttl, args.noExpiry);
|
|
174
|
+
try {
|
|
175
|
+
await createWithTtlRecovery(args.instance, sanitized, spec, args.host);
|
|
176
|
+
} catch (err) {
|
|
177
|
+
if (err instanceof LakebaseBranchTtlTooLongError) throw err;
|
|
178
|
+
const landed = await getBranchByName(sanitized, lookup);
|
|
179
|
+
if (!landed) throw err;
|
|
180
|
+
}
|
|
181
|
+
return waitForBranchReady({
|
|
182
|
+
instance: args.instance,
|
|
183
|
+
host: args.host,
|
|
184
|
+
branch: sanitized,
|
|
185
|
+
timeoutMs: args.readyTimeoutMs ?? KIT_TIMEOUTS.readyWait,
|
|
186
|
+
pollIntervalMs: args.pollIntervalMs ?? KIT_TIMEOUTS.readyPoll,
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
|
|
107
190
|
// Resolve the source (parent) branch full path first – needed both for
|
|
108
191
|
// create AND for the idempotency-vs-conflict comparison below.
|
|
109
192
|
let sourceBranchPath: string | undefined;
|
|
@@ -182,22 +265,18 @@ export async function createBranch(args: CreateBranchArgs): Promise<LakebaseBran
|
|
|
182
265
|
return existing;
|
|
183
266
|
}
|
|
184
267
|
|
|
185
|
-
|
|
186
|
-
// Refuse the inconsistent combination (ttl + noExpiry: true both set).
|
|
187
|
-
if (args.ttl && args.noExpiry === true) {
|
|
268
|
+
if (args.sourceBranchTime && args.sourceBranchLsn) {
|
|
188
269
|
throw new LakebaseBranchError(
|
|
189
|
-
`
|
|
190
|
-
`branch – they are mutually exclusive. Pass one or the other.`,
|
|
270
|
+
`sourceBranchTime and sourceBranchLsn are mutually exclusive – pass at most one point-in-time.`,
|
|
191
271
|
);
|
|
192
272
|
}
|
|
193
|
-
const specObj: { source_branch:
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
if (args.
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
}
|
|
273
|
+
const specObj: BranchSpec = { source_branch: sourceBranchPath };
|
|
274
|
+
// PITR: pin the source branch at a timestamp or LSN (source_branch_time /
|
|
275
|
+
// source_branch_lsn) — e.g. branch the higher env AS OF the committed-schema point.
|
|
276
|
+
if (args.sourceBranchTime) specObj.source_branch_time = args.sourceBranchTime;
|
|
277
|
+
else if (args.sourceBranchLsn) specObj.source_branch_lsn = args.sourceBranchLsn;
|
|
278
|
+
// Expiration policy: ttl wins if set; otherwise default to no_expiry: true.
|
|
279
|
+
applyExpiration(specObj, args.ttl, args.noExpiry);
|
|
201
280
|
try {
|
|
202
281
|
await createWithTtlRecovery(args.instance, sanitized, specObj, args.host);
|
|
203
282
|
} catch (err) {
|
|
@@ -299,7 +378,7 @@ function assertSourceMatches(
|
|
|
299
378
|
async function createWithTtlRecovery(
|
|
300
379
|
instance: string,
|
|
301
380
|
sanitized: string,
|
|
302
|
-
specObj:
|
|
381
|
+
specObj: BranchSpec,
|
|
303
382
|
host: string | undefined,
|
|
304
383
|
): Promise<void> {
|
|
305
384
|
const originalTtl = specObj.ttl;
|
|
@@ -39,6 +39,9 @@ interface ParsedArgs {
|
|
|
39
39
|
instance?: string;
|
|
40
40
|
branch?: string;
|
|
41
41
|
parentBranch?: string;
|
|
42
|
+
fromSnapshot?: string;
|
|
43
|
+
atTime?: string;
|
|
44
|
+
atLsn?: string;
|
|
42
45
|
ttl?: string;
|
|
43
46
|
noExpiry?: boolean;
|
|
44
47
|
cwd?: string;
|
|
@@ -85,6 +88,15 @@ function parseArgs(argv: string[]): ParsedArgs {
|
|
|
85
88
|
case "--parent-branch":
|
|
86
89
|
out.parentBranch = argv[++i];
|
|
87
90
|
break;
|
|
91
|
+
case "--from-snapshot":
|
|
92
|
+
out.fromSnapshot = argv[++i];
|
|
93
|
+
break;
|
|
94
|
+
case "--at-time":
|
|
95
|
+
out.atTime = argv[++i];
|
|
96
|
+
break;
|
|
97
|
+
case "--at-lsn":
|
|
98
|
+
out.atLsn = argv[++i];
|
|
99
|
+
break;
|
|
88
100
|
case "--ttl":
|
|
89
101
|
out.ttl = argv[++i];
|
|
90
102
|
break;
|
|
@@ -259,6 +271,9 @@ async function main(): Promise<number> {
|
|
|
259
271
|
host: args.host,
|
|
260
272
|
branch: args.branch!,
|
|
261
273
|
parentBranch: args.parentBranch,
|
|
274
|
+
fromSnapshot: args.fromSnapshot,
|
|
275
|
+
sourceBranchTime: args.atTime,
|
|
276
|
+
sourceBranchLsn: args.atLsn,
|
|
262
277
|
ttl: args.ttl,
|
|
263
278
|
noExpiry: args.noExpiry,
|
|
264
279
|
});
|
|
@@ -136,7 +136,6 @@ export async function getConnection(args: ConnectionArgs): Promise<DsnResult | P
|
|
|
136
136
|
|
|
137
137
|
// output === "pool"
|
|
138
138
|
const host = await resolveEndpointHost(args.instance, branchId);
|
|
139
|
-
const email = await resolveCurrentUser();
|
|
140
139
|
// Label the POOLED path too , it is the PRIMARY one (schema-diff, reconcile-tier). Passing
|
|
141
140
|
// application_name in createLakebasePool's config does NOT work: it builds its own pg
|
|
142
141
|
// PoolConfig and drops the field (verified live , the pooled connection came back with an
|
|
@@ -145,17 +144,35 @@ export async function getConnection(args: ConnectionArgs): Promise<DsnResult | P
|
|
|
145
144
|
// call time (not import) so CONSORT_VERSION , which Consort exports at its entry point , is
|
|
146
145
|
// already in the env when connectionApplicationName() reads it.
|
|
147
146
|
process.env.PGAPPNAME = connectionApplicationName();
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
:
|
|
158
|
-
|
|
147
|
+
|
|
148
|
+
// AppKit OBO: an explicit workspaceClient is a pre-authenticated caller , keep the
|
|
149
|
+
// OAuth/token-rotation path with it verbatim.
|
|
150
|
+
if (args.workspaceClient !== undefined) {
|
|
151
|
+
const email = await resolveCurrentUser();
|
|
152
|
+
return createLakebasePool({
|
|
153
|
+
endpoint: endpointPath,
|
|
154
|
+
host,
|
|
155
|
+
database,
|
|
156
|
+
user: email,
|
|
157
|
+
workspaceClient: args.workspaceClient as never,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// CLI/kit path (no workspaceClient): DO NOT let createLakebasePool self-authenticate
|
|
162
|
+
// via the SDK default chain. That runs `databricks auth token --host <workspace-host>`
|
|
163
|
+
// WITHOUT a profile, so when several ~/.databrickscfg profiles match that host (e.g.
|
|
164
|
+
// DEFAULT + an adb-* + logfood all pointing at the same workspace) the CLI refuses
|
|
165
|
+
// ("Use --profile to specify which profile to use") and the pool connection fails
|
|
166
|
+
// ("default auth: databricks-cli: cannot get access token"). That silently broke
|
|
167
|
+
// schema-diff at deploy-verify (the diff came back empty, failing the schema-diff
|
|
168
|
+
// test) whenever the host was ambiguous. Instead MINT the credential through the
|
|
169
|
+
// kit's profile-threaded CLI wrapper (mintCredential -> runDatabricksSync -> the
|
|
170
|
+
// resolved `--profile`, the same unambiguous path the DSN output uses) and pass it as
|
|
171
|
+
// `password`: LakebasePoolConfig skips OAuth entirely when a password is supplied. The
|
|
172
|
+
// kit's pool consumers (schema-diff, reconcile-tier) are short batch reads, well inside
|
|
173
|
+
// the credential TTL, so forgoing token rotation here is safe.
|
|
174
|
+
const { token, email } = await mintCredential(endpointPath);
|
|
175
|
+
return createLakebasePool({ host, database, user: email, password: token });
|
|
159
176
|
}
|
|
160
177
|
|
|
161
178
|
/**
|