@databricks-solutions/lakebase-scm-utils 0.3.9 → 0.3.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{pr-e-E4aP4j.d.cts → pr-D-eayWyJ.d.cts} +59 -1
- package/dist/{pr-e-E4aP4j.d.ts → pr-D-eayWyJ.d.ts} +59 -1
- package/dist/scripts/git/index.cjs +30 -14
- package/dist/scripts/git/index.d.cts +32 -1
- package/dist/scripts/git/index.d.ts +32 -1
- package/dist/scripts/git/index.js +28 -14
- package/dist/scripts/github/index.d.cts +2 -60
- package/dist/scripts/github/index.d.ts +2 -60
- package/dist/scripts/index.cjs +301 -98
- package/dist/scripts/index.d.cts +4 -4
- package/dist/scripts/index.d.ts +4 -4
- package/dist/scripts/index.js +296 -98
- package/dist/scripts/lakebase/branch.cli.cjs +55 -10
- package/dist/scripts/lakebase/branch.cli.js +55 -10
- package/dist/scripts/lakebase/ci-resolve-branch.cli.cjs +59 -24
- package/dist/scripts/lakebase/ci-resolve-branch.cli.js +59 -24
- package/dist/scripts/lakebase/collapse-heads.cli.cjs +16 -14
- package/dist/scripts/lakebase/collapse-heads.cli.js +16 -14
- package/dist/scripts/lakebase/cut-backup.cli.cjs +43 -10
- package/dist/scripts/lakebase/cut-backup.cli.js +43 -10
- package/dist/scripts/lakebase/cut-tier.cli.cjs +43 -10
- package/dist/scripts/lakebase/cut-tier.cli.js +43 -10
- package/dist/scripts/lakebase/detect-language.cli.cjs +16 -14
- package/dist/scripts/lakebase/detect-language.cli.js +16 -14
- package/dist/scripts/lakebase/doctor.cli.cjs +16 -14
- package/dist/scripts/lakebase/doctor.cli.js +16 -14
- package/dist/scripts/lakebase/get-connection.cli.cjs +16 -14
- package/dist/scripts/lakebase/get-connection.cli.js +16 -14
- package/dist/scripts/lakebase/index.cjs +313 -99
- package/dist/scripts/lakebase/index.d.cts +97 -2
- package/dist/scripts/lakebase/index.d.ts +97 -2
- package/dist/scripts/lakebase/index.js +310 -99
- package/dist/scripts/lakebase/infra-runner.cli.cjs +16 -14
- package/dist/scripts/lakebase/infra-runner.cli.js +16 -14
- package/dist/scripts/lakebase/new-migration.cli.cjs +16 -14
- package/dist/scripts/lakebase/new-migration.cli.js +16 -14
- package/dist/scripts/lakebase/schema-diff.cli.cjs +16 -14
- package/dist/scripts/lakebase/schema-diff.cli.js +16 -14
- package/dist/scripts/lakebase/schema-migrate.cli.cjs +16 -14
- package/dist/scripts/lakebase/schema-migrate.cli.js +16 -14
- package/dist/scripts/lakebase/scm-claim-feature.cli.cjs +59 -24
- package/dist/scripts/lakebase/scm-claim-feature.cli.js +59 -24
- package/dist/scripts/lakebase/scm-doctor.cli.cjs +59 -24
- package/dist/scripts/lakebase/scm-doctor.cli.js +59 -24
- package/dist/scripts/lakebase/scm-merge.cli.cjs +16 -14
- package/dist/scripts/lakebase/scm-merge.cli.js +16 -14
- package/dist/scripts/lakebase/scm-reconcile-tier.cli.cjs +16 -14
- package/dist/scripts/lakebase/scm-reconcile-tier.cli.js +16 -14
- package/dist/scripts/lakebase/scm-recover-orphans.cli.cjs +43 -10
- package/dist/scripts/lakebase/scm-recover-orphans.cli.js +43 -10
- package/dist/scripts/lakebase/snapshot.cli.cjs +418 -0
- package/dist/scripts/lakebase/snapshot.cli.d.cts +1 -0
- package/dist/scripts/lakebase/snapshot.cli.d.ts +1 -0
- package/dist/scripts/lakebase/snapshot.cli.js +395 -0
- package/dist/scripts/util/index.cjs +2 -2
- package/dist/scripts/util/index.js +2 -2
- package/package.json +2 -1
- package/scripts/git/commit-push.ts +60 -14
- package/scripts/lakebase/branch-create.ts +93 -14
- package/scripts/lakebase/branch.cli.ts +15 -0
- package/scripts/lakebase/create-preflight.ts +213 -1
- package/scripts/lakebase/create-project.ts +98 -30
- package/scripts/lakebase/get-connection.ts +29 -12
- package/scripts/lakebase/snapshot.cli.ts +117 -0
- package/scripts/lakebase/snapshot.ts +85 -0
- package/templates/project/common/scripts/run-tests.sh +43 -2
- package/templates/project/python/tests/conftest.py +22 -0
- package/templates/project/common/tests/e2e/__pycache__/conftest.cpython-314.pyc +0 -0
- package/templates/project/python/tests/__pycache__/conftest.cpython-314.pyc +0 -0
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// scripts/lakebase/databricks-cli.ts
|
|
4
|
+
import { execFile, execFileSync as execFileSync2 } from "child_process";
|
|
5
|
+
import { promisify } from "util";
|
|
6
|
+
import { join as join2 } from "path";
|
|
7
|
+
|
|
8
|
+
// scripts/lakebase/kit-config.ts
|
|
9
|
+
function intFromEnv(name, fallback) {
|
|
10
|
+
const raw = process.env[name];
|
|
11
|
+
if (!raw) return fallback;
|
|
12
|
+
const parsed = Number.parseInt(raw, 10);
|
|
13
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
|
|
14
|
+
return parsed;
|
|
15
|
+
}
|
|
16
|
+
var DAY_MS = 24 * 60 * 60 * 1e3;
|
|
17
|
+
var KIT_TIMEOUTS = {
|
|
18
|
+
cliDefault: intFromEnv("LAKEBASE_KIT_TIMEOUT_CLI_DEFAULT_MS", 3e4),
|
|
19
|
+
cliCreateProject: intFromEnv("LAKEBASE_KIT_TIMEOUT_CLI_CREATE_PROJECT_MS", 18e4),
|
|
20
|
+
cliCreateBranch: intFromEnv("LAKEBASE_KIT_TIMEOUT_CLI_CREATE_BRANCH_MS", 6e4),
|
|
21
|
+
cliCreateEndpoint: intFromEnv("LAKEBASE_KIT_TIMEOUT_CLI_CREATE_ENDPOINT_MS", 6e4),
|
|
22
|
+
readyWait: intFromEnv("LAKEBASE_KIT_TIMEOUT_READY_WAIT_MS", 12e4),
|
|
23
|
+
readyPoll: intFromEnv("LAKEBASE_KIT_TIMEOUT_READY_POLL_MS", 5e3),
|
|
24
|
+
pgConnect: intFromEnv("LAKEBASE_KIT_TIMEOUT_PG_CONNECT_MS", 1e4),
|
|
25
|
+
pgStatement: intFromEnv("LAKEBASE_KIT_TIMEOUT_PG_STATEMENT_MS", 15e3),
|
|
26
|
+
gitDefault: intFromEnv("LAKEBASE_KIT_TIMEOUT_GIT_DEFAULT_MS", 5e3),
|
|
27
|
+
gitCheckout: intFromEnv("LAKEBASE_KIT_TIMEOUT_GIT_CHECKOUT_MS", 1e4),
|
|
28
|
+
gitNetwork: intFromEnv("LAKEBASE_KIT_TIMEOUT_GIT_NETWORK_MS", 15e3),
|
|
29
|
+
gitPush: intFromEnv("LAKEBASE_KIT_TIMEOUT_GIT_PUSH_MS", 3e4),
|
|
30
|
+
cliLong: intFromEnv("LAKEBASE_KIT_TIMEOUT_CLI_LONG_MS", 6e4),
|
|
31
|
+
cmdShort: intFromEnv("LAKEBASE_KIT_TIMEOUT_CMD_SHORT_MS", 5e3),
|
|
32
|
+
initializrCacheTtl: intFromEnv("LAKEBASE_KIT_INITIALIZR_CACHE_TTL_MS", 10 * 60 * 1e3),
|
|
33
|
+
featureBranchTtlMs: intFromEnv("LAKEBASE_KIT_FEATURE_BRANCH_TTL_MS", 30 * DAY_MS),
|
|
34
|
+
testBranchTtlMs: intFromEnv("LAKEBASE_KIT_TEST_BRANCH_TTL_MS", 14 * DAY_MS),
|
|
35
|
+
uatBranchTtlMs: intFromEnv("LAKEBASE_KIT_UAT_BRANCH_TTL_MS", 14 * DAY_MS),
|
|
36
|
+
perfBranchTtlMs: intFromEnv("LAKEBASE_KIT_PERF_BRANCH_TTL_MS", 7 * DAY_MS)
|
|
37
|
+
};
|
|
38
|
+
function urlFromEnv(name, fallback) {
|
|
39
|
+
const raw = process.env[name];
|
|
40
|
+
if (!raw) return fallback;
|
|
41
|
+
return raw.replace(/\/+$/, "");
|
|
42
|
+
}
|
|
43
|
+
var KIT_REGISTRIES = {
|
|
44
|
+
mavenCentral: urlFromEnv("LAKEBASE_KIT_REGISTRY_MAVEN_CENTRAL", "https://repo1.maven.org/maven2"),
|
|
45
|
+
springInitializr: urlFromEnv("LAKEBASE_KIT_REGISTRY_SPRING_INITIALIZR", "https://start.spring.io")
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
// scripts/lakebase/databricks-profile.ts
|
|
49
|
+
import * as fs from "fs";
|
|
50
|
+
import { execFileSync } from "child_process";
|
|
51
|
+
|
|
52
|
+
// scripts/util/exec.ts
|
|
53
|
+
import * as cp from "child_process";
|
|
54
|
+
|
|
55
|
+
// scripts/lakebase/databricks-profile.ts
|
|
56
|
+
function normalizeHost(host) {
|
|
57
|
+
return host.trim().replace(/\/+$/, "").toLowerCase();
|
|
58
|
+
}
|
|
59
|
+
function selectProfileForHost(profilesJson, host) {
|
|
60
|
+
const target = normalizeHost(host);
|
|
61
|
+
if (!target) return void 0;
|
|
62
|
+
const start = profilesJson.indexOf("{");
|
|
63
|
+
if (start < 0) return void 0;
|
|
64
|
+
let parsed;
|
|
65
|
+
try {
|
|
66
|
+
parsed = JSON.parse(profilesJson.slice(start));
|
|
67
|
+
} catch {
|
|
68
|
+
return void 0;
|
|
69
|
+
}
|
|
70
|
+
const profiles = parsed.profiles;
|
|
71
|
+
if (!Array.isArray(profiles)) return void 0;
|
|
72
|
+
const names = profiles.filter((p) => {
|
|
73
|
+
if (!p || typeof p !== "object") return false;
|
|
74
|
+
const rec = p;
|
|
75
|
+
return typeof rec.name === "string" && typeof rec.host === "string" && rec.valid === true && normalizeHost(rec.host) === target;
|
|
76
|
+
}).map((p) => p.name);
|
|
77
|
+
const distinct = Array.from(new Set(names));
|
|
78
|
+
return distinct.length === 1 ? distinct[0] : void 0;
|
|
79
|
+
}
|
|
80
|
+
function resolveProfileForHostSync(host, timeoutMs = KIT_TIMEOUTS.cliDefault) {
|
|
81
|
+
if (!normalizeHost(host)) return void 0;
|
|
82
|
+
let out2;
|
|
83
|
+
try {
|
|
84
|
+
out2 = execFileSync("databricks", ["auth", "profiles", "-o", "json"], {
|
|
85
|
+
encoding: "utf8",
|
|
86
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
87
|
+
timeout: timeoutMs
|
|
88
|
+
});
|
|
89
|
+
} catch {
|
|
90
|
+
return void 0;
|
|
91
|
+
}
|
|
92
|
+
return selectProfileForHost(out2, host);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// scripts/lakebase/env-file.ts
|
|
96
|
+
import * as fs2 from "fs";
|
|
97
|
+
import * as path from "path";
|
|
98
|
+
function readEnvVar(envPath, key) {
|
|
99
|
+
if (!fs2.existsSync(envPath)) return void 0;
|
|
100
|
+
let value;
|
|
101
|
+
for (const line of fs2.readFileSync(envPath, "utf-8").split("\n")) {
|
|
102
|
+
const trimmed = line.trimStart();
|
|
103
|
+
if (trimmed.startsWith("#") || !trimmed.startsWith(`${key}=`)) continue;
|
|
104
|
+
value = trimmed.slice(key.length + 1).trim().replace(/^["']|["']$/g, "");
|
|
105
|
+
}
|
|
106
|
+
return value && value.length > 0 ? value : void 0;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// scripts/lakebase/databricks-cli.ts
|
|
110
|
+
var execFileP = promisify(execFile);
|
|
111
|
+
var DatabricksCliError = class extends Error {
|
|
112
|
+
constructor(message, profile, stderr) {
|
|
113
|
+
super(message);
|
|
114
|
+
this.profile = profile;
|
|
115
|
+
this.stderr = stderr;
|
|
116
|
+
this.name = "DatabricksCliError";
|
|
117
|
+
}
|
|
118
|
+
profile;
|
|
119
|
+
stderr;
|
|
120
|
+
};
|
|
121
|
+
var DatabricksAuthError = class extends DatabricksCliError {
|
|
122
|
+
constructor(profile, detail) {
|
|
123
|
+
const login = `databricks auth login${profile ? ` --profile ${profile}` : ""}`;
|
|
124
|
+
super(
|
|
125
|
+
`Databricks authentication failed${profile ? ` for profile "${profile}"` : ""}: the cached token is missing or expired. Re-authenticate, then re-run:
|
|
126
|
+
${login}
|
|
127
|
+
${detail}`,
|
|
128
|
+
profile,
|
|
129
|
+
detail
|
|
130
|
+
);
|
|
131
|
+
this.name = "DatabricksAuthError";
|
|
132
|
+
}
|
|
133
|
+
};
|
|
134
|
+
var profileByHost = /* @__PURE__ */ new Map();
|
|
135
|
+
var profileByEnvFile = /* @__PURE__ */ new Map();
|
|
136
|
+
var hostByEnvFile = /* @__PURE__ */ new Map();
|
|
137
|
+
function envFileHost(cwd) {
|
|
138
|
+
if (hostByEnvFile.has(cwd)) return hostByEnvFile.get(cwd);
|
|
139
|
+
const v = readEnvVar(join2(cwd, ".env"), "DATABRICKS_HOST");
|
|
140
|
+
hostByEnvFile.set(cwd, v);
|
|
141
|
+
return v;
|
|
142
|
+
}
|
|
143
|
+
function effectiveHost(opts) {
|
|
144
|
+
const base = opts.env ?? process.env;
|
|
145
|
+
const cwd = opts.cwd ?? process.cwd();
|
|
146
|
+
const h = opts.host ?? base.DATABRICKS_HOST ?? envFileHost(cwd);
|
|
147
|
+
return h?.trim() || void 0;
|
|
148
|
+
}
|
|
149
|
+
function isAuthFailure(text) {
|
|
150
|
+
return /refresh token is invalid|auth login|could not be retrieved because|not authenticated|no valid.*(credential|token)|invalid.*(access token|credential)|\b401\b|unauthorized/i.test(
|
|
151
|
+
text
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
function resolveProfile(opts) {
|
|
155
|
+
const base = opts.env ?? process.env;
|
|
156
|
+
if (opts.profile) return opts.profile;
|
|
157
|
+
const envProfile = base.DATABRICKS_CONFIG_PROFILE?.trim();
|
|
158
|
+
if (envProfile) return envProfile;
|
|
159
|
+
const cwd = opts.cwd ?? process.cwd();
|
|
160
|
+
let fromEnvFile;
|
|
161
|
+
if (profileByEnvFile.has(cwd)) {
|
|
162
|
+
fromEnvFile = profileByEnvFile.get(cwd);
|
|
163
|
+
} else {
|
|
164
|
+
fromEnvFile = readEnvVar(join2(cwd, ".env"), "DATABRICKS_CONFIG_PROFILE");
|
|
165
|
+
profileByEnvFile.set(cwd, fromEnvFile);
|
|
166
|
+
}
|
|
167
|
+
if (fromEnvFile) return fromEnvFile;
|
|
168
|
+
const host = effectiveHost(opts);
|
|
169
|
+
if (!host) return void 0;
|
|
170
|
+
if (profileByHost.has(host)) return profileByHost.get(host);
|
|
171
|
+
const resolved = resolveProfileForHostSync(host, opts.timeout);
|
|
172
|
+
profileByHost.set(host, resolved);
|
|
173
|
+
return resolved;
|
|
174
|
+
}
|
|
175
|
+
function buildInvocation(args, opts) {
|
|
176
|
+
const base = opts.env ?? process.env;
|
|
177
|
+
const trimmedHost = effectiveHost(opts)?.replace(/\/+$/, "");
|
|
178
|
+
const env = { ...base };
|
|
179
|
+
if (trimmedHost) env.DATABRICKS_HOST = trimmedHost;
|
|
180
|
+
delete env.DATABRICKS_WORKSPACE_ID;
|
|
181
|
+
const profile = resolveProfile(opts);
|
|
182
|
+
const argv = profile && !opts.noProfile && !args.includes("--profile") ? [...args, "--profile", profile] : args;
|
|
183
|
+
return { argv, env, profile };
|
|
184
|
+
}
|
|
185
|
+
function classifyDatabricksError(err, argv, profile) {
|
|
186
|
+
const e = err;
|
|
187
|
+
const asText = (v) => typeof v === "string" ? v : Buffer.isBuffer(v) ? v.toString("utf8") : "";
|
|
188
|
+
const stderr = asText(e.stderr).trim();
|
|
189
|
+
const stdout = asText(e.stdout).trim();
|
|
190
|
+
const haystack = `${e.message ?? ""}
|
|
191
|
+
${stderr}
|
|
192
|
+
${stdout}`;
|
|
193
|
+
if (isAuthFailure(haystack)) {
|
|
194
|
+
return new DatabricksAuthError(profile, stderr || stdout || (e.message ?? ""));
|
|
195
|
+
}
|
|
196
|
+
const killed = e.killed === true;
|
|
197
|
+
const signal = e.signal ?? void 0;
|
|
198
|
+
const detail = stderr ? `
|
|
199
|
+
stderr: ${stderr}` : stdout ? `
|
|
200
|
+
stdout: ${stdout}` : killed || signal ? `
|
|
201
|
+
(no output; the CLI was killed${signal ? ` by ${signal}` : ""}, likely a TIMEOUT; raise the budget via the matching LAKEBASE_KIT_TIMEOUT_* env var)` : e.code !== void 0 ? `
|
|
202
|
+
(no stderr/stdout; exit ${e.code})` : "";
|
|
203
|
+
return new DatabricksCliError(
|
|
204
|
+
`databricks ${argv.join(" ")} failed: ${e.message}${detail}`,
|
|
205
|
+
profile,
|
|
206
|
+
stderr || stdout
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
async function runDatabricks(args, opts = {}) {
|
|
210
|
+
const { argv, env, profile } = buildInvocation(args, opts);
|
|
211
|
+
const timeout = opts.timeout ?? KIT_TIMEOUTS.cliDefault;
|
|
212
|
+
try {
|
|
213
|
+
if (opts.input !== void 0) {
|
|
214
|
+
return await execDatabricksWithStdin(argv, opts.input, env, timeout);
|
|
215
|
+
}
|
|
216
|
+
const { stdout } = await execFileP("databricks", argv, { env, timeout });
|
|
217
|
+
return stdout.toString();
|
|
218
|
+
} catch (err) {
|
|
219
|
+
throw classifyDatabricksError(err, argv, profile);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
function execDatabricksWithStdin(argv, input, env, timeout) {
|
|
223
|
+
return new Promise((resolve, reject) => {
|
|
224
|
+
const child = execFile("databricks", argv, { env, timeout }, (err, stdout, stderr) => {
|
|
225
|
+
if (err) {
|
|
226
|
+
err.stdout = String(stdout ?? "");
|
|
227
|
+
err.stderr = String(stderr ?? "");
|
|
228
|
+
reject(err);
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
resolve(String(stdout ?? ""));
|
|
232
|
+
});
|
|
233
|
+
child.stdin?.on("error", () => {
|
|
234
|
+
});
|
|
235
|
+
child.stdin?.end(input);
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// scripts/lakebase/branch-utils.ts
|
|
240
|
+
function projectPath(instance) {
|
|
241
|
+
return `projects/${instance}`;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// scripts/lakebase/snapshot.ts
|
|
245
|
+
function branchResource(instance, branch) {
|
|
246
|
+
return branch.includes("/") ? branch : `${projectPath(instance)}/branches/${branch}`;
|
|
247
|
+
}
|
|
248
|
+
function snapshotResource(instance, snapshot) {
|
|
249
|
+
return snapshot.includes("/") ? snapshot : `${projectPath(instance)}/snapshots/${snapshot}`;
|
|
250
|
+
}
|
|
251
|
+
function buildSnapshotSpec(args) {
|
|
252
|
+
if (args.sourceBranchTime && args.sourceBranchLsn) {
|
|
253
|
+
throw new Error("sourceBranchTime and sourceBranchLsn are mutually exclusive \u2014 pass at most one point-in-time.");
|
|
254
|
+
}
|
|
255
|
+
if (args.ttl && args.noExpiry === true) {
|
|
256
|
+
throw new Error(`Cannot set both ttl ("${args.ttl}") and noExpiry: true \u2014 they are mutually exclusive.`);
|
|
257
|
+
}
|
|
258
|
+
const spec = { source_branch: branchResource(args.instance, args.sourceBranch) };
|
|
259
|
+
if (args.sourceBranchTime) spec.source_branch_time = args.sourceBranchTime;
|
|
260
|
+
else if (args.sourceBranchLsn) spec.source_branch_lsn = args.sourceBranchLsn;
|
|
261
|
+
if (args.ttl) spec.ttl = args.ttl;
|
|
262
|
+
else if (args.noExpiry ?? true) spec.no_expiry = true;
|
|
263
|
+
return spec;
|
|
264
|
+
}
|
|
265
|
+
async function createSnapshot(args) {
|
|
266
|
+
const spec = buildSnapshotSpec(args);
|
|
267
|
+
return runDatabricks(
|
|
268
|
+
["postgres", "create-snapshot", projectPath(args.instance), args.snapshot, "--json", JSON.stringify({ spec })],
|
|
269
|
+
{ host: args.host, timeout: KIT_TIMEOUTS.cliCreateBranch }
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
async function deleteSnapshot(args) {
|
|
273
|
+
return runDatabricks(["postgres", "delete-snapshot", snapshotResource(args.instance, args.snapshot)], {
|
|
274
|
+
host: args.host
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
async function listSnapshots(args) {
|
|
278
|
+
return runDatabricks(["postgres", "list-snapshots", projectPath(args.instance), "-o", "json"], { host: args.host });
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// scripts/lakebase/snapshot.cli.ts
|
|
282
|
+
function parseArgs(argv) {
|
|
283
|
+
const out2 = {};
|
|
284
|
+
let i = 0;
|
|
285
|
+
if (argv[0] && !argv[0].startsWith("-")) {
|
|
286
|
+
out2.subcommand = argv[0];
|
|
287
|
+
i = 1;
|
|
288
|
+
}
|
|
289
|
+
for (; i < argv.length; i++) {
|
|
290
|
+
switch (argv[i]) {
|
|
291
|
+
case "--instance":
|
|
292
|
+
out2.instance = argv[++i];
|
|
293
|
+
break;
|
|
294
|
+
case "--snapshot":
|
|
295
|
+
out2.snapshot = argv[++i];
|
|
296
|
+
break;
|
|
297
|
+
case "--branch":
|
|
298
|
+
out2.branch = argv[++i];
|
|
299
|
+
break;
|
|
300
|
+
case "--at-time":
|
|
301
|
+
out2.atTime = argv[++i];
|
|
302
|
+
break;
|
|
303
|
+
case "--at-lsn":
|
|
304
|
+
out2.atLsn = argv[++i];
|
|
305
|
+
break;
|
|
306
|
+
case "--ttl":
|
|
307
|
+
out2.ttl = argv[++i];
|
|
308
|
+
break;
|
|
309
|
+
case "--no-expiry":
|
|
310
|
+
out2.noExpiry = true;
|
|
311
|
+
break;
|
|
312
|
+
case "--host":
|
|
313
|
+
out2.host = argv[++i];
|
|
314
|
+
break;
|
|
315
|
+
case "--pretty":
|
|
316
|
+
out2.pretty = true;
|
|
317
|
+
break;
|
|
318
|
+
case "--help":
|
|
319
|
+
case "-h":
|
|
320
|
+
out2.help = true;
|
|
321
|
+
break;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
return out2;
|
|
325
|
+
}
|
|
326
|
+
var USAGE = `usage: lakebase-snapshot <create|delete|list> --instance <project> [flags]
|
|
327
|
+
create --snapshot <id> --branch <src> [--at-time <iso> | --at-lsn <lsn>] [--ttl <s> | --no-expiry]
|
|
328
|
+
delete --snapshot <id>
|
|
329
|
+
list
|
|
330
|
+
Then branch from it: lakebase-branch create --branch <v> --from-snapshot <id> --instance <project>`;
|
|
331
|
+
function out(result, pretty) {
|
|
332
|
+
process.stdout.write(JSON.stringify(result, null, pretty ? 2 : void 0) + "\n");
|
|
333
|
+
}
|
|
334
|
+
function need(a, keys) {
|
|
335
|
+
const missing = keys.filter((k) => a[k] === void 0 || a[k] === "");
|
|
336
|
+
if (missing.length) {
|
|
337
|
+
process.stderr.write(`missing ${missing.map((k) => "--" + String(k)).join(", ")}
|
|
338
|
+
${USAGE}
|
|
339
|
+
`);
|
|
340
|
+
return false;
|
|
341
|
+
}
|
|
342
|
+
return true;
|
|
343
|
+
}
|
|
344
|
+
async function main() {
|
|
345
|
+
const a = parseArgs(process.argv.slice(2));
|
|
346
|
+
if (a.help || !a.subcommand) {
|
|
347
|
+
process.stderr.write(USAGE + "\n");
|
|
348
|
+
return a.help ? 0 : 2;
|
|
349
|
+
}
|
|
350
|
+
switch (a.subcommand) {
|
|
351
|
+
case "create": {
|
|
352
|
+
if (!need(a, ["instance", "snapshot", "branch"])) return 2;
|
|
353
|
+
const raw = await createSnapshot({
|
|
354
|
+
instance: a.instance,
|
|
355
|
+
snapshot: a.snapshot,
|
|
356
|
+
sourceBranch: a.branch,
|
|
357
|
+
sourceBranchTime: a.atTime,
|
|
358
|
+
sourceBranchLsn: a.atLsn,
|
|
359
|
+
ttl: a.ttl,
|
|
360
|
+
noExpiry: a.noExpiry,
|
|
361
|
+
host: a.host
|
|
362
|
+
});
|
|
363
|
+
out({ created: a.snapshot, raw: raw.trim() || void 0 }, !!a.pretty);
|
|
364
|
+
return 0;
|
|
365
|
+
}
|
|
366
|
+
case "delete": {
|
|
367
|
+
if (!need(a, ["instance", "snapshot"])) return 2;
|
|
368
|
+
await deleteSnapshot({ instance: a.instance, snapshot: a.snapshot, host: a.host });
|
|
369
|
+
out({ deleted: a.snapshot }, !!a.pretty);
|
|
370
|
+
return 0;
|
|
371
|
+
}
|
|
372
|
+
case "list": {
|
|
373
|
+
if (!need(a, ["instance"])) return 2;
|
|
374
|
+
const raw = await listSnapshots({ instance: a.instance, host: a.host });
|
|
375
|
+
try {
|
|
376
|
+
out(JSON.parse(raw), !!a.pretty);
|
|
377
|
+
} catch {
|
|
378
|
+
process.stdout.write(raw);
|
|
379
|
+
}
|
|
380
|
+
return 0;
|
|
381
|
+
}
|
|
382
|
+
default:
|
|
383
|
+
process.stderr.write(`unknown subcommand '${a.subcommand}'
|
|
384
|
+
${USAGE}
|
|
385
|
+
`);
|
|
386
|
+
return 2;
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
main().then(
|
|
390
|
+
(code) => process.exit(code),
|
|
391
|
+
(err) => {
|
|
392
|
+
process.stderr.write(String(err instanceof Error ? err.message : err) + "\n");
|
|
393
|
+
process.exit(1);
|
|
394
|
+
}
|
|
395
|
+
);
|
|
@@ -1667,8 +1667,8 @@ var PKG_NAME = "@databricks-solutions/lakebase-scm-utils";
|
|
|
1667
1667
|
var cached;
|
|
1668
1668
|
function substrateSelfVersion() {
|
|
1669
1669
|
if (cached !== void 0) return cached;
|
|
1670
|
-
if ("0.3.
|
|
1671
|
-
cached = "0.3.
|
|
1670
|
+
if ("0.3.11".length > 0) {
|
|
1671
|
+
cached = "0.3.11";
|
|
1672
1672
|
return cached;
|
|
1673
1673
|
}
|
|
1674
1674
|
cached = "unknown";
|
|
@@ -1630,8 +1630,8 @@ var PKG_NAME = "@databricks-solutions/lakebase-scm-utils";
|
|
|
1630
1630
|
var cached;
|
|
1631
1631
|
function substrateSelfVersion() {
|
|
1632
1632
|
if (cached !== void 0) return cached;
|
|
1633
|
-
if ("0.3.
|
|
1634
|
-
cached = "0.3.
|
|
1633
|
+
if ("0.3.11".length > 0) {
|
|
1634
|
+
cached = "0.3.11";
|
|
1635
1635
|
return cached;
|
|
1636
1636
|
}
|
|
1637
1637
|
cached = "unknown";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@databricks-solutions/lakebase-scm-utils",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.11",
|
|
4
4
|
"description": "Lakebase SCM + substrate engine: database branching, paired-branch SCM workflows, connection/credential minting, schema migration, project scaffold + deploy primitives, and the git/github utility layer. Extracted from consort for portable, standalone consumption by lakebase-scm-extension and the SFTDD orchestration.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"private": false,
|
|
@@ -107,6 +107,7 @@
|
|
|
107
107
|
"lakebase-scm-recover-orphans": "./dist/scripts/lakebase/scm-recover-orphans.cli.js",
|
|
108
108
|
"lakebase-scm-state": "./dist/scripts/lakebase/scm-state.cli.js",
|
|
109
109
|
"lakebase-scm-wait-ci": "./dist/scripts/lakebase/scm-wait-ci.cli.js",
|
|
110
|
+
"lakebase-snapshot": "./dist/scripts/lakebase/snapshot.cli.js",
|
|
110
111
|
"lakebase-sync-ci-secrets": "./dist/scripts/lakebase/sync-ci-secrets.cli.js"
|
|
111
112
|
},
|
|
112
113
|
"scripts": {
|
|
@@ -45,24 +45,70 @@ export class WorkflowScopeError extends Error {
|
|
|
45
45
|
* For non-initial-commit flows, prefer the primitives in ./commits.ts.
|
|
46
46
|
*/
|
|
47
47
|
export async function commitAndPush(args: CommitAndPushArgs): Promise<void> {
|
|
48
|
-
await
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
await commitInitial(args.projectDir, args.message);
|
|
49
|
+
if (args.push === false) return;
|
|
50
|
+
await pushInitial({ projectDir: args.projectDir, remote: args.remote, branch: args.branch });
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Stage everything and make the initial commit (no push). */
|
|
54
|
+
export async function commitInitial(projectDir: string, message: string): Promise<void> {
|
|
55
|
+
await exec("git add -A", { cwd: projectDir });
|
|
56
|
+
await exec(`git commit -m ${JSON.stringify(message)}`, {
|
|
57
|
+
cwd: projectDir,
|
|
51
58
|
timeout: 30_000,
|
|
52
59
|
});
|
|
53
|
-
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface PushInitialArgs {
|
|
63
|
+
projectDir: string;
|
|
64
|
+
/** Remote name. Default: "origin". */
|
|
65
|
+
remote?: string;
|
|
66
|
+
/** Branch name. Default: "main". */
|
|
67
|
+
branch?: string;
|
|
68
|
+
/**
|
|
69
|
+
* Total push attempts (>= 1). Default 1 (preserves commitAndPush's behavior).
|
|
70
|
+
* The create flow passes 2: the FIRST push fires the project's pre-push hook,
|
|
71
|
+
* which triggers the one-time cold download of the Consort toolkit — that race
|
|
72
|
+
* can fail the push, but the toolkit is CACHED afterward, so a single retry then
|
|
73
|
+
* sails through (the cold-download incident). A `workflow`-scope rejection is a
|
|
74
|
+
* real auth gap, never retried.
|
|
75
|
+
*/
|
|
76
|
+
attempts?: number;
|
|
77
|
+
/** Delay between attempts (ms). Default 1500. Injected in tests. */
|
|
78
|
+
delayMs?: number;
|
|
79
|
+
/** Injectable sleep (tests). */
|
|
80
|
+
sleep?: (ms: number) => Promise<void>;
|
|
81
|
+
/** Injectable runner (tests); defaults to the real `git push`. */
|
|
82
|
+
run?: (cmd: string, cwd: string) => Promise<void>;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Push the initial branch with `-u`, retrying a transient failure up to
|
|
87
|
+
* `attempts` times. Maps the GitHub `workflow`-scope rejection to
|
|
88
|
+
* {@link WorkflowScopeError} (thrown immediately, never retried). Throws the last
|
|
89
|
+
* error after the final attempt.
|
|
90
|
+
*/
|
|
91
|
+
export async function pushInitial(args: PushInitialArgs): Promise<void> {
|
|
54
92
|
const remote = args.remote ?? "origin";
|
|
55
93
|
const branch = args.branch ?? "main";
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
94
|
+
const attempts = Math.max(1, args.attempts ?? 1);
|
|
95
|
+
const sleep = args.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
|
|
96
|
+
const run =
|
|
97
|
+
args.run ??
|
|
98
|
+
((cmd: string, cwd: string) => exec(cmd, { cwd, timeout: 30_000 }).then(() => undefined));
|
|
99
|
+
let lastErr: unknown;
|
|
100
|
+
for (let i = 1; i <= attempts; i++) {
|
|
101
|
+
try {
|
|
102
|
+
await run(`git push -u ${remote} ${branch}`, args.projectDir);
|
|
103
|
+
return;
|
|
104
|
+
} catch (err) {
|
|
105
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
106
|
+
if (/without `?workflow`? scope|workflow scope/i.test(msg)) {
|
|
107
|
+
throw new WorkflowScopeError(args.projectDir);
|
|
108
|
+
}
|
|
109
|
+
lastErr = err;
|
|
110
|
+
if (i < attempts) await sleep(args.delayMs ?? 1500);
|
|
65
111
|
}
|
|
66
|
-
throw err;
|
|
67
112
|
}
|
|
113
|
+
throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
|
|
68
114
|
}
|
|
@@ -87,6 +87,57 @@ export interface CreateBranchArgs extends BranchLookupOpts {
|
|
|
87
87
|
* caller's expectation. Default: false (fallback enabled).
|
|
88
88
|
*/
|
|
89
89
|
strictParent?: boolean;
|
|
90
|
+
/**
|
|
91
|
+
* Branch from an immutable SNAPSHOT instead of a source branch (the kit's
|
|
92
|
+
* fast verify-branch provisioning: snapshot the committed schema once, then
|
|
93
|
+
* branch copy-on-write from it per cycle — no `alembic upgrade head` rebuild,
|
|
94
|
+
* and a FIXED point so the fork never goes stale). Accepts a bare snapshot id
|
|
95
|
+
* or a full `projects/<p>/snapshots/<id>` path. Maps to `spec.source_snapshot`,
|
|
96
|
+
* which the Lakebase API treats as mutually exclusive with source_branch /
|
|
97
|
+
* source_branch_time / source_branch_lsn — so when set, the parentBranch
|
|
98
|
+
* resolution below is skipped entirely.
|
|
99
|
+
*/
|
|
100
|
+
fromSnapshot?: string;
|
|
101
|
+
/**
|
|
102
|
+
* Point-in-time for a branch-from-branch create: an ISO-8601 timestamp.
|
|
103
|
+
* Maps to `spec.source_branch_time` — branch the source branch AS OF this
|
|
104
|
+
* time (PITR). Mutually exclusive with sourceBranchLsn and with fromSnapshot.
|
|
105
|
+
*/
|
|
106
|
+
sourceBranchTime?: string;
|
|
107
|
+
/**
|
|
108
|
+
* Point-in-time for a branch-from-branch create: an LSN (e.g. "0/16B3748").
|
|
109
|
+
* Maps to `spec.source_branch_lsn`. Mutually exclusive with sourceBranchTime
|
|
110
|
+
* and with fromSnapshot.
|
|
111
|
+
*/
|
|
112
|
+
sourceBranchLsn?: string;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** The create-branch request `spec` — one source group (source_branch [+time|+lsn]
|
|
116
|
+
* OR source_snapshot) plus exactly one expiration (no_expiry | ttl). */
|
|
117
|
+
interface BranchSpec {
|
|
118
|
+
source_branch?: string;
|
|
119
|
+
source_snapshot?: string;
|
|
120
|
+
source_branch_time?: string;
|
|
121
|
+
source_branch_lsn?: string;
|
|
122
|
+
no_expiry?: boolean;
|
|
123
|
+
ttl?: string;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Resolve a snapshot arg (bare id or full path) to the full snapshot resource path. */
|
|
127
|
+
function snapshotPath(instance: string, snapshot: string): string {
|
|
128
|
+
return snapshot.includes("/") ? snapshot : `${projectPath(instance)}/snapshots/${snapshot}`;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Apply the expiration policy (ttl wins; else default no_expiry: true) onto a spec,
|
|
132
|
+
* refusing the inconsistent ttl + noExpiry:true combination. */
|
|
133
|
+
function applyExpiration(spec: BranchSpec, ttl: string | undefined, noExpiry: boolean | undefined): void {
|
|
134
|
+
if (ttl && noExpiry === true) {
|
|
135
|
+
throw new LakebaseBranchError(
|
|
136
|
+
`Cannot set both ttl ("${ttl}") and noExpiry: true on the same branch – they are mutually exclusive. Pass one or the other.`,
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
if (ttl) spec.ttl = ttl;
|
|
140
|
+
else if (noExpiry ?? true) spec.no_expiry = true;
|
|
90
141
|
}
|
|
91
142
|
|
|
92
143
|
/**
|
|
@@ -104,6 +155,38 @@ export async function createBranch(args: CreateBranchArgs): Promise<LakebaseBran
|
|
|
104
155
|
const sanitized = sanitizeBranchName(args.branch);
|
|
105
156
|
const lookup: BranchLookupOpts = { instance: args.instance, host: args.host };
|
|
106
157
|
|
|
158
|
+
// SNAPSHOT SOURCE (fast verify-branch provisioning): branch copy-on-write from an
|
|
159
|
+
// immutable snapshot. source_snapshot is mutually exclusive with the source_branch
|
|
160
|
+
// family, so this path skips parentBranch resolution entirely. Idempotent: a branch
|
|
161
|
+
// that already exists under this name is returned as-is (the ephemeral verify branches
|
|
162
|
+
// this serves are single-use; we don't re-assert the snapshot lineage of a survivor).
|
|
163
|
+
if (args.fromSnapshot) {
|
|
164
|
+
if (args.parentBranch || args.sourceBranchTime || args.sourceBranchLsn) {
|
|
165
|
+
throw new LakebaseBranchError(
|
|
166
|
+
`fromSnapshot is mutually exclusive with parentBranch / sourceBranchTime / sourceBranchLsn ` +
|
|
167
|
+
`(the Lakebase API rejects source_snapshot combined with any source_branch field).`,
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
const existing = await getBranchByName(sanitized, lookup);
|
|
171
|
+
if (existing) return existing;
|
|
172
|
+
const spec: BranchSpec = { source_snapshot: snapshotPath(args.instance, args.fromSnapshot) };
|
|
173
|
+
applyExpiration(spec, args.ttl, args.noExpiry);
|
|
174
|
+
try {
|
|
175
|
+
await createWithTtlRecovery(args.instance, sanitized, spec, args.host);
|
|
176
|
+
} catch (err) {
|
|
177
|
+
if (err instanceof LakebaseBranchTtlTooLongError) throw err;
|
|
178
|
+
const landed = await getBranchByName(sanitized, lookup);
|
|
179
|
+
if (!landed) throw err;
|
|
180
|
+
}
|
|
181
|
+
return waitForBranchReady({
|
|
182
|
+
instance: args.instance,
|
|
183
|
+
host: args.host,
|
|
184
|
+
branch: sanitized,
|
|
185
|
+
timeoutMs: args.readyTimeoutMs ?? KIT_TIMEOUTS.readyWait,
|
|
186
|
+
pollIntervalMs: args.pollIntervalMs ?? KIT_TIMEOUTS.readyPoll,
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
|
|
107
190
|
// Resolve the source (parent) branch full path first – needed both for
|
|
108
191
|
// create AND for the idempotency-vs-conflict comparison below.
|
|
109
192
|
let sourceBranchPath: string | undefined;
|
|
@@ -182,22 +265,18 @@ export async function createBranch(args: CreateBranchArgs): Promise<LakebaseBran
|
|
|
182
265
|
return existing;
|
|
183
266
|
}
|
|
184
267
|
|
|
185
|
-
|
|
186
|
-
// Refuse the inconsistent combination (ttl + noExpiry: true both set).
|
|
187
|
-
if (args.ttl && args.noExpiry === true) {
|
|
268
|
+
if (args.sourceBranchTime && args.sourceBranchLsn) {
|
|
188
269
|
throw new LakebaseBranchError(
|
|
189
|
-
`
|
|
190
|
-
`branch – they are mutually exclusive. Pass one or the other.`,
|
|
270
|
+
`sourceBranchTime and sourceBranchLsn are mutually exclusive – pass at most one point-in-time.`,
|
|
191
271
|
);
|
|
192
272
|
}
|
|
193
|
-
const specObj: { source_branch:
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
if (args.
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
}
|
|
273
|
+
const specObj: BranchSpec = { source_branch: sourceBranchPath };
|
|
274
|
+
// PITR: pin the source branch at a timestamp or LSN (source_branch_time /
|
|
275
|
+
// source_branch_lsn) — e.g. branch the higher env AS OF the committed-schema point.
|
|
276
|
+
if (args.sourceBranchTime) specObj.source_branch_time = args.sourceBranchTime;
|
|
277
|
+
else if (args.sourceBranchLsn) specObj.source_branch_lsn = args.sourceBranchLsn;
|
|
278
|
+
// Expiration policy: ttl wins if set; otherwise default to no_expiry: true.
|
|
279
|
+
applyExpiration(specObj, args.ttl, args.noExpiry);
|
|
201
280
|
try {
|
|
202
281
|
await createWithTtlRecovery(args.instance, sanitized, specObj, args.host);
|
|
203
282
|
} catch (err) {
|
|
@@ -299,7 +378,7 @@ function assertSourceMatches(
|
|
|
299
378
|
async function createWithTtlRecovery(
|
|
300
379
|
instance: string,
|
|
301
380
|
sanitized: string,
|
|
302
|
-
specObj:
|
|
381
|
+
specObj: BranchSpec,
|
|
303
382
|
host: string | undefined,
|
|
304
383
|
): Promise<void> {
|
|
305
384
|
const originalTtl = specObj.ttl;
|