@fabricorg/databricks-testkit 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +375 -18
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +57 -5
- package/dist/index.d.ts +57 -5
- package/dist/index.js +364 -20
- package/dist/index.js.map +1 -1
- package/package.json +12 -14
- package/LICENSE +0 -21
package/dist/index.cjs
CHANGED
|
@@ -500,7 +500,7 @@ function looselyEqual(a, b) {
|
|
|
500
500
|
|
|
501
501
|
// src/mock-client.ts
|
|
502
502
|
function createMockDatabricksClient() {
|
|
503
|
-
const queues = { GET: [], POST: [] };
|
|
503
|
+
const queues = { GET: [], POST: [], DELETE: [] };
|
|
504
504
|
const calls = [];
|
|
505
505
|
function take(method, path) {
|
|
506
506
|
const index = queues[method].findIndex((q) => path.startsWith(q.pathPrefix));
|
|
@@ -524,6 +524,12 @@ function createMockDatabricksClient() {
|
|
|
524
524
|
if (queued.error) throw queued.error;
|
|
525
525
|
return queued.response;
|
|
526
526
|
},
|
|
527
|
+
async request(method, path, body, query) {
|
|
528
|
+
calls.push({ method, path, body, query });
|
|
529
|
+
const queued = take(method, path);
|
|
530
|
+
if (queued.error) throw queued.error;
|
|
531
|
+
return queued.response;
|
|
532
|
+
},
|
|
527
533
|
queue(method, pathPrefix, response) {
|
|
528
534
|
queues[method].push({ pathPrefix, response });
|
|
529
535
|
return client;
|
|
@@ -692,6 +698,26 @@ function sqlRoundTripCheck() {
|
|
|
692
698
|
}
|
|
693
699
|
};
|
|
694
700
|
}
|
|
701
|
+
function restrictedPrincipalCheck() {
|
|
702
|
+
return {
|
|
703
|
+
id: "restricted-principal",
|
|
704
|
+
description: "Live checks use the designated least-privilege service principal",
|
|
705
|
+
configured: (env) => Boolean(env.DBX_TEST_RESTRICTED_PRINCIPAL_ID),
|
|
706
|
+
async run({ env }) {
|
|
707
|
+
const expected = env.DBX_TEST_RESTRICTED_PRINCIPAL_ID?.toLowerCase();
|
|
708
|
+
const actual = env.DATABRICKS_CLIENT_ID?.toLowerCase();
|
|
709
|
+
if (!expected || !actual) {
|
|
710
|
+
throw new Error(
|
|
711
|
+
"Restricted-principal check requires DBX_TEST_RESTRICTED_PRINCIPAL_ID and DATABRICKS_CLIENT_ID"
|
|
712
|
+
);
|
|
713
|
+
}
|
|
714
|
+
if (actual !== expected) {
|
|
715
|
+
throw new Error(`Live suite is using principal '${actual}', expected '${expected}'`);
|
|
716
|
+
}
|
|
717
|
+
return { principalId: actual };
|
|
718
|
+
}
|
|
719
|
+
};
|
|
720
|
+
}
|
|
695
721
|
function assertDeltaContract(table, rows, columns) {
|
|
696
722
|
for (const expected of columns) {
|
|
697
723
|
const actual = rows.find((row) => String(row.col_name ?? row.column_name) === expected.name);
|
|
@@ -783,17 +809,21 @@ function notebookSubmitCheck() {
|
|
|
783
809
|
return {
|
|
784
810
|
id: "notebook",
|
|
785
811
|
description: "Databricks notebook submit run reaches SUCCESS",
|
|
786
|
-
configured: (env) => Boolean(
|
|
812
|
+
configured: (env) => Boolean(
|
|
813
|
+
env.DBX_TEST_NOTEBOOK_PATH && (env.DBX_TEST_CLUSTER_ID || env.DBX_TEST_NOTEBOOK_SERVERLESS === "1")
|
|
814
|
+
),
|
|
787
815
|
async run({ env, client }) {
|
|
788
|
-
if (!env.DBX_TEST_NOTEBOOK_PATH || !env.DBX_TEST_CLUSTER_ID) {
|
|
789
|
-
throw new Error(
|
|
816
|
+
if (!env.DBX_TEST_NOTEBOOK_PATH || !env.DBX_TEST_CLUSTER_ID && env.DBX_TEST_NOTEBOOK_SERVERLESS !== "1") {
|
|
817
|
+
throw new Error(
|
|
818
|
+
"Notebook submit requires DBX_TEST_NOTEBOOK_PATH and either DBX_TEST_CLUSTER_ID or DBX_TEST_NOTEBOOK_SERVERLESS=1"
|
|
819
|
+
);
|
|
790
820
|
}
|
|
791
821
|
const submitted = await client.post("/api/2.1/jobs/runs/submit", {
|
|
792
822
|
run_name: `fabric-experiments-notebook-${Date.now()}`,
|
|
793
823
|
tasks: [
|
|
794
824
|
{
|
|
795
825
|
task_key: "notebook",
|
|
796
|
-
existing_cluster_id: env.DBX_TEST_CLUSTER_ID,
|
|
826
|
+
...env.DBX_TEST_CLUSTER_ID ? { existing_cluster_id: env.DBX_TEST_CLUSTER_ID } : {},
|
|
797
827
|
notebook_task: { notebook_path: env.DBX_TEST_NOTEBOOK_PATH }
|
|
798
828
|
}
|
|
799
829
|
]
|
|
@@ -813,29 +843,59 @@ function dbtBuildCheck() {
|
|
|
813
843
|
return {
|
|
814
844
|
id: "dbt",
|
|
815
845
|
description: "Databricks dbt task reaches SUCCESS",
|
|
816
|
-
configured: (env) => Boolean(
|
|
846
|
+
configured: (env) => Boolean(
|
|
847
|
+
(env.DBX_TEST_DBT_GIT_URL || env.DBX_TEST_DBT_PROJECT_DIR) && env.DATABRICKS_WAREHOUSE_ID
|
|
848
|
+
),
|
|
817
849
|
async run({ env, client }) {
|
|
818
|
-
if (!env.DBX_TEST_DBT_GIT_URL || !env.DATABRICKS_WAREHOUSE_ID) {
|
|
819
|
-
throw new Error(
|
|
850
|
+
if (!env.DBX_TEST_DBT_GIT_URL && !env.DBX_TEST_DBT_PROJECT_DIR || !env.DATABRICKS_WAREHOUSE_ID) {
|
|
851
|
+
throw new Error(
|
|
852
|
+
"dbt build requires DBX_TEST_DBT_GIT_URL or DBX_TEST_DBT_PROJECT_DIR, plus DATABRICKS_WAREHOUSE_ID"
|
|
853
|
+
);
|
|
820
854
|
}
|
|
821
|
-
const
|
|
855
|
+
const sourceVars = JSON.stringify({
|
|
856
|
+
source_catalog: env.DBX_TEST_DBT_SOURCE_CATALOG ?? env.DBX_TEST_CATALOG,
|
|
857
|
+
source_schema: env.DBX_TEST_DBT_SOURCE_SCHEMA
|
|
858
|
+
});
|
|
859
|
+
const serverless = env.DBX_TEST_DBT_SERVERLESS === "1";
|
|
860
|
+
const request = {
|
|
822
861
|
run_name: `fabric-experiments-dbt-${Date.now()}`,
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
862
|
+
...env.DBX_TEST_DBT_GIT_URL ? {
|
|
863
|
+
git_source: {
|
|
864
|
+
git_url: env.DBX_TEST_DBT_GIT_URL,
|
|
865
|
+
git_provider: env.DBX_TEST_DBT_GIT_PROVIDER ?? "gitHub",
|
|
866
|
+
git_branch: env.DBX_TEST_DBT_GIT_BRANCH ?? "main"
|
|
867
|
+
}
|
|
868
|
+
} : {},
|
|
828
869
|
tasks: [
|
|
829
870
|
{
|
|
830
871
|
task_key: "dbt_build",
|
|
872
|
+
...serverless ? { environment_key: "dbt" } : {},
|
|
831
873
|
dbt_task: {
|
|
832
|
-
commands: ["dbt deps",
|
|
874
|
+
commands: ["dbt deps", `dbt build --vars '${sourceVars}'`],
|
|
833
875
|
warehouse_id: env.DATABRICKS_WAREHOUSE_ID,
|
|
876
|
+
...env.DBX_TEST_CATALOG ? { catalog: env.DBX_TEST_CATALOG } : {},
|
|
877
|
+
...env.DBX_TEST_SCHEMA ? { schema: env.DBX_TEST_SCHEMA } : {},
|
|
878
|
+
...env.DBX_TEST_DBT_GIT_URL ? {} : { source: "WORKSPACE" },
|
|
834
879
|
...env.DBX_TEST_DBT_PROJECT_DIR ? { project_directory: env.DBX_TEST_DBT_PROJECT_DIR } : {}
|
|
835
880
|
}
|
|
836
881
|
}
|
|
837
|
-
]
|
|
838
|
-
|
|
882
|
+
],
|
|
883
|
+
...serverless ? {
|
|
884
|
+
environments: [
|
|
885
|
+
{
|
|
886
|
+
environment_key: "dbt",
|
|
887
|
+
spec: {
|
|
888
|
+
environment_version: "2",
|
|
889
|
+
dependencies: ["dbt-databricks>=1.8,<2"]
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
]
|
|
893
|
+
} : {}
|
|
894
|
+
};
|
|
895
|
+
const submitted = await client.post(
|
|
896
|
+
"/api/2.1/jobs/runs/submit",
|
|
897
|
+
request
|
|
898
|
+
);
|
|
839
899
|
if (!submitted.run_id) throw new Error("dbt submit returned no run_id");
|
|
840
900
|
const run = await databricks.waitForDatabricksRun(client, submitted.run_id, {
|
|
841
901
|
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
@@ -843,7 +903,10 @@ function dbtBuildCheck() {
|
|
|
843
903
|
});
|
|
844
904
|
const result = run.state?.result_state;
|
|
845
905
|
if (result !== "SUCCESS") throw new Error(`dbt run ended in ${result ?? "UNKNOWN"}`);
|
|
846
|
-
return {
|
|
906
|
+
return {
|
|
907
|
+
runId: submitted.run_id,
|
|
908
|
+
source: env.DBX_TEST_DBT_GIT_URL ?? env.DBX_TEST_DBT_PROJECT_DIR
|
|
909
|
+
};
|
|
847
910
|
}
|
|
848
911
|
};
|
|
849
912
|
}
|
|
@@ -922,6 +985,99 @@ function lakebaseRoundTripCheck(options = {}) {
|
|
|
922
985
|
};
|
|
923
986
|
}
|
|
924
987
|
|
|
988
|
+
// src/lakebase-branch.ts
|
|
989
|
+
init_workspace_context();
|
|
990
|
+
var DEFAULT_POLL_INTERVAL_MS = 2e3;
|
|
991
|
+
var DEFAULT_TIMEOUT_MS = 5 * 6e4;
|
|
992
|
+
function resolveLakebaseProject(env, explicit) {
|
|
993
|
+
const project = explicit ?? env.DBX_TEST_LAKEBASE_PROJECT;
|
|
994
|
+
if (!project || !/^projects\/[a-z0-9][a-z0-9-]{0,62}$/.test(project)) {
|
|
995
|
+
throw new Error(
|
|
996
|
+
`Lakebase project must look like projects/fabric-experiments; got '${project ?? ""}'`
|
|
997
|
+
);
|
|
998
|
+
}
|
|
999
|
+
return project;
|
|
1000
|
+
}
|
|
1001
|
+
function toLakebaseBranchId(environmentName) {
|
|
1002
|
+
const normalized = environmentName.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
1003
|
+
const prefixed = normalized.startsWith("fx-") ? normalized : `fx-${normalized}`;
|
|
1004
|
+
const trimmed = prefixed.slice(0, 63).replace(/-+$/g, "");
|
|
1005
|
+
if (!/^fx-[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(trimmed)) {
|
|
1006
|
+
throw new Error(`Cannot derive a safe Lakebase branch ID from '${environmentName}'`);
|
|
1007
|
+
}
|
|
1008
|
+
return trimmed;
|
|
1009
|
+
}
|
|
1010
|
+
async function createEphemeralLakebaseBranch(options) {
|
|
1011
|
+
const env = options.env ?? process.env;
|
|
1012
|
+
const client = options.client ?? createClientFromEnv(env);
|
|
1013
|
+
const project = resolveLakebaseProject(env, options.project);
|
|
1014
|
+
const branchId = toLakebaseBranchId(options.branchId);
|
|
1015
|
+
const ttlSeconds = Math.max(60, Math.ceil(options.ttlMs / 1e3));
|
|
1016
|
+
const operation = await client.post(
|
|
1017
|
+
`/api/2.0/postgres/${project}/branches`,
|
|
1018
|
+
{ spec: { ttl: `${ttlSeconds}s` } },
|
|
1019
|
+
{ branch_id: branchId, replace_existing: false }
|
|
1020
|
+
);
|
|
1021
|
+
const branch = await waitForOperation(client, operation, options);
|
|
1022
|
+
const branchName = branch?.name ?? `${project}/branches/${branchId}`;
|
|
1023
|
+
const [endpointResult, databaseResult] = await Promise.all([
|
|
1024
|
+
client.get(`/api/2.0/postgres/${branchName}/endpoints`),
|
|
1025
|
+
client.get(`/api/2.0/postgres/${branchName}/databases`)
|
|
1026
|
+
]);
|
|
1027
|
+
const endpoint = endpointResult.endpoints?.find((item) => item.endpoint_id === "primary") ?? endpointResult.endpoints?.[0];
|
|
1028
|
+
const database = databaseResult.databases?.[0];
|
|
1029
|
+
const host = endpoint?.status?.hosts?.host;
|
|
1030
|
+
const postgresDatabase = database?.status?.postgres_database;
|
|
1031
|
+
if (!endpoint?.name || !host || !database?.name || !postgresDatabase) {
|
|
1032
|
+
throw new Error(`Lakebase branch '${branchName}' did not expose an endpoint and database`);
|
|
1033
|
+
}
|
|
1034
|
+
const createdAt = branch?.create_time ?? (options.now?.() ?? /* @__PURE__ */ new Date()).toISOString();
|
|
1035
|
+
return {
|
|
1036
|
+
project,
|
|
1037
|
+
branch: branchName,
|
|
1038
|
+
branchId,
|
|
1039
|
+
endpoint: endpoint.name,
|
|
1040
|
+
host,
|
|
1041
|
+
databaseResource: database.name,
|
|
1042
|
+
database: postgresDatabase,
|
|
1043
|
+
createdAt,
|
|
1044
|
+
expiresAt: new Date(Date.parse(createdAt) + ttlSeconds * 1e3).toISOString()
|
|
1045
|
+
};
|
|
1046
|
+
}
|
|
1047
|
+
async function deleteEphemeralLakebaseBranch(branch, options = {}) {
|
|
1048
|
+
if (!/^projects\/[a-z0-9-]+\/branches\/fx-[a-z0-9-]+$/.test(branch.branch)) {
|
|
1049
|
+
throw new Error(`Refusing to delete non-ephemeral Lakebase branch '${branch.branch}'`);
|
|
1050
|
+
}
|
|
1051
|
+
const env = options.env ?? process.env;
|
|
1052
|
+
const client = options.client ?? createClientFromEnv(env);
|
|
1053
|
+
const operation = await client.request(
|
|
1054
|
+
"DELETE",
|
|
1055
|
+
`/api/2.0/postgres/${branch.branch}`,
|
|
1056
|
+
void 0,
|
|
1057
|
+
{ purge: true }
|
|
1058
|
+
);
|
|
1059
|
+
await waitForOperation(client, operation, options);
|
|
1060
|
+
}
|
|
1061
|
+
async function waitForOperation(client, initial, options) {
|
|
1062
|
+
let operation = initial;
|
|
1063
|
+
const deadline = Date.now() + (options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
1064
|
+
while (!operation.done) {
|
|
1065
|
+
if (!operation.name) throw new Error("Lakebase operation did not return an operation name");
|
|
1066
|
+
if (Date.now() >= deadline) throw new Error(`Lakebase operation '${operation.name}' timed out`);
|
|
1067
|
+
await delay(options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS);
|
|
1068
|
+
operation = await client.get(`/api/2.0/postgres/${operation.name}`);
|
|
1069
|
+
}
|
|
1070
|
+
if (operation.error) {
|
|
1071
|
+
throw new Error(
|
|
1072
|
+
`Lakebase operation failed${operation.error.code ? ` (${operation.error.code})` : ""}: ${operation.error.message ?? "unknown error"}`
|
|
1073
|
+
);
|
|
1074
|
+
}
|
|
1075
|
+
return operation.response;
|
|
1076
|
+
}
|
|
1077
|
+
function delay(ms) {
|
|
1078
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
1079
|
+
}
|
|
1080
|
+
|
|
925
1081
|
// src/databricks-http.ts
|
|
926
1082
|
init_workspace_context();
|
|
927
1083
|
async function databricksFetch(env, pathOrUrl, init = {}, fetchImpl = fetch) {
|
|
@@ -1100,6 +1256,195 @@ function autoLoaderIngestionCheck(options = {}) {
|
|
|
1100
1256
|
}
|
|
1101
1257
|
};
|
|
1102
1258
|
}
|
|
1259
|
+
function safeName(prefix) {
|
|
1260
|
+
return `${prefix}_${crypto.randomUUID().replaceAll("-", "_")}`;
|
|
1261
|
+
}
|
|
1262
|
+
function performanceBudgetCheck() {
|
|
1263
|
+
return {
|
|
1264
|
+
id: "performance",
|
|
1265
|
+
description: "SQL Statement Execution p95 stays within the configured latency budget",
|
|
1266
|
+
configured: (env) => Boolean(env.DBX_TEST_PERFORMANCE_P95_MS),
|
|
1267
|
+
async run({ env, warehouse }) {
|
|
1268
|
+
const budgetMs = Number(env.DBX_TEST_PERFORMANCE_P95_MS);
|
|
1269
|
+
const runs = Number(env.DBX_TEST_PERFORMANCE_RUNS ?? 7);
|
|
1270
|
+
if (!Number.isFinite(budgetMs) || budgetMs <= 0) {
|
|
1271
|
+
throw new Error("DBX_TEST_PERFORMANCE_P95_MS must be a positive number");
|
|
1272
|
+
}
|
|
1273
|
+
if (!Number.isInteger(runs) || runs < 3 || runs > 100) {
|
|
1274
|
+
throw new Error("DBX_TEST_PERFORMANCE_RUNS must be an integer from 3 to 100");
|
|
1275
|
+
}
|
|
1276
|
+
await warehouse.query("SELECT 1 AS warmup");
|
|
1277
|
+
const durations = [];
|
|
1278
|
+
for (let index = 0; index < runs; index += 1) {
|
|
1279
|
+
const started = performance.now();
|
|
1280
|
+
await warehouse.query("SELECT 1 AS performance_probe");
|
|
1281
|
+
durations.push(performance.now() - started);
|
|
1282
|
+
}
|
|
1283
|
+
durations.sort((left, right) => left - right);
|
|
1284
|
+
const p95 = durations[Math.ceil(durations.length * 0.95) - 1] ?? 0;
|
|
1285
|
+
if (p95 > budgetMs) {
|
|
1286
|
+
throw new Error(`SQL p95 ${Math.round(p95)}ms exceeded ${budgetMs}ms budget`);
|
|
1287
|
+
}
|
|
1288
|
+
return {
|
|
1289
|
+
runs,
|
|
1290
|
+
p50Ms: Math.round(durations[Math.floor(durations.length * 0.5)] ?? 0),
|
|
1291
|
+
p95Ms: Math.round(p95),
|
|
1292
|
+
budgetMs
|
|
1293
|
+
};
|
|
1294
|
+
}
|
|
1295
|
+
};
|
|
1296
|
+
}
|
|
1297
|
+
function failureRecoveryCheck() {
|
|
1298
|
+
return {
|
|
1299
|
+
id: "failure-recovery",
|
|
1300
|
+
description: "A failed statement is isolated and a subsequent statement recovers",
|
|
1301
|
+
async run({ warehouse }) {
|
|
1302
|
+
let failedAsExpected = false;
|
|
1303
|
+
try {
|
|
1304
|
+
await warehouse.query("SELECT * FROM __fabric_experiments_expected_missing_table__");
|
|
1305
|
+
} catch {
|
|
1306
|
+
failedAsExpected = true;
|
|
1307
|
+
}
|
|
1308
|
+
if (!failedAsExpected) throw new Error("Injected failure unexpectedly succeeded");
|
|
1309
|
+
const rows = await warehouse.query("SELECT 1 AS recovered");
|
|
1310
|
+
if (Number(rows[0]?.recovered) !== 1) {
|
|
1311
|
+
throw new Error("Statement execution did not recover after the injected failure");
|
|
1312
|
+
}
|
|
1313
|
+
return { injectedFailure: true, recovered: true };
|
|
1314
|
+
}
|
|
1315
|
+
};
|
|
1316
|
+
}
|
|
1317
|
+
function secretRotationCheck(options = {}) {
|
|
1318
|
+
return {
|
|
1319
|
+
id: "secret-rotation",
|
|
1320
|
+
description: "Databricks secret put, overwrite, metadata verification, and cleanup",
|
|
1321
|
+
configured: (env) => Boolean(env.DBX_TEST_ROTATION_SECRET_SCOPE),
|
|
1322
|
+
async run({ env, client }) {
|
|
1323
|
+
const scope = env.DBX_TEST_ROTATION_SECRET_SCOPE;
|
|
1324
|
+
const key = options.randomId?.() ?? safeName("fx_rotation_probe");
|
|
1325
|
+
const put = async (value) => client.post("/api/2.0/secrets/put", { scope, key, string_value: value });
|
|
1326
|
+
try {
|
|
1327
|
+
await put(crypto.randomBytes(32).toString("base64url"));
|
|
1328
|
+
await put(crypto.randomBytes(32).toString("base64url"));
|
|
1329
|
+
const listed = await client.get("/api/2.0/secrets/list", {
|
|
1330
|
+
scope
|
|
1331
|
+
});
|
|
1332
|
+
if (!(listed.secrets ?? []).some((secret) => secret.key === key)) {
|
|
1333
|
+
throw new Error(`Rotated secret metadata was not visible in scope ${scope}`);
|
|
1334
|
+
}
|
|
1335
|
+
return { scope, rotated: true, cleanedUp: true };
|
|
1336
|
+
} finally {
|
|
1337
|
+
await client.post("/api/2.0/secrets/delete", { scope, key }).catch(() => void 0);
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
};
|
|
1341
|
+
}
|
|
1342
|
+
function backupRestoreCheck() {
|
|
1343
|
+
return {
|
|
1344
|
+
id: "backup-restore",
|
|
1345
|
+
description: "Delta DEEP CLONE backup restores a dropped scratch table",
|
|
1346
|
+
async run({ warehouse }) {
|
|
1347
|
+
const source = warehouse.qual(safeName("backup_source"));
|
|
1348
|
+
const backup = warehouse.qual(safeName("backup_clone"));
|
|
1349
|
+
try {
|
|
1350
|
+
await warehouse.exec(`CREATE TABLE ${source} USING DELTA AS SELECT 7 AS value`);
|
|
1351
|
+
await warehouse.exec(`CREATE TABLE ${backup} DEEP CLONE ${source}`);
|
|
1352
|
+
await warehouse.exec(`DROP TABLE ${source}`);
|
|
1353
|
+
await warehouse.exec(`CREATE TABLE ${source} DEEP CLONE ${backup}`);
|
|
1354
|
+
const rows = await warehouse.query(`SELECT value FROM ${source}`);
|
|
1355
|
+
if (Number(rows[0]?.value) !== 7) throw new Error("Restored Delta table lost its data");
|
|
1356
|
+
return { restoredRows: rows.length, deepClone: true };
|
|
1357
|
+
} finally {
|
|
1358
|
+
await warehouse.exec(`DROP TABLE IF EXISTS ${source}`).catch(() => void 0);
|
|
1359
|
+
await warehouse.exec(`DROP TABLE IF EXISTS ${backup}`).catch(() => void 0);
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
};
|
|
1363
|
+
}
|
|
1364
|
+
function rollbackCheck() {
|
|
1365
|
+
return {
|
|
1366
|
+
id: "rollback",
|
|
1367
|
+
description: "Delta time-travel rollback restores a known-good table version",
|
|
1368
|
+
async run({ warehouse }) {
|
|
1369
|
+
const table = warehouse.qual(safeName("rollback_probe"));
|
|
1370
|
+
try {
|
|
1371
|
+
await warehouse.exec(`CREATE TABLE ${table} (value INT) USING DELTA`);
|
|
1372
|
+
await warehouse.exec(`INSERT INTO ${table} VALUES (1)`);
|
|
1373
|
+
const history = await warehouse.query(`DESCRIBE HISTORY ${table} LIMIT 1`);
|
|
1374
|
+
const knownGoodVersion = Number(history[0]?.version);
|
|
1375
|
+
if (!Number.isInteger(knownGoodVersion)) {
|
|
1376
|
+
throw new Error("Delta history did not return a known-good version");
|
|
1377
|
+
}
|
|
1378
|
+
await warehouse.exec(`INSERT INTO ${table} VALUES (999)`);
|
|
1379
|
+
await warehouse.exec(`RESTORE TABLE ${table} TO VERSION AS OF ${knownGoodVersion}`);
|
|
1380
|
+
const rows = await warehouse.query(`SELECT value FROM ${table} ORDER BY value`);
|
|
1381
|
+
if (rows.length !== 1 || Number(rows[0]?.value) !== 1) {
|
|
1382
|
+
throw new Error("Delta rollback did not restore the known-good contents");
|
|
1383
|
+
}
|
|
1384
|
+
return { knownGoodVersion, restoredRows: rows.length };
|
|
1385
|
+
} finally {
|
|
1386
|
+
await warehouse.exec(`DROP TABLE IF EXISTS ${table}`).catch(() => void 0);
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
};
|
|
1390
|
+
}
|
|
1391
|
+
function serverlessComputeCheck() {
|
|
1392
|
+
return {
|
|
1393
|
+
id: "azure-serverless",
|
|
1394
|
+
description: "Azure Databricks serverless SQL warehouse is enabled and executes SQL",
|
|
1395
|
+
configured: (env) => Boolean(env.DATABRICKS_WAREHOUSE_ID),
|
|
1396
|
+
async run({ env, client, warehouse }) {
|
|
1397
|
+
const id = env.DATABRICKS_WAREHOUSE_ID;
|
|
1398
|
+
const details = await client.get(`/api/2.0/sql/warehouses/${encodeURIComponent(id)}`);
|
|
1399
|
+
if (details.enable_serverless_compute !== true) {
|
|
1400
|
+
throw new Error(`SQL warehouse ${id} is not configured for serverless compute`);
|
|
1401
|
+
}
|
|
1402
|
+
const rows = await warehouse.query("SELECT current_catalog() AS catalog");
|
|
1403
|
+
return { warehouseId: id, name: details.name, state: details.state, rows: rows.length };
|
|
1404
|
+
}
|
|
1405
|
+
};
|
|
1406
|
+
}
|
|
1407
|
+
function classicComputeCheck() {
|
|
1408
|
+
return {
|
|
1409
|
+
id: "azure-classic",
|
|
1410
|
+
description: "Azure Databricks classic job cluster starts and completes a notebook workload",
|
|
1411
|
+
configured: (env) => Boolean(
|
|
1412
|
+
env.DBX_TEST_CLASSIC_SPARK_VERSION && env.DBX_TEST_CLASSIC_NODE_TYPE && env.DBX_TEST_NOTEBOOK_PATH
|
|
1413
|
+
),
|
|
1414
|
+
async run({ env, client }) {
|
|
1415
|
+
const submitted = await client.post("/api/2.1/jobs/runs/submit", {
|
|
1416
|
+
run_name: `fabric-experiments-classic-certification-${Date.now()}`,
|
|
1417
|
+
tasks: [
|
|
1418
|
+
{
|
|
1419
|
+
task_key: "classic_notebook",
|
|
1420
|
+
new_cluster: {
|
|
1421
|
+
spark_version: env.DBX_TEST_CLASSIC_SPARK_VERSION,
|
|
1422
|
+
node_type_id: env.DBX_TEST_CLASSIC_NODE_TYPE,
|
|
1423
|
+
num_workers: Number(env.DBX_TEST_CLASSIC_WORKERS ?? 1),
|
|
1424
|
+
autotermination_minutes: 10,
|
|
1425
|
+
data_security_mode: env.DBX_TEST_CLASSIC_SECURITY_MODE ?? "SINGLE_USER"
|
|
1426
|
+
},
|
|
1427
|
+
notebook_task: { notebook_path: env.DBX_TEST_NOTEBOOK_PATH }
|
|
1428
|
+
}
|
|
1429
|
+
]
|
|
1430
|
+
});
|
|
1431
|
+
if (!submitted.run_id) throw new Error("Classic compute submit returned no run_id");
|
|
1432
|
+
const run = await databricks.waitForDatabricksRun(client, submitted.run_id, {
|
|
1433
|
+
timeoutMs: Number(env.DBX_TEST_CLASSIC_TIMEOUT_MS ?? 18e5),
|
|
1434
|
+
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 15e3)
|
|
1435
|
+
});
|
|
1436
|
+
const result = run.state?.result_state;
|
|
1437
|
+
if (result !== "SUCCESS") {
|
|
1438
|
+
throw new Error(`Classic compute run ended in ${result ?? "UNKNOWN"}`);
|
|
1439
|
+
}
|
|
1440
|
+
return {
|
|
1441
|
+
runId: submitted.run_id,
|
|
1442
|
+
sparkVersion: env.DBX_TEST_CLASSIC_SPARK_VERSION,
|
|
1443
|
+
nodeType: env.DBX_TEST_CLASSIC_NODE_TYPE
|
|
1444
|
+
};
|
|
1445
|
+
}
|
|
1446
|
+
};
|
|
1447
|
+
}
|
|
1103
1448
|
|
|
1104
1449
|
Object.defineProperty(exports, "waitForDatabricksRun", {
|
|
1105
1450
|
enumerable: true,
|
|
@@ -1112,9 +1457,12 @@ Object.defineProperty(exports, "waitForDatabricksStatement", {
|
|
|
1112
1457
|
exports.appHealthCheck = appHealthCheck;
|
|
1113
1458
|
exports.assertDeltaContract = assertDeltaContract;
|
|
1114
1459
|
exports.autoLoaderIngestionCheck = autoLoaderIngestionCheck;
|
|
1460
|
+
exports.backupRestoreCheck = backupRestoreCheck;
|
|
1461
|
+
exports.classicComputeCheck = classicComputeCheck;
|
|
1115
1462
|
exports.compareToGolden = compareToGolden;
|
|
1116
1463
|
exports.createClientFromEnv = createClientFromEnv;
|
|
1117
1464
|
exports.createDuckDbContext = createDuckDbContext;
|
|
1465
|
+
exports.createEphemeralLakebaseBranch = createEphemeralLakebaseBranch;
|
|
1118
1466
|
exports.createExecutionContext = createExecutionContext;
|
|
1119
1467
|
exports.createLakebaseTestProvider = createLakebaseTestProvider;
|
|
1120
1468
|
exports.createMockDatabricksClient = createMockDatabricksClient;
|
|
@@ -1122,11 +1470,13 @@ exports.createWorkspaceContext = createWorkspaceContext;
|
|
|
1122
1470
|
exports.customLiveCheck = customLiveCheck;
|
|
1123
1471
|
exports.dbtBuildCheck = dbtBuildCheck;
|
|
1124
1472
|
exports.defineLiveSuite = defineLiveSuite;
|
|
1473
|
+
exports.deleteEphemeralLakebaseBranch = deleteEphemeralLakebaseBranch;
|
|
1125
1474
|
exports.deleteVolumeFile = deleteVolumeFile;
|
|
1126
1475
|
exports.deltaContractCheck = deltaContractCheck;
|
|
1127
1476
|
exports.ensureVolumeDirectory = ensureVolumeDirectory;
|
|
1128
1477
|
exports.expectQueryToMatchGolden = expectQueryToMatchGolden;
|
|
1129
1478
|
exports.expectTable = expectTable;
|
|
1479
|
+
exports.failureRecoveryCheck = failureRecoveryCheck;
|
|
1130
1480
|
exports.jobRunCheck = jobRunCheck;
|
|
1131
1481
|
exports.lakebaseCredentialCheck = lakebaseCredentialCheck;
|
|
1132
1482
|
exports.lakebaseRoundTripCheck = lakebaseRoundTripCheck;
|
|
@@ -1135,14 +1485,21 @@ exports.normalizeVolumeRoot = normalizeVolumeRoot;
|
|
|
1135
1485
|
exports.notebookSubmitCheck = notebookSubmitCheck;
|
|
1136
1486
|
exports.parseFixtureColumns = parseFixtureColumns;
|
|
1137
1487
|
exports.parseStatementRows = parseStatementRows;
|
|
1488
|
+
exports.performanceBudgetCheck = performanceBudgetCheck;
|
|
1138
1489
|
exports.pipelineRefreshCheck = pipelineRefreshCheck;
|
|
1139
1490
|
exports.renderJUnit = renderJUnit;
|
|
1140
1491
|
exports.resolveFixtureRows = resolveFixtureRows;
|
|
1492
|
+
exports.resolveLakebaseProject = resolveLakebaseProject;
|
|
1141
1493
|
exports.resolveProfile = resolveProfile;
|
|
1142
1494
|
exports.resolveTokenProvider = resolveTokenProvider;
|
|
1495
|
+
exports.restrictedPrincipalCheck = restrictedPrincipalCheck;
|
|
1496
|
+
exports.rollbackCheck = rollbackCheck;
|
|
1143
1497
|
exports.runLiveSuite = runLiveSuite;
|
|
1498
|
+
exports.secretRotationCheck = secretRotationCheck;
|
|
1144
1499
|
exports.secretScopeCheck = secretScopeCheck;
|
|
1500
|
+
exports.serverlessComputeCheck = serverlessComputeCheck;
|
|
1145
1501
|
exports.sqlRoundTripCheck = sqlRoundTripCheck;
|
|
1502
|
+
exports.toLakebaseBranchId = toLakebaseBranchId;
|
|
1146
1503
|
exports.toNamedParameters = toNamedParameters;
|
|
1147
1504
|
exports.unityCatalogAccessCheck = unityCatalogAccessCheck;
|
|
1148
1505
|
exports.uploadVolumeFile = uploadVolumeFile;
|