@fabricorg/databricks-testkit 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -2
- package/dist/index.cjs +1187 -77
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +118 -1
- package/dist/index.d.ts +118 -1
- package/dist/index.js +1164 -78
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -142,15 +142,15 @@ function defineLiveSuite(spec) {
|
|
|
142
142
|
async function runLiveSuite(spec, runtime = {}) {
|
|
143
143
|
const env = spec.env ?? process.env;
|
|
144
144
|
if (env.DBX_TEST_LIVE !== "1") return void 0;
|
|
145
|
-
const
|
|
145
|
+
const required3 = [...spec.required ?? []];
|
|
146
146
|
const known = new Set(spec.checks.map((check) => check.id));
|
|
147
|
-
for (const id of
|
|
147
|
+
for (const id of required3) {
|
|
148
148
|
if (!known.has(id)) throw new Error(`Required live check '${id}' is not defined`);
|
|
149
149
|
}
|
|
150
150
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
151
151
|
const ownedWarehouse = runtime.warehouse === void 0;
|
|
152
152
|
const client = runtime.client ?? createClientFromEnv(env);
|
|
153
|
-
const
|
|
153
|
+
const warehouse2 = runtime.warehouse ?? (hasWarehouseConfig(env) ? await createWorkspaceContext({ env, client }) : unavailableWarehouseContext());
|
|
154
154
|
const results = [];
|
|
155
155
|
try {
|
|
156
156
|
for (const check of spec.checks) {
|
|
@@ -165,7 +165,7 @@ async function runLiveSuite(spec, runtime = {}) {
|
|
|
165
165
|
}
|
|
166
166
|
const before = performance.now();
|
|
167
167
|
try {
|
|
168
|
-
const details = await check.run({ env, warehouse, client });
|
|
168
|
+
const details = await check.run({ env, warehouse: warehouse2, client });
|
|
169
169
|
results.push({
|
|
170
170
|
id: check.id,
|
|
171
171
|
description: check.description,
|
|
@@ -184,9 +184,9 @@ async function runLiveSuite(spec, runtime = {}) {
|
|
|
184
184
|
}
|
|
185
185
|
}
|
|
186
186
|
} finally {
|
|
187
|
-
if (ownedWarehouse) await
|
|
187
|
+
if (ownedWarehouse) await warehouse2.close();
|
|
188
188
|
}
|
|
189
|
-
const success =
|
|
189
|
+
const success = required3.every(
|
|
190
190
|
(id) => results.find((result) => result.id === id)?.status === "pass"
|
|
191
191
|
);
|
|
192
192
|
const evidence = {
|
|
@@ -195,7 +195,7 @@ async function runLiveSuite(spec, runtime = {}) {
|
|
|
195
195
|
startedAt,
|
|
196
196
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
197
197
|
success,
|
|
198
|
-
required:
|
|
198
|
+
required: required3,
|
|
199
199
|
results,
|
|
200
200
|
...spec.targetPack ? { targetPack: spec.targetPack } : {}
|
|
201
201
|
};
|
|
@@ -298,8 +298,8 @@ function sqlRoundTripCheck() {
|
|
|
298
298
|
id: "sql",
|
|
299
299
|
description: "SQL Statement Execution round-trip",
|
|
300
300
|
configured: (env) => Boolean(env.DATABRICKS_WAREHOUSE_ID || env.DATABRICKS_HTTP_PATH),
|
|
301
|
-
async run({ warehouse }) {
|
|
302
|
-
const rows = await
|
|
301
|
+
async run({ warehouse: warehouse2 }) {
|
|
302
|
+
const rows = await warehouse2.query("SELECT 1 AS dbx_test_ok");
|
|
303
303
|
if (Number(rows[0]?.dbx_test_ok) !== 1)
|
|
304
304
|
throw new Error("SQL round-trip returned an unexpected value");
|
|
305
305
|
return { rows: rows.length };
|
|
@@ -343,9 +343,9 @@ function deltaContractCheck(tableEnv = "DBX_TEST_DELTA_TABLE", columns = []) {
|
|
|
343
343
|
id: "delta-contract",
|
|
344
344
|
description: "Delta table schema contract",
|
|
345
345
|
configured: (env) => Boolean(env[tableEnv]),
|
|
346
|
-
async run({ env, warehouse }) {
|
|
346
|
+
async run({ env, warehouse: warehouse2 }) {
|
|
347
347
|
const table = env[tableEnv];
|
|
348
|
-
const rows = await
|
|
348
|
+
const rows = await warehouse2.query(`DESCRIBE TABLE ${table}`);
|
|
349
349
|
assertDeltaContract(table, rows, columns);
|
|
350
350
|
return { table, columns: rows.length };
|
|
351
351
|
}
|
|
@@ -356,10 +356,10 @@ function unityCatalogAccessCheck() {
|
|
|
356
356
|
id: "uc-grants",
|
|
357
357
|
description: "Unity Catalog allow/deny access assertions",
|
|
358
358
|
configured: (env) => Boolean(env.DBX_TEST_UC_ALLOWED_TABLE && env.DBX_TEST_UC_DENIED_TABLE),
|
|
359
|
-
async run({ env, warehouse }) {
|
|
360
|
-
await
|
|
359
|
+
async run({ env, warehouse: warehouse2 }) {
|
|
360
|
+
await warehouse2.query(`SELECT * FROM ${env.DBX_TEST_UC_ALLOWED_TABLE} LIMIT 1`);
|
|
361
361
|
try {
|
|
362
|
-
await
|
|
362
|
+
await warehouse2.query(`SELECT * FROM ${env.DBX_TEST_UC_DENIED_TABLE} LIMIT 1`);
|
|
363
363
|
} catch (error) {
|
|
364
364
|
if (/permission.?denied|insufficient.?priv|403|unauthoriz/i.test(String(error))) {
|
|
365
365
|
return { allowed: env.DBX_TEST_UC_ALLOWED_TABLE, denied: env.DBX_TEST_UC_DENIED_TABLE };
|
|
@@ -819,7 +819,7 @@ function autoLoaderIngestionCheck(options = {}) {
|
|
|
819
819
|
configured: (env) => Boolean(
|
|
820
820
|
env.DBX_TEST_AUTOLOADER_PIPELINE_ID && env.DBX_TEST_AUTOLOADER_TABLE && env.DBX_TEST_AUTOLOADER_FIXTURE && env.DBX_TEST_VOLUME
|
|
821
821
|
),
|
|
822
|
-
async run({ env, client, warehouse }) {
|
|
822
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
823
823
|
const pipelineId = env.DBX_TEST_AUTOLOADER_PIPELINE_ID;
|
|
824
824
|
if (pipelineId === env.DBX_TEST_PIPELINE_ID) {
|
|
825
825
|
throw new Error("DBX_TEST_AUTOLOADER_PIPELINE_ID must not be the production pipeline");
|
|
@@ -844,7 +844,7 @@ function autoLoaderIngestionCheck(options = {}) {
|
|
|
844
844
|
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
845
845
|
});
|
|
846
846
|
const table = env.DBX_TEST_AUTOLOADER_TABLE;
|
|
847
|
-
const rows = await
|
|
847
|
+
const rows = await warehouse2.query(
|
|
848
848
|
`SELECT COUNT(*) AS n, SUM(CASE WHEN _rescued_data IS NOT NULL THEN 1 ELSE 0 END) AS rescued FROM ${table}`
|
|
849
849
|
);
|
|
850
850
|
const count = Number(rows[0]?.n ?? 0);
|
|
@@ -867,7 +867,7 @@ function performanceBudgetCheck() {
|
|
|
867
867
|
id: "performance",
|
|
868
868
|
description: "SQL Statement Execution p95 stays within the configured latency budget",
|
|
869
869
|
configured: (env) => Boolean(env.DBX_TEST_PERFORMANCE_P95_MS),
|
|
870
|
-
async run({ env, warehouse }) {
|
|
870
|
+
async run({ env, warehouse: warehouse2 }) {
|
|
871
871
|
const budgetMs = Number(env.DBX_TEST_PERFORMANCE_P95_MS);
|
|
872
872
|
const runs = Number(env.DBX_TEST_PERFORMANCE_RUNS ?? 7);
|
|
873
873
|
if (!Number.isFinite(budgetMs) || budgetMs <= 0) {
|
|
@@ -876,11 +876,11 @@ function performanceBudgetCheck() {
|
|
|
876
876
|
if (!Number.isInteger(runs) || runs < 3 || runs > 100) {
|
|
877
877
|
throw new Error("DBX_TEST_PERFORMANCE_RUNS must be an integer from 3 to 100");
|
|
878
878
|
}
|
|
879
|
-
await
|
|
879
|
+
await warehouse2.query("SELECT 1 AS warmup");
|
|
880
880
|
const durations = [];
|
|
881
881
|
for (let index = 0; index < runs; index += 1) {
|
|
882
882
|
const started = performance.now();
|
|
883
|
-
await
|
|
883
|
+
await warehouse2.query("SELECT 1 AS performance_probe");
|
|
884
884
|
durations.push(performance.now() - started);
|
|
885
885
|
}
|
|
886
886
|
durations.sort((left, right) => left - right);
|
|
@@ -902,15 +902,15 @@ function failureRecoveryCheck() {
|
|
|
902
902
|
id: "failure-recovery",
|
|
903
903
|
description: "A failed statement is isolated and a subsequent statement recovers",
|
|
904
904
|
configured: (env) => Boolean(env.DATABRICKS_WAREHOUSE_ID || env.DATABRICKS_HTTP_PATH),
|
|
905
|
-
async run({ warehouse }) {
|
|
905
|
+
async run({ warehouse: warehouse2 }) {
|
|
906
906
|
let failedAsExpected = false;
|
|
907
907
|
try {
|
|
908
|
-
await
|
|
908
|
+
await warehouse2.query("SELECT * FROM __fabric_experiments_expected_missing_table__");
|
|
909
909
|
} catch {
|
|
910
910
|
failedAsExpected = true;
|
|
911
911
|
}
|
|
912
912
|
if (!failedAsExpected) throw new Error("Injected failure unexpectedly succeeded");
|
|
913
|
-
const rows = await
|
|
913
|
+
const rows = await warehouse2.query("SELECT 1 AS recovered");
|
|
914
914
|
if (Number(rows[0]?.recovered) !== 1) {
|
|
915
915
|
throw new Error("Statement execution did not recover after the injected failure");
|
|
916
916
|
}
|
|
@@ -948,20 +948,20 @@ function backupRestoreCheck() {
|
|
|
948
948
|
id: "backup-restore",
|
|
949
949
|
description: "Delta DEEP CLONE backup restores a dropped scratch table",
|
|
950
950
|
configured: (env) => Boolean(env.DATABRICKS_WAREHOUSE_ID || env.DATABRICKS_HTTP_PATH),
|
|
951
|
-
async run({ warehouse }) {
|
|
952
|
-
const source =
|
|
953
|
-
const backup =
|
|
951
|
+
async run({ warehouse: warehouse2 }) {
|
|
952
|
+
const source = warehouse2.qual(safeName("backup_source"));
|
|
953
|
+
const backup = warehouse2.qual(safeName("backup_clone"));
|
|
954
954
|
try {
|
|
955
|
-
await
|
|
956
|
-
await
|
|
957
|
-
await
|
|
958
|
-
await
|
|
959
|
-
const rows = await
|
|
955
|
+
await warehouse2.exec(`CREATE TABLE ${source} USING DELTA AS SELECT 7 AS value`);
|
|
956
|
+
await warehouse2.exec(`CREATE TABLE ${backup} DEEP CLONE ${source}`);
|
|
957
|
+
await warehouse2.exec(`DROP TABLE ${source}`);
|
|
958
|
+
await warehouse2.exec(`CREATE TABLE ${source} DEEP CLONE ${backup}`);
|
|
959
|
+
const rows = await warehouse2.query(`SELECT value FROM ${source}`);
|
|
960
960
|
if (Number(rows[0]?.value) !== 7) throw new Error("Restored Delta table lost its data");
|
|
961
961
|
return { restoredRows: rows.length, deepClone: true };
|
|
962
962
|
} finally {
|
|
963
|
-
await
|
|
964
|
-
await
|
|
963
|
+
await warehouse2.exec(`DROP TABLE IF EXISTS ${source}`).catch(() => void 0);
|
|
964
|
+
await warehouse2.exec(`DROP TABLE IF EXISTS ${backup}`).catch(() => void 0);
|
|
965
965
|
}
|
|
966
966
|
}
|
|
967
967
|
};
|
|
@@ -971,25 +971,25 @@ function rollbackCheck() {
|
|
|
971
971
|
id: "rollback",
|
|
972
972
|
description: "Delta time-travel rollback restores a known-good table version",
|
|
973
973
|
configured: (env) => Boolean(env.DATABRICKS_WAREHOUSE_ID || env.DATABRICKS_HTTP_PATH),
|
|
974
|
-
async run({ warehouse }) {
|
|
975
|
-
const table =
|
|
974
|
+
async run({ warehouse: warehouse2 }) {
|
|
975
|
+
const table = warehouse2.qual(safeName("rollback_probe"));
|
|
976
976
|
try {
|
|
977
|
-
await
|
|
978
|
-
await
|
|
979
|
-
const history = await
|
|
977
|
+
await warehouse2.exec(`CREATE TABLE ${table} (value INT) USING DELTA`);
|
|
978
|
+
await warehouse2.exec(`INSERT INTO ${table} VALUES (1)`);
|
|
979
|
+
const history = await warehouse2.query(`DESCRIBE HISTORY ${table} LIMIT 1`);
|
|
980
980
|
const knownGoodVersion = Number(history[0]?.version);
|
|
981
981
|
if (!Number.isInteger(knownGoodVersion)) {
|
|
982
982
|
throw new Error("Delta history did not return a known-good version");
|
|
983
983
|
}
|
|
984
|
-
await
|
|
985
|
-
await
|
|
986
|
-
const rows = await
|
|
984
|
+
await warehouse2.exec(`INSERT INTO ${table} VALUES (999)`);
|
|
985
|
+
await warehouse2.exec(`RESTORE TABLE ${table} TO VERSION AS OF ${knownGoodVersion}`);
|
|
986
|
+
const rows = await warehouse2.query(`SELECT value FROM ${table} ORDER BY value`);
|
|
987
987
|
if (rows.length !== 1 || Number(rows[0]?.value) !== 1) {
|
|
988
988
|
throw new Error("Delta rollback did not restore the known-good contents");
|
|
989
989
|
}
|
|
990
990
|
return { knownGoodVersion, restoredRows: rows.length };
|
|
991
991
|
} finally {
|
|
992
|
-
await
|
|
992
|
+
await warehouse2.exec(`DROP TABLE IF EXISTS ${table}`).catch(() => void 0);
|
|
993
993
|
}
|
|
994
994
|
}
|
|
995
995
|
};
|
|
@@ -999,13 +999,13 @@ function serverlessComputeCheck() {
|
|
|
999
999
|
id: "azure-serverless",
|
|
1000
1000
|
description: "Azure Databricks serverless SQL warehouse is enabled and executes SQL",
|
|
1001
1001
|
configured: (env) => Boolean(env.DATABRICKS_WAREHOUSE_ID),
|
|
1002
|
-
async run({ env, client, warehouse }) {
|
|
1002
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
1003
1003
|
const id = env.DATABRICKS_WAREHOUSE_ID;
|
|
1004
1004
|
const details = await client.get(`/api/2.0/sql/warehouses/${encodeURIComponent(id)}`);
|
|
1005
1005
|
if (details.enable_serverless_compute !== true) {
|
|
1006
1006
|
throw new Error(`SQL warehouse ${id} is not configured for serverless compute`);
|
|
1007
1007
|
}
|
|
1008
|
-
const rows = await
|
|
1008
|
+
const rows = await warehouse2.query("SELECT current_catalog() AS catalog");
|
|
1009
1009
|
return { warehouseId: id, name: details.name, state: details.state, rows: rows.length };
|
|
1010
1010
|
}
|
|
1011
1011
|
};
|
|
@@ -1100,9 +1100,9 @@ function defineTargetPack(spec) {
|
|
|
1100
1100
|
}
|
|
1101
1101
|
ids.add(check.id);
|
|
1102
1102
|
}
|
|
1103
|
-
for (const
|
|
1104
|
-
if (!ids.has(
|
|
1105
|
-
throw new Error(`Target pack '${spec.id}' requires unknown check '${
|
|
1103
|
+
for (const required3 of spec.requiredChecks ?? []) {
|
|
1104
|
+
if (!ids.has(required3)) {
|
|
1105
|
+
throw new Error(`Target pack '${spec.id}' requires unknown check '${required3}'`);
|
|
1106
1106
|
}
|
|
1107
1107
|
}
|
|
1108
1108
|
return checks;
|
|
@@ -1126,25 +1126,25 @@ function createTargetPackRegistry(packs) {
|
|
|
1126
1126
|
return registry;
|
|
1127
1127
|
}
|
|
1128
1128
|
function doctorTargetPack(pack, env = process.env, requiredChecks = pack.requiredChecks ?? []) {
|
|
1129
|
-
const requirements = (pack.requirements ?? []).map((
|
|
1130
|
-
id:
|
|
1131
|
-
description:
|
|
1132
|
-
satisfied:
|
|
1133
|
-
required:
|
|
1134
|
-
variables: [...
|
|
1129
|
+
const requirements = (pack.requirements ?? []).map((requirement2) => ({
|
|
1130
|
+
id: requirement2.id,
|
|
1131
|
+
description: requirement2.description,
|
|
1132
|
+
satisfied: requirement2.anyOf.some((name) => Boolean(env[name]?.trim())),
|
|
1133
|
+
required: requirement2.required !== false,
|
|
1134
|
+
variables: [...requirement2.anyOf]
|
|
1135
1135
|
}));
|
|
1136
|
-
const
|
|
1136
|
+
const required3 = new Set(requiredChecks);
|
|
1137
1137
|
const checks = pack.checks().map((check) => ({
|
|
1138
1138
|
id: check.id,
|
|
1139
1139
|
description: check.description,
|
|
1140
1140
|
configured: check.configured ? check.configured(env) : true,
|
|
1141
|
-
required:
|
|
1141
|
+
required: required3.has(check.id)
|
|
1142
1142
|
}));
|
|
1143
1143
|
const known = new Set(checks.map((check) => check.id));
|
|
1144
|
-
const missingRequirements = requirements.filter((
|
|
1144
|
+
const missingRequirements = requirements.filter((requirement2) => requirement2.required && !requirement2.satisfied).map((requirement2) => requirement2.id);
|
|
1145
1145
|
const missingRequiredChecks = [
|
|
1146
1146
|
...checks.filter((check) => check.required && !check.configured).map((check) => check.id),
|
|
1147
|
-
...[...
|
|
1147
|
+
...[...required3].filter((id) => !known.has(id))
|
|
1148
1148
|
];
|
|
1149
1149
|
return {
|
|
1150
1150
|
pack: {
|
|
@@ -1259,7 +1259,7 @@ function createSqlDeltaFoundationPack() {
|
|
|
1259
1259
|
],
|
|
1260
1260
|
requiredChecks: ["sql"],
|
|
1261
1261
|
requirements: [workspaceRequirement, authRequirement, warehouseRequirement],
|
|
1262
|
-
docsUrl: `${DOCS}#
|
|
1262
|
+
docsUrl: `${DOCS}#sql-and-delta-foundation`,
|
|
1263
1263
|
certificationScopes: [AZURE_SCOPE]
|
|
1264
1264
|
});
|
|
1265
1265
|
}
|
|
@@ -1284,7 +1284,7 @@ function createLakeflowFoundationPack() {
|
|
|
1284
1284
|
],
|
|
1285
1285
|
requiredChecks: ["pipeline"],
|
|
1286
1286
|
requirements: [workspaceRequirement, authRequirement],
|
|
1287
|
-
docsUrl: `${DOCS}#
|
|
1287
|
+
docsUrl: `${DOCS}#lakeflow-ingestion-foundation`,
|
|
1288
1288
|
certificationScopes: [AZURE_SCOPE]
|
|
1289
1289
|
});
|
|
1290
1290
|
}
|
|
@@ -1312,7 +1312,7 @@ function createJobsCodeFoundationPack() {
|
|
|
1312
1312
|
],
|
|
1313
1313
|
requiredChecks: ["jobs"],
|
|
1314
1314
|
requirements: [workspaceRequirement, authRequirement],
|
|
1315
|
-
docsUrl: `${DOCS}#
|
|
1315
|
+
docsUrl: `${DOCS}#jobs-and-code-foundation`,
|
|
1316
1316
|
certificationScopes: [AZURE_SCOPE]
|
|
1317
1317
|
});
|
|
1318
1318
|
}
|
|
@@ -1338,7 +1338,7 @@ function createUnityStorageFoundationPack() {
|
|
|
1338
1338
|
],
|
|
1339
1339
|
requiredChecks: ["uc-grants"],
|
|
1340
1340
|
requirements: [workspaceRequirement, authRequirement, warehouseRequirement],
|
|
1341
|
-
docsUrl: `${DOCS}#
|
|
1341
|
+
docsUrl: `${DOCS}#unity-catalog-and-storage-foundation`,
|
|
1342
1342
|
certificationScopes: [AZURE_SCOPE]
|
|
1343
1343
|
});
|
|
1344
1344
|
}
|
|
@@ -1364,7 +1364,7 @@ function createAppsOperationalFoundationPack() {
|
|
|
1364
1364
|
],
|
|
1365
1365
|
requiredChecks: ["app"],
|
|
1366
1366
|
requirements: [workspaceRequirement, authRequirement],
|
|
1367
|
-
docsUrl: `${DOCS}#
|
|
1367
|
+
docsUrl: `${DOCS}#apps-and-operational-data-foundation`,
|
|
1368
1368
|
certificationScopes: [AZURE_SCOPE]
|
|
1369
1369
|
});
|
|
1370
1370
|
}
|
|
@@ -1395,7 +1395,8 @@ var DatabricksJobsAdapter = class {
|
|
|
1395
1395
|
async get(runId) {
|
|
1396
1396
|
const raw = await this.client.get("/api/2.1/jobs/runs/get", {
|
|
1397
1397
|
run_id: runId,
|
|
1398
|
-
include_history: true
|
|
1398
|
+
include_history: true,
|
|
1399
|
+
include_resolved_values: true
|
|
1399
1400
|
});
|
|
1400
1401
|
return normalizeJobRun(runId, raw);
|
|
1401
1402
|
}
|
|
@@ -1409,15 +1410,46 @@ var DatabricksJobsAdapter = class {
|
|
|
1409
1410
|
async cancel(runId) {
|
|
1410
1411
|
await this.client.post("/api/2.1/jobs/runs/cancel", { run_id: runId });
|
|
1411
1412
|
}
|
|
1413
|
+
async waitUntilActive(runId, options = {}) {
|
|
1414
|
+
const timeoutMs = options.timeoutMs ?? 3e5;
|
|
1415
|
+
const pollIntervalMs = options.pollIntervalMs ?? 2e3;
|
|
1416
|
+
const deadline = Date.now() + timeoutMs;
|
|
1417
|
+
while (Date.now() < deadline) {
|
|
1418
|
+
const run = await this.get(runId);
|
|
1419
|
+
if (run.lifeCycleState === "RUNNING") return run;
|
|
1420
|
+
if (isTerminal(run.lifeCycleState)) {
|
|
1421
|
+
throw new Error(
|
|
1422
|
+
`Databricks run ${runId} reached ${run.lifeCycleState}/${run.resultState} before it could be canceled`
|
|
1423
|
+
);
|
|
1424
|
+
}
|
|
1425
|
+
await delay2(pollIntervalMs);
|
|
1426
|
+
}
|
|
1427
|
+
throw new Error(`Databricks run ${runId} did not become active within ${timeoutMs}ms`);
|
|
1428
|
+
}
|
|
1412
1429
|
async repair(runId, options = {}) {
|
|
1413
1430
|
const response = await this.client.post("/api/2.1/jobs/runs/repair", {
|
|
1414
1431
|
run_id: runId,
|
|
1415
1432
|
...options.latestRepairId ? { latest_repair_id: options.latestRepairId } : {},
|
|
1433
|
+
...options.rerunDependentTasks ? { rerun_dependent_tasks: true } : {},
|
|
1416
1434
|
...options.rerunTasks?.length ? { rerun_tasks: options.rerunTasks } : { rerun_all_failed_tasks: true }
|
|
1417
1435
|
});
|
|
1418
1436
|
if (!response.repair_id) throw new Error("Databricks Jobs repair returned no repair_id");
|
|
1419
1437
|
return response.repair_id;
|
|
1420
1438
|
}
|
|
1439
|
+
async waitForRepair(runId, repairId, options = {}) {
|
|
1440
|
+
const timeoutMs = options.timeoutMs ?? 9e5;
|
|
1441
|
+
const pollIntervalMs = options.pollIntervalMs ?? 5e3;
|
|
1442
|
+
const deadline = Date.now() + timeoutMs;
|
|
1443
|
+
while (Date.now() < deadline) {
|
|
1444
|
+
const run = await this.get(runId);
|
|
1445
|
+
const repair = run.repairs.find((candidate) => candidate.repairId === repairId);
|
|
1446
|
+
if (repair && isTerminal(repair.lifeCycleState)) return run;
|
|
1447
|
+
await delay2(pollIntervalMs);
|
|
1448
|
+
}
|
|
1449
|
+
throw new Error(
|
|
1450
|
+
`Databricks repair ${repairId} for run ${runId} timed out after ${timeoutMs}ms`
|
|
1451
|
+
);
|
|
1452
|
+
}
|
|
1421
1453
|
};
|
|
1422
1454
|
function validateJobGraph(tasks) {
|
|
1423
1455
|
if (tasks.length === 0) throw new Error("Jobs orchestration requires at least one task");
|
|
@@ -1463,16 +1495,20 @@ function jobsOrchestrationCheck() {
|
|
|
1463
1495
|
const configured = env.DBX_TEST_JOBS_ORCHESTRATION_JSON;
|
|
1464
1496
|
if (!configured) throw new Error("DBX_TEST_JOBS_ORCHESTRATION_JSON is required");
|
|
1465
1497
|
const spec = parseJobOrchestrationLiveSpec(configured);
|
|
1466
|
-
const
|
|
1467
|
-
const runId = await
|
|
1468
|
-
let run = await
|
|
1498
|
+
const adapter2 = new DatabricksJobsAdapter(client);
|
|
1499
|
+
const runId = await adapter2.submit(spec.request);
|
|
1500
|
+
let run = await adapter2.wait(runId, {
|
|
1469
1501
|
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
1470
1502
|
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
1471
1503
|
});
|
|
1504
|
+
let repairId;
|
|
1472
1505
|
if (spec.repairFailedTasks && run.resultState !== "SUCCESS") {
|
|
1473
1506
|
const failed = run.tasks.filter((task) => task.resultState === "FAILED").map((task) => task.taskKey);
|
|
1474
|
-
await
|
|
1475
|
-
|
|
1507
|
+
repairId = await adapter2.repair(runId, {
|
|
1508
|
+
rerunTasks: failed.length > 0 ? failed : void 0,
|
|
1509
|
+
rerunDependentTasks: true
|
|
1510
|
+
});
|
|
1511
|
+
run = await adapter2.waitForRepair(runId, repairId, {
|
|
1476
1512
|
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
1477
1513
|
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
1478
1514
|
});
|
|
@@ -1490,10 +1526,32 @@ function jobsOrchestrationCheck() {
|
|
|
1490
1526
|
throw new Error(`Task '${taskKey}' ended in ${task.resultState}, expected ${expected}`);
|
|
1491
1527
|
}
|
|
1492
1528
|
}
|
|
1529
|
+
let cancellation;
|
|
1530
|
+
if (spec.cancellation) {
|
|
1531
|
+
const cancellationRunId = await adapter2.submit(spec.cancellation.request);
|
|
1532
|
+
await adapter2.waitUntilActive(cancellationRunId, {
|
|
1533
|
+
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
1534
|
+
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
1535
|
+
});
|
|
1536
|
+
await adapter2.cancel(cancellationRunId);
|
|
1537
|
+
const canceled = await adapter2.wait(cancellationRunId, {
|
|
1538
|
+
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
1539
|
+
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
1540
|
+
});
|
|
1541
|
+
const expectedCancellation = spec.cancellation.expectedResult ?? "CANCELED";
|
|
1542
|
+
if (canceled.resultState !== expectedCancellation) {
|
|
1543
|
+
throw new Error(
|
|
1544
|
+
`Cancellation run ${cancellationRunId} ended in ${canceled.resultState}, expected ${expectedCancellation}`
|
|
1545
|
+
);
|
|
1546
|
+
}
|
|
1547
|
+
cancellation = { runId: cancellationRunId, resultState: canceled.resultState };
|
|
1548
|
+
}
|
|
1493
1549
|
return {
|
|
1494
1550
|
runId,
|
|
1551
|
+
...repairId ? { repairId } : {},
|
|
1495
1552
|
resultState: run.resultState,
|
|
1496
|
-
tasks: run.tasks
|
|
1553
|
+
tasks: run.tasks,
|
|
1554
|
+
...cancellation ? { cancellation } : {}
|
|
1497
1555
|
};
|
|
1498
1556
|
}
|
|
1499
1557
|
};
|
|
@@ -1513,11 +1571,24 @@ function parseJobOrchestrationLiveSpec(value) {
|
|
|
1513
1571
|
throw new Error("Jobs orchestration spec requires request.tasks");
|
|
1514
1572
|
}
|
|
1515
1573
|
validateJobGraph(spec.request.tasks);
|
|
1574
|
+
if (spec.cancellation) {
|
|
1575
|
+
if (typeof spec.cancellation !== "object" || !spec.cancellation.request || !Array.isArray(spec.cancellation.request.tasks)) {
|
|
1576
|
+
throw new Error("Jobs orchestration cancellation requires request.tasks");
|
|
1577
|
+
}
|
|
1578
|
+
validateJobGraph(spec.cancellation.request.tasks);
|
|
1579
|
+
}
|
|
1516
1580
|
return spec;
|
|
1517
1581
|
}
|
|
1518
1582
|
function normalizeJobRun(fallbackRunId, raw) {
|
|
1519
1583
|
const state = objectOf(raw.state);
|
|
1520
|
-
const
|
|
1584
|
+
const repairEntries = Array.isArray(raw.repair_history) ? raw.repair_history.filter(
|
|
1585
|
+
(repair) => Boolean(repair && typeof repair === "object")
|
|
1586
|
+
) : [];
|
|
1587
|
+
const taskRunRanks = /* @__PURE__ */ new Map();
|
|
1588
|
+
for (const [rank, repair] of repairEntries.entries()) {
|
|
1589
|
+
for (const runId of numberArray(repair.task_run_ids)) taskRunRanks.set(runId, rank);
|
|
1590
|
+
}
|
|
1591
|
+
const taskAttempts = Array.isArray(raw.tasks) ? raw.tasks.filter(
|
|
1521
1592
|
(task) => Boolean(task && typeof task === "object")
|
|
1522
1593
|
).map((task) => {
|
|
1523
1594
|
const taskState = objectOf(task.state);
|
|
@@ -1526,20 +1597,236 @@ function normalizeJobRun(fallbackRunId, raw) {
|
|
|
1526
1597
|
...Number.isFinite(Number(task.run_id)) ? { runId: Number(task.run_id) } : {},
|
|
1527
1598
|
lifeCycleState: String(taskState.life_cycle_state ?? "UNKNOWN"),
|
|
1528
1599
|
resultState: String(taskState.result_state ?? "UNKNOWN"),
|
|
1529
|
-
...typeof taskState.state_message === "string" ? { stateMessage: taskState.state_message } : {}
|
|
1600
|
+
...typeof taskState.state_message === "string" ? { stateMessage: taskState.state_message } : {},
|
|
1601
|
+
...Number.isFinite(Number(task.attempt_number)) ? { attemptNumber: Number(task.attempt_number) } : {}
|
|
1530
1602
|
};
|
|
1531
1603
|
}) : [];
|
|
1604
|
+
const latestTasks = /* @__PURE__ */ new Map();
|
|
1605
|
+
for (const task of taskAttempts) {
|
|
1606
|
+
const current = latestTasks.get(task.taskKey);
|
|
1607
|
+
const currentAttempt = current?.attemptNumber ?? -1;
|
|
1608
|
+
const candidateAttempt = task.attemptNumber ?? -1;
|
|
1609
|
+
const currentRank = current?.runId ? taskRunRanks.get(current.runId) ?? -1 : -1;
|
|
1610
|
+
const candidateRank = task.runId ? taskRunRanks.get(task.runId) ?? -1 : -1;
|
|
1611
|
+
if (!current || candidateAttempt > currentAttempt || candidateAttempt === currentAttempt && candidateRank >= currentRank) {
|
|
1612
|
+
latestTasks.set(task.taskKey, task);
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
const tasks = [...latestTasks.values()];
|
|
1616
|
+
const repairs = repairEntries.map((repair) => {
|
|
1617
|
+
const repairState = objectOf(repair.state);
|
|
1618
|
+
return {
|
|
1619
|
+
repairId: Number(repair.id ?? repair.repair_id ?? 0),
|
|
1620
|
+
type: String(repair.type ?? "UNKNOWN"),
|
|
1621
|
+
lifeCycleState: String(repairState.life_cycle_state ?? "UNKNOWN"),
|
|
1622
|
+
resultState: String(repairState.result_state ?? "UNKNOWN"),
|
|
1623
|
+
taskRunIds: numberArray(repair.task_run_ids)
|
|
1624
|
+
};
|
|
1625
|
+
});
|
|
1532
1626
|
return {
|
|
1533
1627
|
runId: Number(raw.run_id ?? fallbackRunId),
|
|
1534
1628
|
lifeCycleState: String(state.life_cycle_state ?? "UNKNOWN"),
|
|
1535
1629
|
resultState: String(state.result_state ?? "UNKNOWN"),
|
|
1536
1630
|
tasks,
|
|
1631
|
+
repairs,
|
|
1537
1632
|
raw
|
|
1538
1633
|
};
|
|
1539
1634
|
}
|
|
1635
|
+
function isTerminal(state) {
|
|
1636
|
+
return ["TERMINATED", "SKIPPED", "INTERNAL_ERROR", "BLOCKED"].includes(state);
|
|
1637
|
+
}
|
|
1638
|
+
function delay2(ms) {
|
|
1639
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
1640
|
+
}
|
|
1540
1641
|
function objectOf(value) {
|
|
1541
1642
|
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
1542
1643
|
}
|
|
1644
|
+
function numberArray(value) {
|
|
1645
|
+
return Array.isArray(value) ? value.map(Number).filter((candidate) => Number.isFinite(candidate)) : [];
|
|
1646
|
+
}
|
|
1647
|
+
|
|
1648
|
+
// src/jobs-certification-check.ts
|
|
1649
|
+
var NOTEBOOKS = {
|
|
1650
|
+
seed: `# Databricks notebook source
|
|
1651
|
+
dbutils.jobs.taskValues.set(key="branch", value="run")
|
|
1652
|
+
dbutils.jobs.taskValues.set(key="items", value=["alpha", "beta"])
|
|
1653
|
+
dbutils.notebook.exit("seeded")
|
|
1654
|
+
`,
|
|
1655
|
+
process: `# Databricks notebook source
|
|
1656
|
+
dbutils.widgets.text("item", "")
|
|
1657
|
+
item = dbutils.widgets.get("item")
|
|
1658
|
+
assert item, "item parameter is required"
|
|
1659
|
+
dbutils.notebook.exit(item)
|
|
1660
|
+
`,
|
|
1661
|
+
retry: `# Databricks notebook source
|
|
1662
|
+
dbutils.widgets.text("execution_count", "0")
|
|
1663
|
+
execution_count = int(dbutils.widgets.get("execution_count"))
|
|
1664
|
+
if execution_count <= 1:
|
|
1665
|
+
raise RuntimeError("intentional retryable failure")
|
|
1666
|
+
dbutils.notebook.exit("retried")
|
|
1667
|
+
`,
|
|
1668
|
+
repair: `# Databricks notebook source
|
|
1669
|
+
dbutils.widgets.text("repair_count", "0")
|
|
1670
|
+
repair_count = int(dbutils.widgets.get("repair_count"))
|
|
1671
|
+
if repair_count == 0:
|
|
1672
|
+
raise RuntimeError("intentional first-run failure")
|
|
1673
|
+
dbutils.notebook.exit("repaired")
|
|
1674
|
+
`,
|
|
1675
|
+
sleep: `# Databricks notebook source
|
|
1676
|
+
import time
|
|
1677
|
+
time.sleep(300)
|
|
1678
|
+
`
|
|
1679
|
+
};
|
|
1680
|
+
function jobsCertificationCheck() {
|
|
1681
|
+
return {
|
|
1682
|
+
id: "jobs-certification",
|
|
1683
|
+
description: "Disposable Lakeflow Jobs branch, loop, failure, repair, cancellation, and cleanup certification",
|
|
1684
|
+
configured: (env) => env.DBX_TEST_JOBS_CERTIFY === "1",
|
|
1685
|
+
async run({ env, client }) {
|
|
1686
|
+
const root = `${env.DBX_TEST_JOBS_FIXTURE_ROOT ?? "/Workspace/Shared/fabric-experiments-target-pack"}/${randomUUID()}`;
|
|
1687
|
+
const paths = Object.fromEntries(
|
|
1688
|
+
Object.keys(NOTEBOOKS).map((name) => [name, `${root}/${name}`])
|
|
1689
|
+
);
|
|
1690
|
+
try {
|
|
1691
|
+
await importNotebooks(client, paths);
|
|
1692
|
+
const adapter2 = new DatabricksJobsAdapter(client);
|
|
1693
|
+
const runId = await adapter2.submit(certificationRequest(paths));
|
|
1694
|
+
const initial = await adapter2.wait(runId, waitOptions(env));
|
|
1695
|
+
if (initial.resultState !== "FAILED") {
|
|
1696
|
+
throw new Error(
|
|
1697
|
+
`Certification run ${runId} expected FAILED before repair, got ${initial.resultState}`
|
|
1698
|
+
);
|
|
1699
|
+
}
|
|
1700
|
+
const retried = assertTask(initial.tasks, "retry", "SUCCESS");
|
|
1701
|
+
if ((retried.attemptNumber ?? 0) < 1) {
|
|
1702
|
+
throw new Error("Certification retry task did not record a second attempt");
|
|
1703
|
+
}
|
|
1704
|
+
assertTask(initial.tasks, "seed", "SUCCESS");
|
|
1705
|
+
assertTask(initial.tasks, "branch", "SUCCESS");
|
|
1706
|
+
assertTask(initial.tasks, "foreach", "SUCCESS");
|
|
1707
|
+
assertTask(initial.tasks, "repair", "FAILED");
|
|
1708
|
+
const repairId = await adapter2.repair(runId, {
|
|
1709
|
+
rerunTasks: ["repair"],
|
|
1710
|
+
rerunDependentTasks: true
|
|
1711
|
+
});
|
|
1712
|
+
const repaired = await adapter2.waitForRepair(runId, repairId, waitOptions(env));
|
|
1713
|
+
if (repaired.resultState !== "SUCCESS") {
|
|
1714
|
+
throw new Error(`Certification repair ${repairId} ended in ${repaired.resultState}`);
|
|
1715
|
+
}
|
|
1716
|
+
assertTask(repaired.tasks, "repair", "SUCCESS");
|
|
1717
|
+
const cancellationRunId = await adapter2.submit(cancellationRequest(paths));
|
|
1718
|
+
await adapter2.waitUntilActive(cancellationRunId, waitOptions(env));
|
|
1719
|
+
await adapter2.cancel(cancellationRunId);
|
|
1720
|
+
const canceled = await adapter2.wait(cancellationRunId, waitOptions(env));
|
|
1721
|
+
if (canceled.resultState !== "CANCELED") {
|
|
1722
|
+
throw new Error(`Certification cancellation ended in ${canceled.resultState}`);
|
|
1723
|
+
}
|
|
1724
|
+
return {
|
|
1725
|
+
runId,
|
|
1726
|
+
repairId,
|
|
1727
|
+
cancellationRunId,
|
|
1728
|
+
taskValues: true,
|
|
1729
|
+
retry: true,
|
|
1730
|
+
condition: true,
|
|
1731
|
+
forEach: true,
|
|
1732
|
+
repair: true,
|
|
1733
|
+
cancellation: true,
|
|
1734
|
+
fixtureRoot: root
|
|
1735
|
+
};
|
|
1736
|
+
} finally {
|
|
1737
|
+
await client.post("/api/2.0/workspace/delete", { path: root, recursive: true });
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
};
|
|
1741
|
+
}
|
|
1742
|
+
async function importNotebooks(client, paths) {
|
|
1743
|
+
await client.post("/api/2.0/workspace/mkdirs", {
|
|
1744
|
+
path: paths.seed.slice(0, paths.seed.lastIndexOf("/"))
|
|
1745
|
+
});
|
|
1746
|
+
for (const [name, source] of Object.entries(NOTEBOOKS)) {
|
|
1747
|
+
await client.post("/api/2.0/workspace/import", {
|
|
1748
|
+
path: paths[name],
|
|
1749
|
+
format: "SOURCE",
|
|
1750
|
+
language: "PYTHON",
|
|
1751
|
+
overwrite: true,
|
|
1752
|
+
content: Buffer.from(source).toString("base64")
|
|
1753
|
+
});
|
|
1754
|
+
}
|
|
1755
|
+
}
|
|
1756
|
+
function certificationRequest(paths) {
|
|
1757
|
+
return {
|
|
1758
|
+
run_name: "fabric-experiments-jobs-certification",
|
|
1759
|
+
tasks: [
|
|
1760
|
+
{
|
|
1761
|
+
task_key: "retry",
|
|
1762
|
+
notebook_task: {
|
|
1763
|
+
notebook_path: paths.retry,
|
|
1764
|
+
base_parameters: { execution_count: "{{task.execution_count}}" }
|
|
1765
|
+
},
|
|
1766
|
+
max_retries: 1
|
|
1767
|
+
},
|
|
1768
|
+
{
|
|
1769
|
+
task_key: "seed",
|
|
1770
|
+
depends_on: [{ task_key: "retry" }],
|
|
1771
|
+
notebook_task: { notebook_path: paths.seed }
|
|
1772
|
+
},
|
|
1773
|
+
{
|
|
1774
|
+
task_key: "branch",
|
|
1775
|
+
depends_on: [{ task_key: "seed" }],
|
|
1776
|
+
condition_task: {
|
|
1777
|
+
left: "{{tasks.seed.values.branch}}",
|
|
1778
|
+
op: "EQUAL_TO",
|
|
1779
|
+
right: "run"
|
|
1780
|
+
}
|
|
1781
|
+
},
|
|
1782
|
+
{
|
|
1783
|
+
task_key: "foreach",
|
|
1784
|
+
depends_on: [{ task_key: "branch", outcome: "true" }],
|
|
1785
|
+
for_each_task: {
|
|
1786
|
+
inputs: "{{tasks.seed.values.items}}",
|
|
1787
|
+
concurrency: 2,
|
|
1788
|
+
task: {
|
|
1789
|
+
task_key: "process_item",
|
|
1790
|
+
notebook_task: {
|
|
1791
|
+
notebook_path: paths.process,
|
|
1792
|
+
base_parameters: { item: "{{input}}" }
|
|
1793
|
+
}
|
|
1794
|
+
}
|
|
1795
|
+
}
|
|
1796
|
+
},
|
|
1797
|
+
{
|
|
1798
|
+
task_key: "repair",
|
|
1799
|
+
depends_on: [{ task_key: "foreach" }],
|
|
1800
|
+
notebook_task: {
|
|
1801
|
+
notebook_path: paths.repair,
|
|
1802
|
+
base_parameters: { repair_count: "{{job.repair_count}}" }
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
]
|
|
1806
|
+
};
|
|
1807
|
+
}
|
|
1808
|
+
function cancellationRequest(paths) {
|
|
1809
|
+
return {
|
|
1810
|
+
run_name: "fabric-experiments-jobs-cancellation-certification",
|
|
1811
|
+
tasks: [{ task_key: "sleep", notebook_task: { notebook_path: paths.sleep } }]
|
|
1812
|
+
};
|
|
1813
|
+
}
|
|
1814
|
+
function waitOptions(env) {
|
|
1815
|
+
return {
|
|
1816
|
+
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
1817
|
+
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 5e3)
|
|
1818
|
+
};
|
|
1819
|
+
}
|
|
1820
|
+
function assertTask(tasks, taskKey, resultState) {
|
|
1821
|
+
const task = tasks.find((candidate) => candidate.taskKey === taskKey);
|
|
1822
|
+
if (!task) throw new Error(`Certification result did not contain task '${taskKey}'`);
|
|
1823
|
+
if (task.resultState !== resultState) {
|
|
1824
|
+
throw new Error(
|
|
1825
|
+
`Certification task '${taskKey}' ended in ${task.resultState}, expected ${resultState}`
|
|
1826
|
+
);
|
|
1827
|
+
}
|
|
1828
|
+
return task;
|
|
1829
|
+
}
|
|
1543
1830
|
|
|
1544
1831
|
// src/jobs-target-pack.ts
|
|
1545
1832
|
function createLakeflowJobsTargetPack() {
|
|
@@ -1547,8 +1834,8 @@ function createLakeflowJobsTargetPack() {
|
|
|
1547
1834
|
id: "lakeflow-jobs",
|
|
1548
1835
|
name: "Lakeflow Jobs orchestration",
|
|
1549
1836
|
description: "Typed multi-task DAG, branch/loop, retry, cancellation, repair and task-outcome validation.",
|
|
1550
|
-
version: "0.
|
|
1551
|
-
maturity: "
|
|
1837
|
+
version: "0.2.0",
|
|
1838
|
+
maturity: "live-gated",
|
|
1552
1839
|
capabilities: [
|
|
1553
1840
|
"jobs.multi-task-dag",
|
|
1554
1841
|
"jobs.dependencies",
|
|
@@ -1559,7 +1846,12 @@ function createLakeflowJobsTargetPack() {
|
|
|
1559
1846
|
"jobs.repair",
|
|
1560
1847
|
"jobs.task-outcomes"
|
|
1561
1848
|
],
|
|
1562
|
-
checks: () => [
|
|
1849
|
+
checks: () => [
|
|
1850
|
+
restrictedPrincipalCheck(),
|
|
1851
|
+
jobRunCheck(),
|
|
1852
|
+
jobsOrchestrationCheck(),
|
|
1853
|
+
jobsCertificationCheck()
|
|
1854
|
+
],
|
|
1563
1855
|
requiredChecks: ["jobs-orchestration"],
|
|
1564
1856
|
requirements: [
|
|
1565
1857
|
{
|
|
@@ -1582,20 +1874,814 @@ function createLakeflowJobsTargetPack() {
|
|
|
1582
1874
|
{
|
|
1583
1875
|
id: "orchestration-spec",
|
|
1584
1876
|
description: "JSON multi-task orchestration scenario",
|
|
1585
|
-
anyOf: ["DBX_TEST_JOBS_ORCHESTRATION_JSON"]
|
|
1877
|
+
anyOf: ["DBX_TEST_JOBS_ORCHESTRATION_JSON"],
|
|
1878
|
+
required: false
|
|
1586
1879
|
}
|
|
1587
1880
|
],
|
|
1588
|
-
docsUrl: "https://experiments.fabric.pro/docs/testing/target-packs/#
|
|
1589
|
-
certificationScopes: [
|
|
1881
|
+
docsUrl: "https://experiments.fabric.pro/docs/testing/target-packs/#lakeflow-jobs-orchestration",
|
|
1882
|
+
certificationScopes: [
|
|
1883
|
+
"Azure Databricks serverless Jobs, West US, workspace OAuth, public network path"
|
|
1884
|
+
]
|
|
1590
1885
|
});
|
|
1591
1886
|
}
|
|
1592
1887
|
|
|
1888
|
+
// src/advanced-workload-checks.ts
|
|
1889
|
+
var DatabricksAdvancedWorkloadsAdapter = class {
|
|
1890
|
+
constructor(client) {
|
|
1891
|
+
this.client = client;
|
|
1892
|
+
}
|
|
1893
|
+
client;
|
|
1894
|
+
pipeline(id) {
|
|
1895
|
+
return this.client.get(`/api/2.0/pipelines/${segment(id)}`);
|
|
1896
|
+
}
|
|
1897
|
+
table(fullName) {
|
|
1898
|
+
return this.client.get(`/api/2.1/unity-catalog/tables/${segment(fullName)}`);
|
|
1899
|
+
}
|
|
1900
|
+
dashboard(id) {
|
|
1901
|
+
return this.client.get(`/api/2.0/lakeview/dashboards/${segment(id)}`);
|
|
1902
|
+
}
|
|
1903
|
+
genieSpace(id) {
|
|
1904
|
+
return this.client.get(`/api/2.0/genie/spaces/${segment(id)}`);
|
|
1905
|
+
}
|
|
1906
|
+
startGenieConversation(spaceId, question) {
|
|
1907
|
+
return this.client.post(`/api/2.0/genie/spaces/${segment(spaceId)}/start-conversation`, {
|
|
1908
|
+
content: question
|
|
1909
|
+
});
|
|
1910
|
+
}
|
|
1911
|
+
experiment(id) {
|
|
1912
|
+
return this.client.get("/api/2.0/mlflow/experiments/get", { experiment_id: id });
|
|
1913
|
+
}
|
|
1914
|
+
registeredModel(fullName) {
|
|
1915
|
+
return this.client.get(`/api/2.1/unity-catalog/models/${segment(fullName)}`);
|
|
1916
|
+
}
|
|
1917
|
+
modelVersion(fullName, version) {
|
|
1918
|
+
return this.client.get(
|
|
1919
|
+
`/api/2.1/unity-catalog/models/${segment(fullName)}/versions/${segment(version)}`
|
|
1920
|
+
);
|
|
1921
|
+
}
|
|
1922
|
+
onlineTable(fullName) {
|
|
1923
|
+
return this.client.get(`/api/2.0/online-tables/${segment(fullName)}`);
|
|
1924
|
+
}
|
|
1925
|
+
servingEndpoint(name) {
|
|
1926
|
+
return this.client.get(`/api/2.0/serving-endpoints/${segment(name)}`);
|
|
1927
|
+
}
|
|
1928
|
+
invokeServingEndpoint(name, request) {
|
|
1929
|
+
return this.client.post(`/serving-endpoints/${segment(name)}/invocations`, request);
|
|
1930
|
+
}
|
|
1931
|
+
vectorSearchEndpoint(name) {
|
|
1932
|
+
return this.client.get(`/api/2.0/vector-search/endpoints/${segment(name)}`);
|
|
1933
|
+
}
|
|
1934
|
+
vectorSearchIndex(name) {
|
|
1935
|
+
return this.client.get(`/api/2.0/vector-search/indexes/${segment(name)}`);
|
|
1936
|
+
}
|
|
1937
|
+
queryVectorSearchIndex(name, request) {
|
|
1938
|
+
return this.client.post(`/api/2.0/vector-search/indexes/${segment(name)}/query`, request);
|
|
1939
|
+
}
|
|
1940
|
+
connection(name) {
|
|
1941
|
+
return this.client.get(`/api/2.1/unity-catalog/connections/${segment(name)}`);
|
|
1942
|
+
}
|
|
1943
|
+
share(name) {
|
|
1944
|
+
return this.client.get(`/api/2.1/unity-catalog/shares/${segment(name)}`);
|
|
1945
|
+
}
|
|
1946
|
+
cleanRoom(name) {
|
|
1947
|
+
return this.client.get(`/api/2.0/clean-rooms/${segment(name)}`);
|
|
1948
|
+
}
|
|
1949
|
+
ipAccessList(id) {
|
|
1950
|
+
return this.client.get(`/api/2.0/ip-access-lists/${segment(id)}`);
|
|
1951
|
+
}
|
|
1952
|
+
clusterPolicy(id) {
|
|
1953
|
+
return this.client.get("/api/2.0/policies/clusters/get", { policy_id: id });
|
|
1954
|
+
}
|
|
1955
|
+
warehouse(id) {
|
|
1956
|
+
return this.client.get(`/api/2.0/sql/warehouses/${segment(id)}`);
|
|
1957
|
+
}
|
|
1958
|
+
};
|
|
1959
|
+
function segment(value) {
|
|
1960
|
+
return encodeURIComponent(value);
|
|
1961
|
+
}
|
|
1962
|
+
function adapter(client) {
|
|
1963
|
+
return new DatabricksAdvancedWorkloadsAdapter(client);
|
|
1964
|
+
}
|
|
1965
|
+
function required2(env, names) {
|
|
1966
|
+
return names.filter((name) => !env[name]?.trim());
|
|
1967
|
+
}
|
|
1968
|
+
function requireEnv(env, names) {
|
|
1969
|
+
const missing = required2(env, names);
|
|
1970
|
+
if (missing.length > 0) throw new Error(`Missing required configuration: ${missing.join(", ")}`);
|
|
1971
|
+
}
|
|
1972
|
+
function parseJson(value, name) {
|
|
1973
|
+
if (!value) return void 0;
|
|
1974
|
+
try {
|
|
1975
|
+
return JSON.parse(value);
|
|
1976
|
+
} catch {
|
|
1977
|
+
throw new Error(`${name} must contain valid JSON`);
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
function assertNotFailed(label, state) {
|
|
1981
|
+
if (typeof state === "string" && /FAIL|ERROR|DELET/i.test(state)) {
|
|
1982
|
+
throw new Error(`${label} is ${state}`);
|
|
1983
|
+
}
|
|
1984
|
+
}
|
|
1985
|
+
function assertReady(label, state, accepted) {
|
|
1986
|
+
if (typeof state !== "string" || !accepted.includes(state.toUpperCase())) {
|
|
1987
|
+
throw new Error(`${label} is ${String(state ?? "UNKNOWN")}; expected ${accepted.join(" or ")}`);
|
|
1988
|
+
}
|
|
1989
|
+
}
|
|
1990
|
+
async function booleanSql(label, sql, query) {
|
|
1991
|
+
if (!sql) return;
|
|
1992
|
+
const rows = await query(sql);
|
|
1993
|
+
const first = rows[0] ? Object.values(rows[0])[0] : void 0;
|
|
1994
|
+
if (!(first === true || first === 1 || first === "1" || first === "true")) {
|
|
1995
|
+
throw new Error(`${label} assertion did not return a truthy first column`);
|
|
1996
|
+
}
|
|
1997
|
+
}
|
|
1998
|
+
function advancedStreamingCdcCheck() {
|
|
1999
|
+
const names = [
|
|
2000
|
+
"DBX_TEST_STREAMING_PIPELINE_ID",
|
|
2001
|
+
"DBX_TEST_STREAMING_TABLE",
|
|
2002
|
+
"DBX_TEST_CDC_ASSERTION_SQL",
|
|
2003
|
+
"DBX_TEST_STREAMING_ASSERTION_SQL"
|
|
2004
|
+
];
|
|
2005
|
+
return {
|
|
2006
|
+
id: "streaming-cdc",
|
|
2007
|
+
description: "Streaming pipeline, streaming table and CDC assertions are healthy",
|
|
2008
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2009
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2010
|
+
requireEnv(env, names);
|
|
2011
|
+
const api = adapter(client);
|
|
2012
|
+
const pipeline = await api.pipeline(env.DBX_TEST_STREAMING_PIPELINE_ID);
|
|
2013
|
+
assertNotFailed("Streaming pipeline", pipeline.state);
|
|
2014
|
+
const table = await api.table(env.DBX_TEST_STREAMING_TABLE);
|
|
2015
|
+
const tableType = String(table.table_type ?? table.data_source_format ?? "").toUpperCase();
|
|
2016
|
+
if (!["STREAMING_TABLE", "MATERIALIZED_VIEW", "DELTA"].includes(tableType)) {
|
|
2017
|
+
throw new Error(`Streaming target has unsupported table type ${tableType || "UNKNOWN"}`);
|
|
2018
|
+
}
|
|
2019
|
+
await booleanSql("CDC", env.DBX_TEST_CDC_ASSERTION_SQL, warehouse2.query);
|
|
2020
|
+
await booleanSql(
|
|
2021
|
+
"Streaming freshness",
|
|
2022
|
+
env.DBX_TEST_STREAMING_ASSERTION_SQL,
|
|
2023
|
+
warehouse2.query
|
|
2024
|
+
);
|
|
2025
|
+
return { pipelineId: pipeline.pipeline_id, table: env.DBX_TEST_STREAMING_TABLE, tableType };
|
|
2026
|
+
}
|
|
2027
|
+
};
|
|
2028
|
+
}
|
|
2029
|
+
function lakeflowConnectCheck() {
|
|
2030
|
+
const names = ["DBX_TEST_CONNECT_PIPELINE_ID", "DBX_TEST_CONNECT_ASSERTION_SQL"];
|
|
2031
|
+
return {
|
|
2032
|
+
id: "lakeflow-connect",
|
|
2033
|
+
description: "Lakeflow Connect ingestion pipeline exists and is not failed",
|
|
2034
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2035
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2036
|
+
requireEnv(env, names);
|
|
2037
|
+
const id = env.DBX_TEST_CONNECT_PIPELINE_ID;
|
|
2038
|
+
const pipeline = await adapter(client).pipeline(id);
|
|
2039
|
+
assertNotFailed("Lakeflow Connect pipeline", pipeline.state);
|
|
2040
|
+
await booleanSql(
|
|
2041
|
+
"Lakeflow Connect replication",
|
|
2042
|
+
env.DBX_TEST_CONNECT_ASSERTION_SQL,
|
|
2043
|
+
warehouse2.query
|
|
2044
|
+
);
|
|
2045
|
+
return { pipelineId: pipeline.pipeline_id ?? id, state: pipeline.state ?? "IDLE" };
|
|
2046
|
+
}
|
|
2047
|
+
};
|
|
2048
|
+
}
|
|
2049
|
+
function aiBiDashboardCheck() {
|
|
2050
|
+
return {
|
|
2051
|
+
id: "aibi-dashboard",
|
|
2052
|
+
description: "Published AI/BI dashboard is accessible",
|
|
2053
|
+
configured: (env) => Boolean(env.DBX_TEST_DASHBOARD_ID),
|
|
2054
|
+
async run({ env, client }) {
|
|
2055
|
+
const dashboard = await adapter(client).dashboard(env.DBX_TEST_DASHBOARD_ID);
|
|
2056
|
+
assertNotFailed("AI/BI dashboard", dashboard.lifecycle_state);
|
|
2057
|
+
return {
|
|
2058
|
+
dashboardId: dashboard.dashboard_id ?? env.DBX_TEST_DASHBOARD_ID,
|
|
2059
|
+
name: dashboard.display_name
|
|
2060
|
+
};
|
|
2061
|
+
}
|
|
2062
|
+
};
|
|
2063
|
+
}
|
|
2064
|
+
function genieCheck() {
|
|
2065
|
+
const names = ["DBX_TEST_GENIE_SPACE_ID", "DBX_TEST_GENIE_QUESTION"];
|
|
2066
|
+
return {
|
|
2067
|
+
id: "genie",
|
|
2068
|
+
description: "Genie space is accessible and can optionally start a grounded conversation",
|
|
2069
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2070
|
+
async run({ env, client }) {
|
|
2071
|
+
requireEnv(env, names);
|
|
2072
|
+
const api = adapter(client);
|
|
2073
|
+
const id = env.DBX_TEST_GENIE_SPACE_ID;
|
|
2074
|
+
const space = await api.genieSpace(id);
|
|
2075
|
+
let conversation;
|
|
2076
|
+
if (env.DBX_TEST_GENIE_QUESTION) {
|
|
2077
|
+
conversation = await api.startGenieConversation(id, env.DBX_TEST_GENIE_QUESTION);
|
|
2078
|
+
if (!conversation.conversation_id && !conversation.message_id) {
|
|
2079
|
+
throw new Error("Genie did not return a conversation or message identifier");
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
return {
|
|
2083
|
+
spaceId: space.space_id ?? id,
|
|
2084
|
+
title: space.title,
|
|
2085
|
+
conversationStarted: Boolean(conversation)
|
|
2086
|
+
};
|
|
2087
|
+
}
|
|
2088
|
+
};
|
|
2089
|
+
}
|
|
2090
|
+
function mlflowLifecycleCheck() {
|
|
2091
|
+
const names = [
|
|
2092
|
+
"DBX_TEST_MLFLOW_EXPERIMENT_ID",
|
|
2093
|
+
"DBX_TEST_REGISTERED_MODEL",
|
|
2094
|
+
"DBX_TEST_MODEL_VERSION"
|
|
2095
|
+
];
|
|
2096
|
+
return {
|
|
2097
|
+
id: "mlflow-lifecycle",
|
|
2098
|
+
description: "MLflow experiment and Unity Catalog model version are accessible",
|
|
2099
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2100
|
+
async run({ env, client }) {
|
|
2101
|
+
requireEnv(env, names);
|
|
2102
|
+
const api = adapter(client);
|
|
2103
|
+
const experiment = await api.experiment(env.DBX_TEST_MLFLOW_EXPERIMENT_ID);
|
|
2104
|
+
const model = await api.registeredModel(env.DBX_TEST_REGISTERED_MODEL);
|
|
2105
|
+
const version = await api.modelVersion(
|
|
2106
|
+
env.DBX_TEST_REGISTERED_MODEL,
|
|
2107
|
+
env.DBX_TEST_MODEL_VERSION
|
|
2108
|
+
);
|
|
2109
|
+
assertNotFailed("Model version", version.status);
|
|
2110
|
+
return {
|
|
2111
|
+
experimentId: experiment.experiment?.experiment_id ?? env.DBX_TEST_MLFLOW_EXPERIMENT_ID,
|
|
2112
|
+
model: model.full_name ?? model.name ?? env.DBX_TEST_REGISTERED_MODEL,
|
|
2113
|
+
version: version.version ?? env.DBX_TEST_MODEL_VERSION
|
|
2114
|
+
};
|
|
2115
|
+
}
|
|
2116
|
+
};
|
|
2117
|
+
}
|
|
2118
|
+
function featureEngineeringServingCheck() {
|
|
2119
|
+
return {
|
|
2120
|
+
id: "feature-engineering",
|
|
2121
|
+
description: "Feature table and online materialization are healthy",
|
|
2122
|
+
configured: (env) => Boolean(
|
|
2123
|
+
env.DBX_TEST_FEATURE_TABLE && env.DBX_TEST_FEATURE_ASSERTION_SQL && (env.DBX_TEST_ONLINE_TABLE || env.DBX_TEST_FEATURE_SERVING_PIPELINE_ID)
|
|
2124
|
+
),
|
|
2125
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2126
|
+
requireEnv(env, ["DBX_TEST_FEATURE_TABLE"]);
|
|
2127
|
+
if (!env.DBX_TEST_ONLINE_TABLE && !env.DBX_TEST_FEATURE_SERVING_PIPELINE_ID) {
|
|
2128
|
+
throw new Error(
|
|
2129
|
+
"Feature serving requires DBX_TEST_ONLINE_TABLE or DBX_TEST_FEATURE_SERVING_PIPELINE_ID"
|
|
2130
|
+
);
|
|
2131
|
+
}
|
|
2132
|
+
const api = adapter(client);
|
|
2133
|
+
const source = await api.table(env.DBX_TEST_FEATURE_TABLE);
|
|
2134
|
+
let state;
|
|
2135
|
+
let servingResource;
|
|
2136
|
+
if (env.DBX_TEST_ONLINE_TABLE) {
|
|
2137
|
+
const online = await api.onlineTable(env.DBX_TEST_ONLINE_TABLE);
|
|
2138
|
+
const status = online.status;
|
|
2139
|
+
state = status?.state ?? status?.detailed_state ?? online.state;
|
|
2140
|
+
assertReady("Online table", state, ["ONLINE", "ACTIVE", "READY", "PROVISIONED"]);
|
|
2141
|
+
servingResource = env.DBX_TEST_ONLINE_TABLE;
|
|
2142
|
+
} else {
|
|
2143
|
+
const pipelineId = env.DBX_TEST_FEATURE_SERVING_PIPELINE_ID;
|
|
2144
|
+
const pipeline = await api.pipeline(pipelineId);
|
|
2145
|
+
state = pipeline.state ?? "IDLE";
|
|
2146
|
+
assertNotFailed("Synced Table pipeline", state);
|
|
2147
|
+
servingResource = pipeline.pipeline_id ?? pipelineId;
|
|
2148
|
+
}
|
|
2149
|
+
await booleanSql("Feature freshness", env.DBX_TEST_FEATURE_ASSERTION_SQL, warehouse2.query);
|
|
2150
|
+
return {
|
|
2151
|
+
featureTable: source.full_name ?? env.DBX_TEST_FEATURE_TABLE,
|
|
2152
|
+
servingResource,
|
|
2153
|
+
state
|
|
2154
|
+
};
|
|
2155
|
+
}
|
|
2156
|
+
};
|
|
2157
|
+
}
|
|
2158
|
+
function modelServingCheck() {
|
|
2159
|
+
const names = [
|
|
2160
|
+
"DBX_TEST_SERVING_ENDPOINT",
|
|
2161
|
+
"DBX_TEST_SERVING_REQUEST_JSON",
|
|
2162
|
+
"DBX_TEST_AI_GATEWAY_REQUIRED"
|
|
2163
|
+
];
|
|
2164
|
+
return {
|
|
2165
|
+
id: "model-serving",
|
|
2166
|
+
description: "Model Serving endpoint is ready and can optionally serve an inference request",
|
|
2167
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2168
|
+
async run({ env, client }) {
|
|
2169
|
+
requireEnv(env, names);
|
|
2170
|
+
if (env.DBX_TEST_AI_GATEWAY_REQUIRED !== "1") {
|
|
2171
|
+
throw new Error("DBX_TEST_AI_GATEWAY_REQUIRED must be 1 for the Model Serving target pack");
|
|
2172
|
+
}
|
|
2173
|
+
const api = adapter(client);
|
|
2174
|
+
const name = env.DBX_TEST_SERVING_ENDPOINT;
|
|
2175
|
+
const endpoint = await api.servingEndpoint(name);
|
|
2176
|
+
assertReady("Serving endpoint", endpoint.state?.ready, ["READY"]);
|
|
2177
|
+
if (env.DBX_TEST_AI_GATEWAY_REQUIRED === "1" && !endpoint.ai_gateway) {
|
|
2178
|
+
throw new Error(`Serving endpoint ${name} has no AI Gateway configuration`);
|
|
2179
|
+
}
|
|
2180
|
+
let response;
|
|
2181
|
+
if (env.DBX_TEST_SERVING_REQUEST_JSON) {
|
|
2182
|
+
response = await api.invokeServingEndpoint(
|
|
2183
|
+
name,
|
|
2184
|
+
parseJson(env.DBX_TEST_SERVING_REQUEST_JSON, "DBX_TEST_SERVING_REQUEST_JSON")
|
|
2185
|
+
);
|
|
2186
|
+
if (Object.keys(response).length === 0)
|
|
2187
|
+
throw new Error("Serving invocation returned an empty response");
|
|
2188
|
+
}
|
|
2189
|
+
return {
|
|
2190
|
+
endpoint: endpoint.name ?? name,
|
|
2191
|
+
ready: endpoint.state?.ready,
|
|
2192
|
+
aiGateway: Boolean(endpoint.ai_gateway),
|
|
2193
|
+
invoked: Boolean(response)
|
|
2194
|
+
};
|
|
2195
|
+
}
|
|
2196
|
+
};
|
|
2197
|
+
}
|
|
2198
|
+
function vectorSearchCheck() {
|
|
2199
|
+
const names = ["DBX_TEST_VECTOR_ENDPOINT", "DBX_TEST_VECTOR_INDEX"];
|
|
2200
|
+
return {
|
|
2201
|
+
id: "vector-search",
|
|
2202
|
+
description: "Vector Search endpoint and index are online",
|
|
2203
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2204
|
+
async run({ env, client }) {
|
|
2205
|
+
requireEnv(env, names);
|
|
2206
|
+
const api = adapter(client);
|
|
2207
|
+
const endpoint = await api.vectorSearchEndpoint(env.DBX_TEST_VECTOR_ENDPOINT);
|
|
2208
|
+
assertReady("Vector Search endpoint", endpoint.endpoint_status?.state, ["ONLINE", "READY"]);
|
|
2209
|
+
const index = await api.vectorSearchIndex(env.DBX_TEST_VECTOR_INDEX);
|
|
2210
|
+
if (index.status?.ready !== true)
|
|
2211
|
+
throw new Error(`Vector Search index is not ready: ${index.status?.message ?? "UNKNOWN"}`);
|
|
2212
|
+
return {
|
|
2213
|
+
endpoint: endpoint.name ?? env.DBX_TEST_VECTOR_ENDPOINT,
|
|
2214
|
+
index: index.name ?? env.DBX_TEST_VECTOR_INDEX,
|
|
2215
|
+
indexedRows: index.status.indexed_row_count
|
|
2216
|
+
};
|
|
2217
|
+
}
|
|
2218
|
+
};
|
|
2219
|
+
}
|
|
2220
|
+
function ragAgentCheck() {
|
|
2221
|
+
const names = [
|
|
2222
|
+
"DBX_TEST_VECTOR_INDEX",
|
|
2223
|
+
"DBX_TEST_VECTOR_QUERY_JSON",
|
|
2224
|
+
"DBX_TEST_AGENT_ENDPOINT",
|
|
2225
|
+
"DBX_TEST_AGENT_REQUEST_JSON"
|
|
2226
|
+
];
|
|
2227
|
+
return {
|
|
2228
|
+
id: "rag-agent",
|
|
2229
|
+
description: "RAG retrieval and agent serving endpoint return non-empty responses",
|
|
2230
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2231
|
+
async run({ env, client }) {
|
|
2232
|
+
requireEnv(env, names);
|
|
2233
|
+
const api = adapter(client);
|
|
2234
|
+
let retrieval;
|
|
2235
|
+
if (env.DBX_TEST_VECTOR_QUERY_JSON) {
|
|
2236
|
+
retrieval = await api.queryVectorSearchIndex(
|
|
2237
|
+
env.DBX_TEST_VECTOR_INDEX,
|
|
2238
|
+
parseJson(env.DBX_TEST_VECTOR_QUERY_JSON, "DBX_TEST_VECTOR_QUERY_JSON")
|
|
2239
|
+
);
|
|
2240
|
+
if (!retrieval.result && !retrieval.data_array && !retrieval.manifest) {
|
|
2241
|
+
throw new Error("Vector query returned no result payload");
|
|
2242
|
+
}
|
|
2243
|
+
}
|
|
2244
|
+
let agent;
|
|
2245
|
+
if (env.DBX_TEST_AGENT_ENDPOINT) {
|
|
2246
|
+
const endpoint = await api.servingEndpoint(env.DBX_TEST_AGENT_ENDPOINT);
|
|
2247
|
+
assertReady("Agent endpoint", endpoint.state?.ready, ["READY"]);
|
|
2248
|
+
if (env.DBX_TEST_AGENT_REQUEST_JSON) {
|
|
2249
|
+
agent = await api.invokeServingEndpoint(
|
|
2250
|
+
env.DBX_TEST_AGENT_ENDPOINT,
|
|
2251
|
+
parseJson(env.DBX_TEST_AGENT_REQUEST_JSON, "DBX_TEST_AGENT_REQUEST_JSON")
|
|
2252
|
+
);
|
|
2253
|
+
if (Object.keys(agent).length === 0)
|
|
2254
|
+
throw new Error("Agent invocation returned an empty response");
|
|
2255
|
+
}
|
|
2256
|
+
}
|
|
2257
|
+
return {
|
|
2258
|
+
retrieval: Boolean(retrieval),
|
|
2259
|
+
agentEndpoint: env.DBX_TEST_AGENT_ENDPOINT,
|
|
2260
|
+
agentInvoked: Boolean(agent)
|
|
2261
|
+
};
|
|
2262
|
+
}
|
|
2263
|
+
};
|
|
2264
|
+
}
|
|
2265
|
+
function federationSharingCleanRoomsCheck() {
|
|
2266
|
+
const names = [
|
|
2267
|
+
"DBX_TEST_CONNECTION",
|
|
2268
|
+
"DBX_TEST_SHARE",
|
|
2269
|
+
"DBX_TEST_CLEAN_ROOM",
|
|
2270
|
+
"DBX_TEST_FEDERATION_ASSERTION_SQL"
|
|
2271
|
+
];
|
|
2272
|
+
return {
|
|
2273
|
+
id: "federation-sharing-cleanrooms",
|
|
2274
|
+
description: "Lakehouse Federation connection, Delta Share and Clean Room are accessible",
|
|
2275
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2276
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2277
|
+
requireEnv(env, names);
|
|
2278
|
+
const api = adapter(client);
|
|
2279
|
+
const connection = await api.connection(env.DBX_TEST_CONNECTION);
|
|
2280
|
+
const share = await api.share(env.DBX_TEST_SHARE);
|
|
2281
|
+
const room = await api.cleanRoom(env.DBX_TEST_CLEAN_ROOM);
|
|
2282
|
+
await booleanSql("Federated query", env.DBX_TEST_FEDERATION_ASSERTION_SQL, warehouse2.query);
|
|
2283
|
+
return {
|
|
2284
|
+
connection: connection.name ?? env.DBX_TEST_CONNECTION,
|
|
2285
|
+
share: share.name ?? env.DBX_TEST_SHARE,
|
|
2286
|
+
cleanRoom: room.name ?? env.DBX_TEST_CLEAN_ROOM
|
|
2287
|
+
};
|
|
2288
|
+
}
|
|
2289
|
+
};
|
|
2290
|
+
}
|
|
2291
|
+
function securityPostureCostCheck() {
|
|
2292
|
+
const names = [
|
|
2293
|
+
"DBX_TEST_IP_ACCESS_LIST_ID",
|
|
2294
|
+
"DBX_TEST_CLUSTER_POLICY_ID",
|
|
2295
|
+
"DBX_TEST_COST_ASSERTION_SQL"
|
|
2296
|
+
];
|
|
2297
|
+
return {
|
|
2298
|
+
id: "security-cost",
|
|
2299
|
+
description: "Network access controls, compute policy and cost guardrail are enforced",
|
|
2300
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2301
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2302
|
+
requireEnv(env, names);
|
|
2303
|
+
const api = adapter(client);
|
|
2304
|
+
const access = await api.ipAccessList(env.DBX_TEST_IP_ACCESS_LIST_ID);
|
|
2305
|
+
if (access.enabled === false) throw new Error("Configured IP access list is disabled");
|
|
2306
|
+
const policy = await api.clusterPolicy(env.DBX_TEST_CLUSTER_POLICY_ID);
|
|
2307
|
+
if (!policy.policy_id && !policy.name)
|
|
2308
|
+
throw new Error("Cluster policy response has no identity");
|
|
2309
|
+
if (env.DBX_TEST_EXPECT_HOST_SUFFIX && !env.DATABRICKS_HOST?.endsWith(env.DBX_TEST_EXPECT_HOST_SUFFIX)) {
|
|
2310
|
+
throw new Error(`Workspace host does not end with ${env.DBX_TEST_EXPECT_HOST_SUFFIX}`);
|
|
2311
|
+
}
|
|
2312
|
+
await booleanSql("Cost guardrail", env.DBX_TEST_COST_ASSERTION_SQL, warehouse2.query);
|
|
2313
|
+
return {
|
|
2314
|
+
ipAccessList: access.list_id ?? env.DBX_TEST_IP_ACCESS_LIST_ID,
|
|
2315
|
+
policy: policy.policy_id ?? policy.name,
|
|
2316
|
+
costGuardrail: "pass"
|
|
2317
|
+
};
|
|
2318
|
+
}
|
|
2319
|
+
};
|
|
2320
|
+
}
|
|
2321
|
+
function regionalDrCheck(options = {}) {
|
|
2322
|
+
const names = [
|
|
2323
|
+
"DBX_TEST_DR_HOST",
|
|
2324
|
+
"DBX_TEST_DR_WAREHOUSE_ID",
|
|
2325
|
+
"DBX_TEST_DR_ASSERTION_SQL"
|
|
2326
|
+
];
|
|
2327
|
+
return {
|
|
2328
|
+
id: "regional-dr",
|
|
2329
|
+
description: "Secondary-region workspace authentication and SQL warehouse are reachable",
|
|
2330
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2331
|
+
async run({ env }) {
|
|
2332
|
+
requireEnv(env, names);
|
|
2333
|
+
const drEnv = {
|
|
2334
|
+
...env,
|
|
2335
|
+
DATABRICKS_HOST: env.DBX_TEST_DR_HOST,
|
|
2336
|
+
DATABRICKS_WAREHOUSE_ID: env.DBX_TEST_DR_WAREHOUSE_ID,
|
|
2337
|
+
DATABRICKS_HTTP_PATH: void 0,
|
|
2338
|
+
DATABRICKS_BEARER: env.DBX_TEST_DR_BEARER,
|
|
2339
|
+
DATABRICKS_TOKEN: env.DBX_TEST_DR_TOKEN,
|
|
2340
|
+
DATABRICKS_CLIENT_ID: env.DBX_TEST_DR_CLIENT_ID ?? env.DATABRICKS_CLIENT_ID,
|
|
2341
|
+
DATABRICKS_CLIENT_SECRET: env.DBX_TEST_DR_CLIENT_SECRET ?? env.DATABRICKS_CLIENT_SECRET
|
|
2342
|
+
};
|
|
2343
|
+
const client = options.clientFactory?.(drEnv) ?? createClientFromEnv(drEnv);
|
|
2344
|
+
const secondary = await adapter(client).warehouse(env.DBX_TEST_DR_WAREHOUSE_ID);
|
|
2345
|
+
assertReady("DR SQL warehouse", secondary.state, ["RUNNING"]);
|
|
2346
|
+
const drWarehouse = await createWorkspaceContext({ env: drEnv, client });
|
|
2347
|
+
try {
|
|
2348
|
+
await booleanSql("Regional DR workload", env.DBX_TEST_DR_ASSERTION_SQL, drWarehouse.query);
|
|
2349
|
+
} finally {
|
|
2350
|
+
await drWarehouse.close();
|
|
2351
|
+
}
|
|
2352
|
+
return {
|
|
2353
|
+
secondaryWorkspace: "reachable",
|
|
2354
|
+
warehouseId: secondary.id ?? env.DBX_TEST_DR_WAREHOUSE_ID,
|
|
2355
|
+
state: secondary.state
|
|
2356
|
+
};
|
|
2357
|
+
}
|
|
2358
|
+
};
|
|
2359
|
+
}
|
|
2360
|
+
|
|
2361
|
+
// src/advanced-target-packs.ts
|
|
2362
|
+
var DOCS2 = "https://experiments.fabric.pro/docs/testing/target-packs/";
|
|
2363
|
+
var SHIPPED_SCOPE = "Typed adapter and offline contract tests shipped; live certification requires customer-owned test resources";
|
|
2364
|
+
var AZURE_USER_SCOPE = "Azure Databricks, West US, interactive user OAuth, public network path; existing certification fixtures";
|
|
2365
|
+
var workspace = {
|
|
2366
|
+
id: "workspace",
|
|
2367
|
+
description: "Databricks workspace host",
|
|
2368
|
+
anyOf: ["DATABRICKS_HOST"]
|
|
2369
|
+
};
|
|
2370
|
+
var auth = {
|
|
2371
|
+
id: "authentication",
|
|
2372
|
+
description: "Databricks OAuth, OIDC, bearer, or PAT authentication",
|
|
2373
|
+
anyOf: [
|
|
2374
|
+
"DATABRICKS_BEARER",
|
|
2375
|
+
"DATABRICKS_TOKEN",
|
|
2376
|
+
"DATABRICKS_CLIENT_ID",
|
|
2377
|
+
"DATABRICKS_OIDC_TOKEN",
|
|
2378
|
+
"DATABRICKS_OIDC_TOKEN_FILEPATH"
|
|
2379
|
+
],
|
|
2380
|
+
secret: true
|
|
2381
|
+
};
|
|
2382
|
+
var warehouse = {
|
|
2383
|
+
id: "warehouse",
|
|
2384
|
+
description: "SQL warehouse for data-plane assertions",
|
|
2385
|
+
anyOf: ["DATABRICKS_WAREHOUSE_ID", "DATABRICKS_HTTP_PATH"]
|
|
2386
|
+
};
|
|
2387
|
+
var requirement = (id, description, ...anyOf) => ({
|
|
2388
|
+
id,
|
|
2389
|
+
description,
|
|
2390
|
+
anyOf
|
|
2391
|
+
});
|
|
2392
|
+
function createAdvancedStreamingTargetPack() {
|
|
2393
|
+
return defineTargetPack({
|
|
2394
|
+
id: "streaming-cdc-connect",
|
|
2395
|
+
name: "Advanced streaming, CDC and Lakeflow Connect",
|
|
2396
|
+
description: "Streaming tables, pipeline health, CDC correctness, freshness and managed connector replication.",
|
|
2397
|
+
version: "0.1.0",
|
|
2398
|
+
maturity: "shipped",
|
|
2399
|
+
capabilities: [
|
|
2400
|
+
"streaming.tables",
|
|
2401
|
+
"streaming.freshness",
|
|
2402
|
+
"cdc.correctness",
|
|
2403
|
+
"lakeflow-connect.replication"
|
|
2404
|
+
],
|
|
2405
|
+
checks: () => [restrictedPrincipalCheck(), advancedStreamingCdcCheck(), lakeflowConnectCheck()],
|
|
2406
|
+
requiredChecks: ["streaming-cdc", "lakeflow-connect"],
|
|
2407
|
+
requirements: [
|
|
2408
|
+
workspace,
|
|
2409
|
+
auth,
|
|
2410
|
+
warehouse,
|
|
2411
|
+
requirement("streaming-pipeline", "Streaming pipeline ID", "DBX_TEST_STREAMING_PIPELINE_ID"),
|
|
2412
|
+
requirement("streaming-table", "Streaming target table", "DBX_TEST_STREAMING_TABLE"),
|
|
2413
|
+
requirement(
|
|
2414
|
+
"connect-pipeline",
|
|
2415
|
+
"Lakeflow Connect ingestion pipeline",
|
|
2416
|
+
"DBX_TEST_CONNECT_PIPELINE_ID"
|
|
2417
|
+
),
|
|
2418
|
+
requirement(
|
|
2419
|
+
"streaming-assertion",
|
|
2420
|
+
"Streaming freshness SQL",
|
|
2421
|
+
"DBX_TEST_STREAMING_ASSERTION_SQL"
|
|
2422
|
+
),
|
|
2423
|
+
requirement("cdc-assertion", "CDC correctness SQL", "DBX_TEST_CDC_ASSERTION_SQL"),
|
|
2424
|
+
requirement(
|
|
2425
|
+
"connect-assertion",
|
|
2426
|
+
"Lakeflow Connect replication SQL",
|
|
2427
|
+
"DBX_TEST_CONNECT_ASSERTION_SQL"
|
|
2428
|
+
)
|
|
2429
|
+
],
|
|
2430
|
+
docsUrl: `${DOCS2}#advanced-streaming-cdc-and-lakeflow-connect`,
|
|
2431
|
+
certificationScopes: [SHIPPED_SCOPE]
|
|
2432
|
+
});
|
|
2433
|
+
}
|
|
2434
|
+
function createAiBiGenieTargetPack() {
|
|
2435
|
+
return defineTargetPack({
|
|
2436
|
+
id: "aibi-genie",
|
|
2437
|
+
name: "AI/BI dashboards and Genie",
|
|
2438
|
+
description: "Dashboard accessibility and authenticated Genie space/conversation behavior.",
|
|
2439
|
+
version: "0.1.0",
|
|
2440
|
+
maturity: "live-gated",
|
|
2441
|
+
capabilities: ["aibi.dashboard", "genie.space", "genie.conversation"],
|
|
2442
|
+
checks: () => [restrictedPrincipalCheck(), aiBiDashboardCheck(), genieCheck()],
|
|
2443
|
+
requiredChecks: ["aibi-dashboard", "genie"],
|
|
2444
|
+
requirements: [
|
|
2445
|
+
workspace,
|
|
2446
|
+
auth,
|
|
2447
|
+
requirement("dashboard", "AI/BI dashboard ID", "DBX_TEST_DASHBOARD_ID"),
|
|
2448
|
+
requirement("genie-space", "Genie space ID", "DBX_TEST_GENIE_SPACE_ID"),
|
|
2449
|
+
requirement(
|
|
2450
|
+
"genie-question",
|
|
2451
|
+
"Grounded Genie certification question",
|
|
2452
|
+
"DBX_TEST_GENIE_QUESTION"
|
|
2453
|
+
)
|
|
2454
|
+
],
|
|
2455
|
+
docsUrl: `${DOCS2}#aibi-dashboards-and-genie`,
|
|
2456
|
+
certificationScopes: [
|
|
2457
|
+
`${AZURE_USER_SCOPE}; AI/BI dashboard metadata and Genie conversation start`
|
|
2458
|
+
]
|
|
2459
|
+
});
|
|
2460
|
+
}
|
|
2461
|
+
function createMlflowLifecycleTargetPack() {
|
|
2462
|
+
return defineTargetPack({
|
|
2463
|
+
id: "mlflow-lifecycle",
|
|
2464
|
+
name: "MLflow and model lifecycle",
|
|
2465
|
+
description: "MLflow experiments plus Unity Catalog registered-model and model-version lifecycle evidence.",
|
|
2466
|
+
version: "0.1.0",
|
|
2467
|
+
maturity: "live-gated",
|
|
2468
|
+
capabilities: [
|
|
2469
|
+
"mlflow.experiments",
|
|
2470
|
+
"models.unity-catalog",
|
|
2471
|
+
"models.versions",
|
|
2472
|
+
"models.lifecycle"
|
|
2473
|
+
],
|
|
2474
|
+
checks: () => [restrictedPrincipalCheck(), mlflowLifecycleCheck()],
|
|
2475
|
+
requiredChecks: ["mlflow-lifecycle"],
|
|
2476
|
+
requirements: [
|
|
2477
|
+
workspace,
|
|
2478
|
+
auth,
|
|
2479
|
+
requirement("experiment", "MLflow experiment ID", "DBX_TEST_MLFLOW_EXPERIMENT_ID"),
|
|
2480
|
+
requirement(
|
|
2481
|
+
"registered-model",
|
|
2482
|
+
"Unity Catalog registered model",
|
|
2483
|
+
"DBX_TEST_REGISTERED_MODEL"
|
|
2484
|
+
),
|
|
2485
|
+
requirement("model-version", "Registered model version", "DBX_TEST_MODEL_VERSION")
|
|
2486
|
+
],
|
|
2487
|
+
docsUrl: `${DOCS2}#mlflow-and-model-lifecycle`,
|
|
2488
|
+
certificationScopes: [
|
|
2489
|
+
`${AZURE_USER_SCOPE}; MLflow experiment and Unity Catalog model version 20`
|
|
2490
|
+
]
|
|
2491
|
+
});
|
|
2492
|
+
}
|
|
2493
|
+
function createFeatureEngineeringTargetPack() {
|
|
2494
|
+
return defineTargetPack({
|
|
2495
|
+
id: "feature-engineering-serving",
|
|
2496
|
+
name: "Feature engineering and serving",
|
|
2497
|
+
description: "Unity Catalog feature tables, online materialization state and freshness assertions.",
|
|
2498
|
+
version: "0.1.0",
|
|
2499
|
+
maturity: "live-gated",
|
|
2500
|
+
capabilities: [
|
|
2501
|
+
"features.tables",
|
|
2502
|
+
"features.online-tables",
|
|
2503
|
+
"features.freshness",
|
|
2504
|
+
"features.serving"
|
|
2505
|
+
],
|
|
2506
|
+
checks: () => [restrictedPrincipalCheck(), featureEngineeringServingCheck()],
|
|
2507
|
+
requiredChecks: ["feature-engineering"],
|
|
2508
|
+
requirements: [
|
|
2509
|
+
workspace,
|
|
2510
|
+
auth,
|
|
2511
|
+
warehouse,
|
|
2512
|
+
requirement("feature-table", "Unity Catalog feature table", "DBX_TEST_FEATURE_TABLE"),
|
|
2513
|
+
requirement(
|
|
2514
|
+
"online-materialization",
|
|
2515
|
+
"Databricks Online Table or Synced Table pipeline",
|
|
2516
|
+
"DBX_TEST_ONLINE_TABLE",
|
|
2517
|
+
"DBX_TEST_FEATURE_SERVING_PIPELINE_ID"
|
|
2518
|
+
),
|
|
2519
|
+
requirement(
|
|
2520
|
+
"feature-assertion",
|
|
2521
|
+
"Feature freshness/correctness SQL",
|
|
2522
|
+
"DBX_TEST_FEATURE_ASSERTION_SQL"
|
|
2523
|
+
)
|
|
2524
|
+
],
|
|
2525
|
+
docsUrl: `${DOCS2}#feature-engineering-and-serving`,
|
|
2526
|
+
certificationScopes: [
|
|
2527
|
+
`${AZURE_USER_SCOPE}; Delta feature table, PostgreSQL-backed Synced Table, and SQL freshness`
|
|
2528
|
+
]
|
|
2529
|
+
});
|
|
2530
|
+
}
|
|
2531
|
+
function createModelServingTargetPack() {
|
|
2532
|
+
return defineTargetPack({
|
|
2533
|
+
id: "model-serving-gateway",
|
|
2534
|
+
name: "Model Serving and AI Gateway",
|
|
2535
|
+
description: "Serving readiness, inference behavior and optional AI Gateway policy presence.",
|
|
2536
|
+
version: "0.1.0",
|
|
2537
|
+
maturity: "live-gated",
|
|
2538
|
+
capabilities: [
|
|
2539
|
+
"model-serving.readiness",
|
|
2540
|
+
"model-serving.inference",
|
|
2541
|
+
"ai-gateway.configuration"
|
|
2542
|
+
],
|
|
2543
|
+
checks: () => [restrictedPrincipalCheck(), modelServingCheck()],
|
|
2544
|
+
requiredChecks: ["model-serving"],
|
|
2545
|
+
requirements: [
|
|
2546
|
+
workspace,
|
|
2547
|
+
auth,
|
|
2548
|
+
requirement("serving-endpoint", "Model Serving endpoint name", "DBX_TEST_SERVING_ENDPOINT"),
|
|
2549
|
+
requirement(
|
|
2550
|
+
"serving-request",
|
|
2551
|
+
"Safe inference request JSON",
|
|
2552
|
+
"DBX_TEST_SERVING_REQUEST_JSON"
|
|
2553
|
+
),
|
|
2554
|
+
requirement(
|
|
2555
|
+
"ai-gateway",
|
|
2556
|
+
"AI Gateway enforcement flag set to 1",
|
|
2557
|
+
"DBX_TEST_AI_GATEWAY_REQUIRED"
|
|
2558
|
+
)
|
|
2559
|
+
],
|
|
2560
|
+
docsUrl: `${DOCS2}#model-serving-and-ai-gateway`,
|
|
2561
|
+
certificationScopes: [
|
|
2562
|
+
`${AZURE_USER_SCOPE}; custom-model inference and AI Gateway inference-table configuration`
|
|
2563
|
+
]
|
|
2564
|
+
});
|
|
2565
|
+
}
|
|
2566
|
+
function createVectorRagAgentsTargetPack() {
|
|
2567
|
+
return defineTargetPack({
|
|
2568
|
+
id: "vector-rag-agents",
|
|
2569
|
+
name: "Vector Search, RAG and agents",
|
|
2570
|
+
description: "Vector endpoint/index readiness, retrieval behavior and agent endpoint inference.",
|
|
2571
|
+
version: "0.1.0",
|
|
2572
|
+
maturity: "live-gated",
|
|
2573
|
+
capabilities: [
|
|
2574
|
+
"vector-search.endpoint",
|
|
2575
|
+
"vector-search.index",
|
|
2576
|
+
"rag.retrieval",
|
|
2577
|
+
"agents.serving"
|
|
2578
|
+
],
|
|
2579
|
+
checks: () => [restrictedPrincipalCheck(), vectorSearchCheck(), ragAgentCheck()],
|
|
2580
|
+
requiredChecks: ["vector-search", "rag-agent"],
|
|
2581
|
+
requirements: [
|
|
2582
|
+
workspace,
|
|
2583
|
+
auth,
|
|
2584
|
+
requirement("vector-endpoint", "Vector Search endpoint", "DBX_TEST_VECTOR_ENDPOINT"),
|
|
2585
|
+
requirement("vector-index", "Vector Search index", "DBX_TEST_VECTOR_INDEX"),
|
|
2586
|
+
requirement("vector-query", "Vector retrieval request JSON", "DBX_TEST_VECTOR_QUERY_JSON"),
|
|
2587
|
+
requirement("agent-endpoint", "Agent serving endpoint", "DBX_TEST_AGENT_ENDPOINT"),
|
|
2588
|
+
requirement("agent-request", "Safe agent request JSON", "DBX_TEST_AGENT_REQUEST_JSON")
|
|
2589
|
+
],
|
|
2590
|
+
docsUrl: `${DOCS2}#vector-search-rag-and-agents`,
|
|
2591
|
+
certificationScopes: [
|
|
2592
|
+
`${AZURE_USER_SCOPE}; Delta Sync vector retrieval and custom PyFunc agent invocation`
|
|
2593
|
+
]
|
|
2594
|
+
});
|
|
2595
|
+
}
|
|
2596
|
+
function createFederationSharingTargetPack() {
|
|
2597
|
+
return defineTargetPack({
|
|
2598
|
+
id: "federation-sharing-cleanrooms",
|
|
2599
|
+
name: "Federation, Delta Sharing and Clean Rooms",
|
|
2600
|
+
description: "Federated connection, data-plane query, Delta Share and Clean Room governance access.",
|
|
2601
|
+
version: "0.1.0",
|
|
2602
|
+
maturity: "shipped",
|
|
2603
|
+
capabilities: [
|
|
2604
|
+
"federation.connections",
|
|
2605
|
+
"federation.query",
|
|
2606
|
+
"delta-sharing.shares",
|
|
2607
|
+
"clean-rooms.access"
|
|
2608
|
+
],
|
|
2609
|
+
checks: () => [restrictedPrincipalCheck(), federationSharingCleanRoomsCheck()],
|
|
2610
|
+
requiredChecks: ["federation-sharing-cleanrooms"],
|
|
2611
|
+
requirements: [
|
|
2612
|
+
workspace,
|
|
2613
|
+
auth,
|
|
2614
|
+
warehouse,
|
|
2615
|
+
requirement("connection", "Lakehouse Federation connection", "DBX_TEST_CONNECTION"),
|
|
2616
|
+
requirement("share", "Delta Sharing share", "DBX_TEST_SHARE"),
|
|
2617
|
+
requirement("clean-room", "Clean Room name", "DBX_TEST_CLEAN_ROOM"),
|
|
2618
|
+
requirement(
|
|
2619
|
+
"federation-assertion",
|
|
2620
|
+
"Federated data-plane SQL",
|
|
2621
|
+
"DBX_TEST_FEDERATION_ASSERTION_SQL"
|
|
2622
|
+
)
|
|
2623
|
+
],
|
|
2624
|
+
docsUrl: `${DOCS2}#federation-delta-sharing-and-clean-rooms`,
|
|
2625
|
+
certificationScopes: [SHIPPED_SCOPE]
|
|
2626
|
+
});
|
|
2627
|
+
}
|
|
2628
|
+
function createSecurityCostDrTargetPack() {
|
|
2629
|
+
return defineTargetPack({
|
|
2630
|
+
id: "security-cost-dr",
|
|
2631
|
+
name: "Networking, security posture, cost and regional DR",
|
|
2632
|
+
description: "IP controls, compute policy, cost guardrail and authenticated secondary-region readiness.",
|
|
2633
|
+
version: "0.1.0",
|
|
2634
|
+
maturity: "shipped",
|
|
2635
|
+
capabilities: [
|
|
2636
|
+
"network.ip-access",
|
|
2637
|
+
"security.compute-policy",
|
|
2638
|
+
"cost.system-tables",
|
|
2639
|
+
"dr.secondary-region"
|
|
2640
|
+
],
|
|
2641
|
+
checks: () => [restrictedPrincipalCheck(), securityPostureCostCheck(), regionalDrCheck()],
|
|
2642
|
+
requiredChecks: ["security-cost", "regional-dr"],
|
|
2643
|
+
requirements: [
|
|
2644
|
+
workspace,
|
|
2645
|
+
auth,
|
|
2646
|
+
warehouse,
|
|
2647
|
+
requirement(
|
|
2648
|
+
"ip-access-list",
|
|
2649
|
+
"Enabled workspace IP access list",
|
|
2650
|
+
"DBX_TEST_IP_ACCESS_LIST_ID"
|
|
2651
|
+
),
|
|
2652
|
+
requirement("cluster-policy", "Compute policy ID", "DBX_TEST_CLUSTER_POLICY_ID"),
|
|
2653
|
+
requirement("cost-assertion", "Boolean SQL cost guardrail", "DBX_TEST_COST_ASSERTION_SQL"),
|
|
2654
|
+
requirement("dr-workspace", "Secondary workspace host", "DBX_TEST_DR_HOST"),
|
|
2655
|
+
requirement("dr-warehouse", "Running secondary-region warehouse", "DBX_TEST_DR_WAREHOUSE_ID"),
|
|
2656
|
+
requirement("dr-assertion", "Secondary-region workload SQL", "DBX_TEST_DR_ASSERTION_SQL")
|
|
2657
|
+
],
|
|
2658
|
+
docsUrl: `${DOCS2}#networking-security-cost-and-regional-dr`,
|
|
2659
|
+
certificationScopes: [SHIPPED_SCOPE]
|
|
2660
|
+
});
|
|
2661
|
+
}
|
|
2662
|
+
function createAdvancedTargetPacks() {
|
|
2663
|
+
return [
|
|
2664
|
+
createAdvancedStreamingTargetPack(),
|
|
2665
|
+
createAiBiGenieTargetPack(),
|
|
2666
|
+
createMlflowLifecycleTargetPack(),
|
|
2667
|
+
createFeatureEngineeringTargetPack(),
|
|
2668
|
+
createModelServingTargetPack(),
|
|
2669
|
+
createVectorRagAgentsTargetPack(),
|
|
2670
|
+
createFederationSharingTargetPack(),
|
|
2671
|
+
createSecurityCostDrTargetPack()
|
|
2672
|
+
];
|
|
2673
|
+
}
|
|
2674
|
+
|
|
1593
2675
|
// src/builtin-packs.ts
|
|
1594
2676
|
function createBuiltinTargetPacks() {
|
|
1595
|
-
return [
|
|
2677
|
+
return [
|
|
2678
|
+
...createFoundationTargetPacks(),
|
|
2679
|
+
createLakeflowJobsTargetPack(),
|
|
2680
|
+
...createAdvancedTargetPacks()
|
|
2681
|
+
];
|
|
1596
2682
|
}
|
|
1597
2683
|
var builtinTargetPacks = createBuiltinTargetPacks();
|
|
1598
2684
|
|
|
1599
|
-
export { DatabricksJobsAdapter, appHealthCheck, assertDeltaContract, autoLoaderIngestionCheck, backupRestoreCheck, builtinTargetPacks, classicComputeCheck, createAppsOperationalFoundationPack, createBuiltinTargetPacks, createEphemeralLakebaseBranch, createExecutionContext, createFoundationTargetPacks, createJobsCodeFoundationPack, createLakebaseTestProvider, createLakeflowFoundationPack, createLakeflowJobsTargetPack, createMockDatabricksClient, createSqlDeltaFoundationPack, createTargetPackRegistry, createUnityStorageFoundationPack, customLiveCheck, dbtBuildCheck, defineLiveSuite, defineTargetPack, defineTargetPackRun, deleteEphemeralLakebaseBranch, deleteVolumeFile, deltaContractCheck, doctorTargetPack, ensureVolumeDirectory, expectQueryToMatchGolden, expectTable, failureRecoveryCheck, foundationTargetPacks, jobRunCheck, jobsOrchestrationCheck, lakebaseCredentialCheck, lakebaseRoundTripCheck, normalizeJobRun, normalizeVolumeRoot, notebookSubmitCheck, parseJobOrchestrationLiveSpec, performanceBudgetCheck, pipelineRefreshCheck, renderJUnit, resolveLakebaseProject, resolveProfile, restrictedPrincipalCheck, rollbackCheck, runLiveSuite, runTargetPack, secretRotationCheck, secretScopeCheck, serverlessComputeCheck, sqlRoundTripCheck, toLakebaseBranchId, unityCatalogAccessCheck, uploadVolumeFile, validateJobGraph, volumeIOCheck, waitForPipelineUpdate };
|
|
2685
|
+
export { DatabricksAdvancedWorkloadsAdapter, DatabricksJobsAdapter, advancedStreamingCdcCheck, aiBiDashboardCheck, appHealthCheck, assertDeltaContract, autoLoaderIngestionCheck, backupRestoreCheck, builtinTargetPacks, classicComputeCheck, createAdvancedStreamingTargetPack, createAdvancedTargetPacks, createAiBiGenieTargetPack, createAppsOperationalFoundationPack, createBuiltinTargetPacks, createEphemeralLakebaseBranch, createExecutionContext, createFeatureEngineeringTargetPack, createFederationSharingTargetPack, createFoundationTargetPacks, createJobsCodeFoundationPack, createLakebaseTestProvider, createLakeflowFoundationPack, createLakeflowJobsTargetPack, createMlflowLifecycleTargetPack, createMockDatabricksClient, createModelServingTargetPack, createSecurityCostDrTargetPack, createSqlDeltaFoundationPack, createTargetPackRegistry, createUnityStorageFoundationPack, createVectorRagAgentsTargetPack, customLiveCheck, dbtBuildCheck, defineLiveSuite, defineTargetPack, defineTargetPackRun, deleteEphemeralLakebaseBranch, deleteVolumeFile, deltaContractCheck, doctorTargetPack, ensureVolumeDirectory, expectQueryToMatchGolden, expectTable, failureRecoveryCheck, featureEngineeringServingCheck, federationSharingCleanRoomsCheck, foundationTargetPacks, genieCheck, jobRunCheck, jobsCertificationCheck, jobsOrchestrationCheck, lakebaseCredentialCheck, lakebaseRoundTripCheck, lakeflowConnectCheck, mlflowLifecycleCheck, modelServingCheck, normalizeJobRun, normalizeVolumeRoot, notebookSubmitCheck, parseJobOrchestrationLiveSpec, performanceBudgetCheck, pipelineRefreshCheck, ragAgentCheck, regionalDrCheck, renderJUnit, resolveLakebaseProject, resolveProfile, restrictedPrincipalCheck, rollbackCheck, runLiveSuite, runTargetPack, secretRotationCheck, secretScopeCheck, securityPostureCostCheck, serverlessComputeCheck, sqlRoundTripCheck, toLakebaseBranchId, unityCatalogAccessCheck, uploadVolumeFile, validateJobGraph, vectorSearchCheck, volumeIOCheck, waitForPipelineUpdate };
|
|
1600
2686
|
//# sourceMappingURL=index.js.map
|
|
1601
2687
|
//# sourceMappingURL=index.js.map
|