@fabricorg/databricks-testkit 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -2
- package/dist/index.cjs +1187 -77
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +118 -1
- package/dist/index.d.ts +118 -1
- package/dist/index.js +1164 -78
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -600,15 +600,15 @@ function defineLiveSuite(spec) {
|
|
|
600
600
|
async function runLiveSuite(spec, runtime = {}) {
|
|
601
601
|
const env = spec.env ?? process.env;
|
|
602
602
|
if (env.DBX_TEST_LIVE !== "1") return void 0;
|
|
603
|
-
const
|
|
603
|
+
const required3 = [...spec.required ?? []];
|
|
604
604
|
const known = new Set(spec.checks.map((check) => check.id));
|
|
605
|
-
for (const id of
|
|
605
|
+
for (const id of required3) {
|
|
606
606
|
if (!known.has(id)) throw new Error(`Required live check '${id}' is not defined`);
|
|
607
607
|
}
|
|
608
608
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
609
609
|
const ownedWarehouse = runtime.warehouse === void 0;
|
|
610
610
|
const client = runtime.client ?? createClientFromEnv(env);
|
|
611
|
-
const
|
|
611
|
+
const warehouse2 = runtime.warehouse ?? (hasWarehouseConfig(env) ? await createWorkspaceContext({ env, client }) : unavailableWarehouseContext());
|
|
612
612
|
const results = [];
|
|
613
613
|
try {
|
|
614
614
|
for (const check of spec.checks) {
|
|
@@ -623,7 +623,7 @@ async function runLiveSuite(spec, runtime = {}) {
|
|
|
623
623
|
}
|
|
624
624
|
const before = performance.now();
|
|
625
625
|
try {
|
|
626
|
-
const details = await check.run({ env, warehouse, client });
|
|
626
|
+
const details = await check.run({ env, warehouse: warehouse2, client });
|
|
627
627
|
results.push({
|
|
628
628
|
id: check.id,
|
|
629
629
|
description: check.description,
|
|
@@ -642,9 +642,9 @@ async function runLiveSuite(spec, runtime = {}) {
|
|
|
642
642
|
}
|
|
643
643
|
}
|
|
644
644
|
} finally {
|
|
645
|
-
if (ownedWarehouse) await
|
|
645
|
+
if (ownedWarehouse) await warehouse2.close();
|
|
646
646
|
}
|
|
647
|
-
const success =
|
|
647
|
+
const success = required3.every(
|
|
648
648
|
(id) => results.find((result) => result.id === id)?.status === "pass"
|
|
649
649
|
);
|
|
650
650
|
const evidence = {
|
|
@@ -653,7 +653,7 @@ async function runLiveSuite(spec, runtime = {}) {
|
|
|
653
653
|
startedAt,
|
|
654
654
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
655
655
|
success,
|
|
656
|
-
required:
|
|
656
|
+
required: required3,
|
|
657
657
|
results,
|
|
658
658
|
...spec.targetPack ? { targetPack: spec.targetPack } : {}
|
|
659
659
|
};
|
|
@@ -756,8 +756,8 @@ function sqlRoundTripCheck() {
|
|
|
756
756
|
id: "sql",
|
|
757
757
|
description: "SQL Statement Execution round-trip",
|
|
758
758
|
configured: (env) => Boolean(env.DATABRICKS_WAREHOUSE_ID || env.DATABRICKS_HTTP_PATH),
|
|
759
|
-
async run({ warehouse }) {
|
|
760
|
-
const rows = await
|
|
759
|
+
async run({ warehouse: warehouse2 }) {
|
|
760
|
+
const rows = await warehouse2.query("SELECT 1 AS dbx_test_ok");
|
|
761
761
|
if (Number(rows[0]?.dbx_test_ok) !== 1)
|
|
762
762
|
throw new Error("SQL round-trip returned an unexpected value");
|
|
763
763
|
return { rows: rows.length };
|
|
@@ -801,9 +801,9 @@ function deltaContractCheck(tableEnv = "DBX_TEST_DELTA_TABLE", columns = []) {
|
|
|
801
801
|
id: "delta-contract",
|
|
802
802
|
description: "Delta table schema contract",
|
|
803
803
|
configured: (env) => Boolean(env[tableEnv]),
|
|
804
|
-
async run({ env, warehouse }) {
|
|
804
|
+
async run({ env, warehouse: warehouse2 }) {
|
|
805
805
|
const table = env[tableEnv];
|
|
806
|
-
const rows = await
|
|
806
|
+
const rows = await warehouse2.query(`DESCRIBE TABLE ${table}`);
|
|
807
807
|
assertDeltaContract(table, rows, columns);
|
|
808
808
|
return { table, columns: rows.length };
|
|
809
809
|
}
|
|
@@ -814,10 +814,10 @@ function unityCatalogAccessCheck() {
|
|
|
814
814
|
id: "uc-grants",
|
|
815
815
|
description: "Unity Catalog allow/deny access assertions",
|
|
816
816
|
configured: (env) => Boolean(env.DBX_TEST_UC_ALLOWED_TABLE && env.DBX_TEST_UC_DENIED_TABLE),
|
|
817
|
-
async run({ env, warehouse }) {
|
|
818
|
-
await
|
|
817
|
+
async run({ env, warehouse: warehouse2 }) {
|
|
818
|
+
await warehouse2.query(`SELECT * FROM ${env.DBX_TEST_UC_ALLOWED_TABLE} LIMIT 1`);
|
|
819
819
|
try {
|
|
820
|
-
await
|
|
820
|
+
await warehouse2.query(`SELECT * FROM ${env.DBX_TEST_UC_DENIED_TABLE} LIMIT 1`);
|
|
821
821
|
} catch (error) {
|
|
822
822
|
if (/permission.?denied|insufficient.?priv|403|unauthoriz/i.test(String(error))) {
|
|
823
823
|
return { allowed: env.DBX_TEST_UC_ALLOWED_TABLE, denied: env.DBX_TEST_UC_DENIED_TABLE };
|
|
@@ -1282,7 +1282,7 @@ function autoLoaderIngestionCheck(options = {}) {
|
|
|
1282
1282
|
configured: (env) => Boolean(
|
|
1283
1283
|
env.DBX_TEST_AUTOLOADER_PIPELINE_ID && env.DBX_TEST_AUTOLOADER_TABLE && env.DBX_TEST_AUTOLOADER_FIXTURE && env.DBX_TEST_VOLUME
|
|
1284
1284
|
),
|
|
1285
|
-
async run({ env, client, warehouse }) {
|
|
1285
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
1286
1286
|
const pipelineId = env.DBX_TEST_AUTOLOADER_PIPELINE_ID;
|
|
1287
1287
|
if (pipelineId === env.DBX_TEST_PIPELINE_ID) {
|
|
1288
1288
|
throw new Error("DBX_TEST_AUTOLOADER_PIPELINE_ID must not be the production pipeline");
|
|
@@ -1307,7 +1307,7 @@ function autoLoaderIngestionCheck(options = {}) {
|
|
|
1307
1307
|
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
1308
1308
|
});
|
|
1309
1309
|
const table = env.DBX_TEST_AUTOLOADER_TABLE;
|
|
1310
|
-
const rows = await
|
|
1310
|
+
const rows = await warehouse2.query(
|
|
1311
1311
|
`SELECT COUNT(*) AS n, SUM(CASE WHEN _rescued_data IS NOT NULL THEN 1 ELSE 0 END) AS rescued FROM ${table}`
|
|
1312
1312
|
);
|
|
1313
1313
|
const count = Number(rows[0]?.n ?? 0);
|
|
@@ -1330,7 +1330,7 @@ function performanceBudgetCheck() {
|
|
|
1330
1330
|
id: "performance",
|
|
1331
1331
|
description: "SQL Statement Execution p95 stays within the configured latency budget",
|
|
1332
1332
|
configured: (env) => Boolean(env.DBX_TEST_PERFORMANCE_P95_MS),
|
|
1333
|
-
async run({ env, warehouse }) {
|
|
1333
|
+
async run({ env, warehouse: warehouse2 }) {
|
|
1334
1334
|
const budgetMs = Number(env.DBX_TEST_PERFORMANCE_P95_MS);
|
|
1335
1335
|
const runs = Number(env.DBX_TEST_PERFORMANCE_RUNS ?? 7);
|
|
1336
1336
|
if (!Number.isFinite(budgetMs) || budgetMs <= 0) {
|
|
@@ -1339,11 +1339,11 @@ function performanceBudgetCheck() {
|
|
|
1339
1339
|
if (!Number.isInteger(runs) || runs < 3 || runs > 100) {
|
|
1340
1340
|
throw new Error("DBX_TEST_PERFORMANCE_RUNS must be an integer from 3 to 100");
|
|
1341
1341
|
}
|
|
1342
|
-
await
|
|
1342
|
+
await warehouse2.query("SELECT 1 AS warmup");
|
|
1343
1343
|
const durations = [];
|
|
1344
1344
|
for (let index = 0; index < runs; index += 1) {
|
|
1345
1345
|
const started = performance.now();
|
|
1346
|
-
await
|
|
1346
|
+
await warehouse2.query("SELECT 1 AS performance_probe");
|
|
1347
1347
|
durations.push(performance.now() - started);
|
|
1348
1348
|
}
|
|
1349
1349
|
durations.sort((left, right) => left - right);
|
|
@@ -1365,15 +1365,15 @@ function failureRecoveryCheck() {
|
|
|
1365
1365
|
id: "failure-recovery",
|
|
1366
1366
|
description: "A failed statement is isolated and a subsequent statement recovers",
|
|
1367
1367
|
configured: (env) => Boolean(env.DATABRICKS_WAREHOUSE_ID || env.DATABRICKS_HTTP_PATH),
|
|
1368
|
-
async run({ warehouse }) {
|
|
1368
|
+
async run({ warehouse: warehouse2 }) {
|
|
1369
1369
|
let failedAsExpected = false;
|
|
1370
1370
|
try {
|
|
1371
|
-
await
|
|
1371
|
+
await warehouse2.query("SELECT * FROM __fabric_experiments_expected_missing_table__");
|
|
1372
1372
|
} catch {
|
|
1373
1373
|
failedAsExpected = true;
|
|
1374
1374
|
}
|
|
1375
1375
|
if (!failedAsExpected) throw new Error("Injected failure unexpectedly succeeded");
|
|
1376
|
-
const rows = await
|
|
1376
|
+
const rows = await warehouse2.query("SELECT 1 AS recovered");
|
|
1377
1377
|
if (Number(rows[0]?.recovered) !== 1) {
|
|
1378
1378
|
throw new Error("Statement execution did not recover after the injected failure");
|
|
1379
1379
|
}
|
|
@@ -1411,20 +1411,20 @@ function backupRestoreCheck() {
|
|
|
1411
1411
|
id: "backup-restore",
|
|
1412
1412
|
description: "Delta DEEP CLONE backup restores a dropped scratch table",
|
|
1413
1413
|
configured: (env) => Boolean(env.DATABRICKS_WAREHOUSE_ID || env.DATABRICKS_HTTP_PATH),
|
|
1414
|
-
async run({ warehouse }) {
|
|
1415
|
-
const source =
|
|
1416
|
-
const backup =
|
|
1414
|
+
async run({ warehouse: warehouse2 }) {
|
|
1415
|
+
const source = warehouse2.qual(safeName("backup_source"));
|
|
1416
|
+
const backup = warehouse2.qual(safeName("backup_clone"));
|
|
1417
1417
|
try {
|
|
1418
|
-
await
|
|
1419
|
-
await
|
|
1420
|
-
await
|
|
1421
|
-
await
|
|
1422
|
-
const rows = await
|
|
1418
|
+
await warehouse2.exec(`CREATE TABLE ${source} USING DELTA AS SELECT 7 AS value`);
|
|
1419
|
+
await warehouse2.exec(`CREATE TABLE ${backup} DEEP CLONE ${source}`);
|
|
1420
|
+
await warehouse2.exec(`DROP TABLE ${source}`);
|
|
1421
|
+
await warehouse2.exec(`CREATE TABLE ${source} DEEP CLONE ${backup}`);
|
|
1422
|
+
const rows = await warehouse2.query(`SELECT value FROM ${source}`);
|
|
1423
1423
|
if (Number(rows[0]?.value) !== 7) throw new Error("Restored Delta table lost its data");
|
|
1424
1424
|
return { restoredRows: rows.length, deepClone: true };
|
|
1425
1425
|
} finally {
|
|
1426
|
-
await
|
|
1427
|
-
await
|
|
1426
|
+
await warehouse2.exec(`DROP TABLE IF EXISTS ${source}`).catch(() => void 0);
|
|
1427
|
+
await warehouse2.exec(`DROP TABLE IF EXISTS ${backup}`).catch(() => void 0);
|
|
1428
1428
|
}
|
|
1429
1429
|
}
|
|
1430
1430
|
};
|
|
@@ -1434,25 +1434,25 @@ function rollbackCheck() {
|
|
|
1434
1434
|
id: "rollback",
|
|
1435
1435
|
description: "Delta time-travel rollback restores a known-good table version",
|
|
1436
1436
|
configured: (env) => Boolean(env.DATABRICKS_WAREHOUSE_ID || env.DATABRICKS_HTTP_PATH),
|
|
1437
|
-
async run({ warehouse }) {
|
|
1438
|
-
const table =
|
|
1437
|
+
async run({ warehouse: warehouse2 }) {
|
|
1438
|
+
const table = warehouse2.qual(safeName("rollback_probe"));
|
|
1439
1439
|
try {
|
|
1440
|
-
await
|
|
1441
|
-
await
|
|
1442
|
-
const history = await
|
|
1440
|
+
await warehouse2.exec(`CREATE TABLE ${table} (value INT) USING DELTA`);
|
|
1441
|
+
await warehouse2.exec(`INSERT INTO ${table} VALUES (1)`);
|
|
1442
|
+
const history = await warehouse2.query(`DESCRIBE HISTORY ${table} LIMIT 1`);
|
|
1443
1443
|
const knownGoodVersion = Number(history[0]?.version);
|
|
1444
1444
|
if (!Number.isInteger(knownGoodVersion)) {
|
|
1445
1445
|
throw new Error("Delta history did not return a known-good version");
|
|
1446
1446
|
}
|
|
1447
|
-
await
|
|
1448
|
-
await
|
|
1449
|
-
const rows = await
|
|
1447
|
+
await warehouse2.exec(`INSERT INTO ${table} VALUES (999)`);
|
|
1448
|
+
await warehouse2.exec(`RESTORE TABLE ${table} TO VERSION AS OF ${knownGoodVersion}`);
|
|
1449
|
+
const rows = await warehouse2.query(`SELECT value FROM ${table} ORDER BY value`);
|
|
1450
1450
|
if (rows.length !== 1 || Number(rows[0]?.value) !== 1) {
|
|
1451
1451
|
throw new Error("Delta rollback did not restore the known-good contents");
|
|
1452
1452
|
}
|
|
1453
1453
|
return { knownGoodVersion, restoredRows: rows.length };
|
|
1454
1454
|
} finally {
|
|
1455
|
-
await
|
|
1455
|
+
await warehouse2.exec(`DROP TABLE IF EXISTS ${table}`).catch(() => void 0);
|
|
1456
1456
|
}
|
|
1457
1457
|
}
|
|
1458
1458
|
};
|
|
@@ -1462,13 +1462,13 @@ function serverlessComputeCheck() {
|
|
|
1462
1462
|
id: "azure-serverless",
|
|
1463
1463
|
description: "Azure Databricks serverless SQL warehouse is enabled and executes SQL",
|
|
1464
1464
|
configured: (env) => Boolean(env.DATABRICKS_WAREHOUSE_ID),
|
|
1465
|
-
async run({ env, client, warehouse }) {
|
|
1465
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
1466
1466
|
const id = env.DATABRICKS_WAREHOUSE_ID;
|
|
1467
1467
|
const details = await client.get(`/api/2.0/sql/warehouses/${encodeURIComponent(id)}`);
|
|
1468
1468
|
if (details.enable_serverless_compute !== true) {
|
|
1469
1469
|
throw new Error(`SQL warehouse ${id} is not configured for serverless compute`);
|
|
1470
1470
|
}
|
|
1471
|
-
const rows = await
|
|
1471
|
+
const rows = await warehouse2.query("SELECT current_catalog() AS catalog");
|
|
1472
1472
|
return { warehouseId: id, name: details.name, state: details.state, rows: rows.length };
|
|
1473
1473
|
}
|
|
1474
1474
|
};
|
|
@@ -1563,9 +1563,9 @@ function defineTargetPack(spec) {
|
|
|
1563
1563
|
}
|
|
1564
1564
|
ids.add(check.id);
|
|
1565
1565
|
}
|
|
1566
|
-
for (const
|
|
1567
|
-
if (!ids.has(
|
|
1568
|
-
throw new Error(`Target pack '${spec.id}' requires unknown check '${
|
|
1566
|
+
for (const required3 of spec.requiredChecks ?? []) {
|
|
1567
|
+
if (!ids.has(required3)) {
|
|
1568
|
+
throw new Error(`Target pack '${spec.id}' requires unknown check '${required3}'`);
|
|
1569
1569
|
}
|
|
1570
1570
|
}
|
|
1571
1571
|
return checks;
|
|
@@ -1589,25 +1589,25 @@ function createTargetPackRegistry(packs) {
|
|
|
1589
1589
|
return registry;
|
|
1590
1590
|
}
|
|
1591
1591
|
function doctorTargetPack(pack, env = process.env, requiredChecks = pack.requiredChecks ?? []) {
|
|
1592
|
-
const requirements = (pack.requirements ?? []).map((
|
|
1593
|
-
id:
|
|
1594
|
-
description:
|
|
1595
|
-
satisfied:
|
|
1596
|
-
required:
|
|
1597
|
-
variables: [...
|
|
1592
|
+
const requirements = (pack.requirements ?? []).map((requirement2) => ({
|
|
1593
|
+
id: requirement2.id,
|
|
1594
|
+
description: requirement2.description,
|
|
1595
|
+
satisfied: requirement2.anyOf.some((name) => Boolean(env[name]?.trim())),
|
|
1596
|
+
required: requirement2.required !== false,
|
|
1597
|
+
variables: [...requirement2.anyOf]
|
|
1598
1598
|
}));
|
|
1599
|
-
const
|
|
1599
|
+
const required3 = new Set(requiredChecks);
|
|
1600
1600
|
const checks = pack.checks().map((check) => ({
|
|
1601
1601
|
id: check.id,
|
|
1602
1602
|
description: check.description,
|
|
1603
1603
|
configured: check.configured ? check.configured(env) : true,
|
|
1604
|
-
required:
|
|
1604
|
+
required: required3.has(check.id)
|
|
1605
1605
|
}));
|
|
1606
1606
|
const known = new Set(checks.map((check) => check.id));
|
|
1607
|
-
const missingRequirements = requirements.filter((
|
|
1607
|
+
const missingRequirements = requirements.filter((requirement2) => requirement2.required && !requirement2.satisfied).map((requirement2) => requirement2.id);
|
|
1608
1608
|
const missingRequiredChecks = [
|
|
1609
1609
|
...checks.filter((check) => check.required && !check.configured).map((check) => check.id),
|
|
1610
|
-
...[...
|
|
1610
|
+
...[...required3].filter((id) => !known.has(id))
|
|
1611
1611
|
];
|
|
1612
1612
|
return {
|
|
1613
1613
|
pack: {
|
|
@@ -1722,7 +1722,7 @@ function createSqlDeltaFoundationPack() {
|
|
|
1722
1722
|
],
|
|
1723
1723
|
requiredChecks: ["sql"],
|
|
1724
1724
|
requirements: [workspaceRequirement, authRequirement, warehouseRequirement],
|
|
1725
|
-
docsUrl: `${DOCS}#
|
|
1725
|
+
docsUrl: `${DOCS}#sql-and-delta-foundation`,
|
|
1726
1726
|
certificationScopes: [AZURE_SCOPE]
|
|
1727
1727
|
});
|
|
1728
1728
|
}
|
|
@@ -1747,7 +1747,7 @@ function createLakeflowFoundationPack() {
|
|
|
1747
1747
|
],
|
|
1748
1748
|
requiredChecks: ["pipeline"],
|
|
1749
1749
|
requirements: [workspaceRequirement, authRequirement],
|
|
1750
|
-
docsUrl: `${DOCS}#
|
|
1750
|
+
docsUrl: `${DOCS}#lakeflow-ingestion-foundation`,
|
|
1751
1751
|
certificationScopes: [AZURE_SCOPE]
|
|
1752
1752
|
});
|
|
1753
1753
|
}
|
|
@@ -1775,7 +1775,7 @@ function createJobsCodeFoundationPack() {
|
|
|
1775
1775
|
],
|
|
1776
1776
|
requiredChecks: ["jobs"],
|
|
1777
1777
|
requirements: [workspaceRequirement, authRequirement],
|
|
1778
|
-
docsUrl: `${DOCS}#
|
|
1778
|
+
docsUrl: `${DOCS}#jobs-and-code-foundation`,
|
|
1779
1779
|
certificationScopes: [AZURE_SCOPE]
|
|
1780
1780
|
});
|
|
1781
1781
|
}
|
|
@@ -1801,7 +1801,7 @@ function createUnityStorageFoundationPack() {
|
|
|
1801
1801
|
],
|
|
1802
1802
|
requiredChecks: ["uc-grants"],
|
|
1803
1803
|
requirements: [workspaceRequirement, authRequirement, warehouseRequirement],
|
|
1804
|
-
docsUrl: `${DOCS}#
|
|
1804
|
+
docsUrl: `${DOCS}#unity-catalog-and-storage-foundation`,
|
|
1805
1805
|
certificationScopes: [AZURE_SCOPE]
|
|
1806
1806
|
});
|
|
1807
1807
|
}
|
|
@@ -1827,7 +1827,7 @@ function createAppsOperationalFoundationPack() {
|
|
|
1827
1827
|
],
|
|
1828
1828
|
requiredChecks: ["app"],
|
|
1829
1829
|
requirements: [workspaceRequirement, authRequirement],
|
|
1830
|
-
docsUrl: `${DOCS}#
|
|
1830
|
+
docsUrl: `${DOCS}#apps-and-operational-data-foundation`,
|
|
1831
1831
|
certificationScopes: [AZURE_SCOPE]
|
|
1832
1832
|
});
|
|
1833
1833
|
}
|
|
@@ -1858,7 +1858,8 @@ var DatabricksJobsAdapter = class {
|
|
|
1858
1858
|
async get(runId) {
|
|
1859
1859
|
const raw = await this.client.get("/api/2.1/jobs/runs/get", {
|
|
1860
1860
|
run_id: runId,
|
|
1861
|
-
include_history: true
|
|
1861
|
+
include_history: true,
|
|
1862
|
+
include_resolved_values: true
|
|
1862
1863
|
});
|
|
1863
1864
|
return normalizeJobRun(runId, raw);
|
|
1864
1865
|
}
|
|
@@ -1872,15 +1873,46 @@ var DatabricksJobsAdapter = class {
|
|
|
1872
1873
|
async cancel(runId) {
|
|
1873
1874
|
await this.client.post("/api/2.1/jobs/runs/cancel", { run_id: runId });
|
|
1874
1875
|
}
|
|
1876
|
+
async waitUntilActive(runId, options = {}) {
|
|
1877
|
+
const timeoutMs = options.timeoutMs ?? 3e5;
|
|
1878
|
+
const pollIntervalMs = options.pollIntervalMs ?? 2e3;
|
|
1879
|
+
const deadline = Date.now() + timeoutMs;
|
|
1880
|
+
while (Date.now() < deadline) {
|
|
1881
|
+
const run = await this.get(runId);
|
|
1882
|
+
if (run.lifeCycleState === "RUNNING") return run;
|
|
1883
|
+
if (isTerminal(run.lifeCycleState)) {
|
|
1884
|
+
throw new Error(
|
|
1885
|
+
`Databricks run ${runId} reached ${run.lifeCycleState}/${run.resultState} before it could be canceled`
|
|
1886
|
+
);
|
|
1887
|
+
}
|
|
1888
|
+
await delay2(pollIntervalMs);
|
|
1889
|
+
}
|
|
1890
|
+
throw new Error(`Databricks run ${runId} did not become active within ${timeoutMs}ms`);
|
|
1891
|
+
}
|
|
1875
1892
|
async repair(runId, options = {}) {
|
|
1876
1893
|
const response = await this.client.post("/api/2.1/jobs/runs/repair", {
|
|
1877
1894
|
run_id: runId,
|
|
1878
1895
|
...options.latestRepairId ? { latest_repair_id: options.latestRepairId } : {},
|
|
1896
|
+
...options.rerunDependentTasks ? { rerun_dependent_tasks: true } : {},
|
|
1879
1897
|
...options.rerunTasks?.length ? { rerun_tasks: options.rerunTasks } : { rerun_all_failed_tasks: true }
|
|
1880
1898
|
});
|
|
1881
1899
|
if (!response.repair_id) throw new Error("Databricks Jobs repair returned no repair_id");
|
|
1882
1900
|
return response.repair_id;
|
|
1883
1901
|
}
|
|
1902
|
+
async waitForRepair(runId, repairId, options = {}) {
|
|
1903
|
+
const timeoutMs = options.timeoutMs ?? 9e5;
|
|
1904
|
+
const pollIntervalMs = options.pollIntervalMs ?? 5e3;
|
|
1905
|
+
const deadline = Date.now() + timeoutMs;
|
|
1906
|
+
while (Date.now() < deadline) {
|
|
1907
|
+
const run = await this.get(runId);
|
|
1908
|
+
const repair = run.repairs.find((candidate) => candidate.repairId === repairId);
|
|
1909
|
+
if (repair && isTerminal(repair.lifeCycleState)) return run;
|
|
1910
|
+
await delay2(pollIntervalMs);
|
|
1911
|
+
}
|
|
1912
|
+
throw new Error(
|
|
1913
|
+
`Databricks repair ${repairId} for run ${runId} timed out after ${timeoutMs}ms`
|
|
1914
|
+
);
|
|
1915
|
+
}
|
|
1884
1916
|
};
|
|
1885
1917
|
function validateJobGraph(tasks) {
|
|
1886
1918
|
if (tasks.length === 0) throw new Error("Jobs orchestration requires at least one task");
|
|
@@ -1926,16 +1958,20 @@ function jobsOrchestrationCheck() {
|
|
|
1926
1958
|
const configured = env.DBX_TEST_JOBS_ORCHESTRATION_JSON;
|
|
1927
1959
|
if (!configured) throw new Error("DBX_TEST_JOBS_ORCHESTRATION_JSON is required");
|
|
1928
1960
|
const spec = parseJobOrchestrationLiveSpec(configured);
|
|
1929
|
-
const
|
|
1930
|
-
const runId = await
|
|
1931
|
-
let run = await
|
|
1961
|
+
const adapter2 = new DatabricksJobsAdapter(client);
|
|
1962
|
+
const runId = await adapter2.submit(spec.request);
|
|
1963
|
+
let run = await adapter2.wait(runId, {
|
|
1932
1964
|
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
1933
1965
|
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
1934
1966
|
});
|
|
1967
|
+
let repairId;
|
|
1935
1968
|
if (spec.repairFailedTasks && run.resultState !== "SUCCESS") {
|
|
1936
1969
|
const failed = run.tasks.filter((task) => task.resultState === "FAILED").map((task) => task.taskKey);
|
|
1937
|
-
await
|
|
1938
|
-
|
|
1970
|
+
repairId = await adapter2.repair(runId, {
|
|
1971
|
+
rerunTasks: failed.length > 0 ? failed : void 0,
|
|
1972
|
+
rerunDependentTasks: true
|
|
1973
|
+
});
|
|
1974
|
+
run = await adapter2.waitForRepair(runId, repairId, {
|
|
1939
1975
|
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
1940
1976
|
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
1941
1977
|
});
|
|
@@ -1953,10 +1989,32 @@ function jobsOrchestrationCheck() {
|
|
|
1953
1989
|
throw new Error(`Task '${taskKey}' ended in ${task.resultState}, expected ${expected}`);
|
|
1954
1990
|
}
|
|
1955
1991
|
}
|
|
1992
|
+
let cancellation;
|
|
1993
|
+
if (spec.cancellation) {
|
|
1994
|
+
const cancellationRunId = await adapter2.submit(spec.cancellation.request);
|
|
1995
|
+
await adapter2.waitUntilActive(cancellationRunId, {
|
|
1996
|
+
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
1997
|
+
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
1998
|
+
});
|
|
1999
|
+
await adapter2.cancel(cancellationRunId);
|
|
2000
|
+
const canceled = await adapter2.wait(cancellationRunId, {
|
|
2001
|
+
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
2002
|
+
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
2003
|
+
});
|
|
2004
|
+
const expectedCancellation = spec.cancellation.expectedResult ?? "CANCELED";
|
|
2005
|
+
if (canceled.resultState !== expectedCancellation) {
|
|
2006
|
+
throw new Error(
|
|
2007
|
+
`Cancellation run ${cancellationRunId} ended in ${canceled.resultState}, expected ${expectedCancellation}`
|
|
2008
|
+
);
|
|
2009
|
+
}
|
|
2010
|
+
cancellation = { runId: cancellationRunId, resultState: canceled.resultState };
|
|
2011
|
+
}
|
|
1956
2012
|
return {
|
|
1957
2013
|
runId,
|
|
2014
|
+
...repairId ? { repairId } : {},
|
|
1958
2015
|
resultState: run.resultState,
|
|
1959
|
-
tasks: run.tasks
|
|
2016
|
+
tasks: run.tasks,
|
|
2017
|
+
...cancellation ? { cancellation } : {}
|
|
1960
2018
|
};
|
|
1961
2019
|
}
|
|
1962
2020
|
};
|
|
@@ -1976,11 +2034,24 @@ function parseJobOrchestrationLiveSpec(value) {
|
|
|
1976
2034
|
throw new Error("Jobs orchestration spec requires request.tasks");
|
|
1977
2035
|
}
|
|
1978
2036
|
validateJobGraph(spec.request.tasks);
|
|
2037
|
+
if (spec.cancellation) {
|
|
2038
|
+
if (typeof spec.cancellation !== "object" || !spec.cancellation.request || !Array.isArray(spec.cancellation.request.tasks)) {
|
|
2039
|
+
throw new Error("Jobs orchestration cancellation requires request.tasks");
|
|
2040
|
+
}
|
|
2041
|
+
validateJobGraph(spec.cancellation.request.tasks);
|
|
2042
|
+
}
|
|
1979
2043
|
return spec;
|
|
1980
2044
|
}
|
|
1981
2045
|
function normalizeJobRun(fallbackRunId, raw) {
|
|
1982
2046
|
const state = objectOf(raw.state);
|
|
1983
|
-
const
|
|
2047
|
+
const repairEntries = Array.isArray(raw.repair_history) ? raw.repair_history.filter(
|
|
2048
|
+
(repair) => Boolean(repair && typeof repair === "object")
|
|
2049
|
+
) : [];
|
|
2050
|
+
const taskRunRanks = /* @__PURE__ */ new Map();
|
|
2051
|
+
for (const [rank, repair] of repairEntries.entries()) {
|
|
2052
|
+
for (const runId of numberArray(repair.task_run_ids)) taskRunRanks.set(runId, rank);
|
|
2053
|
+
}
|
|
2054
|
+
const taskAttempts = Array.isArray(raw.tasks) ? raw.tasks.filter(
|
|
1984
2055
|
(task) => Boolean(task && typeof task === "object")
|
|
1985
2056
|
).map((task) => {
|
|
1986
2057
|
const taskState = objectOf(task.state);
|
|
@@ -1989,20 +2060,236 @@ function normalizeJobRun(fallbackRunId, raw) {
|
|
|
1989
2060
|
...Number.isFinite(Number(task.run_id)) ? { runId: Number(task.run_id) } : {},
|
|
1990
2061
|
lifeCycleState: String(taskState.life_cycle_state ?? "UNKNOWN"),
|
|
1991
2062
|
resultState: String(taskState.result_state ?? "UNKNOWN"),
|
|
1992
|
-
...typeof taskState.state_message === "string" ? { stateMessage: taskState.state_message } : {}
|
|
2063
|
+
...typeof taskState.state_message === "string" ? { stateMessage: taskState.state_message } : {},
|
|
2064
|
+
...Number.isFinite(Number(task.attempt_number)) ? { attemptNumber: Number(task.attempt_number) } : {}
|
|
1993
2065
|
};
|
|
1994
2066
|
}) : [];
|
|
2067
|
+
const latestTasks = /* @__PURE__ */ new Map();
|
|
2068
|
+
for (const task of taskAttempts) {
|
|
2069
|
+
const current = latestTasks.get(task.taskKey);
|
|
2070
|
+
const currentAttempt = current?.attemptNumber ?? -1;
|
|
2071
|
+
const candidateAttempt = task.attemptNumber ?? -1;
|
|
2072
|
+
const currentRank = current?.runId ? taskRunRanks.get(current.runId) ?? -1 : -1;
|
|
2073
|
+
const candidateRank = task.runId ? taskRunRanks.get(task.runId) ?? -1 : -1;
|
|
2074
|
+
if (!current || candidateAttempt > currentAttempt || candidateAttempt === currentAttempt && candidateRank >= currentRank) {
|
|
2075
|
+
latestTasks.set(task.taskKey, task);
|
|
2076
|
+
}
|
|
2077
|
+
}
|
|
2078
|
+
const tasks = [...latestTasks.values()];
|
|
2079
|
+
const repairs = repairEntries.map((repair) => {
|
|
2080
|
+
const repairState = objectOf(repair.state);
|
|
2081
|
+
return {
|
|
2082
|
+
repairId: Number(repair.id ?? repair.repair_id ?? 0),
|
|
2083
|
+
type: String(repair.type ?? "UNKNOWN"),
|
|
2084
|
+
lifeCycleState: String(repairState.life_cycle_state ?? "UNKNOWN"),
|
|
2085
|
+
resultState: String(repairState.result_state ?? "UNKNOWN"),
|
|
2086
|
+
taskRunIds: numberArray(repair.task_run_ids)
|
|
2087
|
+
};
|
|
2088
|
+
});
|
|
1995
2089
|
return {
|
|
1996
2090
|
runId: Number(raw.run_id ?? fallbackRunId),
|
|
1997
2091
|
lifeCycleState: String(state.life_cycle_state ?? "UNKNOWN"),
|
|
1998
2092
|
resultState: String(state.result_state ?? "UNKNOWN"),
|
|
1999
2093
|
tasks,
|
|
2094
|
+
repairs,
|
|
2000
2095
|
raw
|
|
2001
2096
|
};
|
|
2002
2097
|
}
|
|
2098
|
+
function isTerminal(state) {
|
|
2099
|
+
return ["TERMINATED", "SKIPPED", "INTERNAL_ERROR", "BLOCKED"].includes(state);
|
|
2100
|
+
}
|
|
2101
|
+
function delay2(ms) {
|
|
2102
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
2103
|
+
}
|
|
2003
2104
|
function objectOf(value) {
|
|
2004
2105
|
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
2005
2106
|
}
|
|
2107
|
+
function numberArray(value) {
|
|
2108
|
+
return Array.isArray(value) ? value.map(Number).filter((candidate) => Number.isFinite(candidate)) : [];
|
|
2109
|
+
}
|
|
2110
|
+
|
|
2111
|
+
// src/jobs-certification-check.ts
|
|
2112
|
+
var NOTEBOOKS = {
|
|
2113
|
+
seed: `# Databricks notebook source
|
|
2114
|
+
dbutils.jobs.taskValues.set(key="branch", value="run")
|
|
2115
|
+
dbutils.jobs.taskValues.set(key="items", value=["alpha", "beta"])
|
|
2116
|
+
dbutils.notebook.exit("seeded")
|
|
2117
|
+
`,
|
|
2118
|
+
process: `# Databricks notebook source
|
|
2119
|
+
dbutils.widgets.text("item", "")
|
|
2120
|
+
item = dbutils.widgets.get("item")
|
|
2121
|
+
assert item, "item parameter is required"
|
|
2122
|
+
dbutils.notebook.exit(item)
|
|
2123
|
+
`,
|
|
2124
|
+
retry: `# Databricks notebook source
|
|
2125
|
+
dbutils.widgets.text("execution_count", "0")
|
|
2126
|
+
execution_count = int(dbutils.widgets.get("execution_count"))
|
|
2127
|
+
if execution_count <= 1:
|
|
2128
|
+
raise RuntimeError("intentional retryable failure")
|
|
2129
|
+
dbutils.notebook.exit("retried")
|
|
2130
|
+
`,
|
|
2131
|
+
repair: `# Databricks notebook source
|
|
2132
|
+
dbutils.widgets.text("repair_count", "0")
|
|
2133
|
+
repair_count = int(dbutils.widgets.get("repair_count"))
|
|
2134
|
+
if repair_count == 0:
|
|
2135
|
+
raise RuntimeError("intentional first-run failure")
|
|
2136
|
+
dbutils.notebook.exit("repaired")
|
|
2137
|
+
`,
|
|
2138
|
+
sleep: `# Databricks notebook source
|
|
2139
|
+
import time
|
|
2140
|
+
time.sleep(300)
|
|
2141
|
+
`
|
|
2142
|
+
};
|
|
2143
|
+
function jobsCertificationCheck() {
|
|
2144
|
+
return {
|
|
2145
|
+
id: "jobs-certification",
|
|
2146
|
+
description: "Disposable Lakeflow Jobs branch, loop, failure, repair, cancellation, and cleanup certification",
|
|
2147
|
+
configured: (env) => env.DBX_TEST_JOBS_CERTIFY === "1",
|
|
2148
|
+
async run({ env, client }) {
|
|
2149
|
+
const root = `${env.DBX_TEST_JOBS_FIXTURE_ROOT ?? "/Workspace/Shared/fabric-experiments-target-pack"}/${crypto.randomUUID()}`;
|
|
2150
|
+
const paths = Object.fromEntries(
|
|
2151
|
+
Object.keys(NOTEBOOKS).map((name) => [name, `${root}/${name}`])
|
|
2152
|
+
);
|
|
2153
|
+
try {
|
|
2154
|
+
await importNotebooks(client, paths);
|
|
2155
|
+
const adapter2 = new DatabricksJobsAdapter(client);
|
|
2156
|
+
const runId = await adapter2.submit(certificationRequest(paths));
|
|
2157
|
+
const initial = await adapter2.wait(runId, waitOptions(env));
|
|
2158
|
+
if (initial.resultState !== "FAILED") {
|
|
2159
|
+
throw new Error(
|
|
2160
|
+
`Certification run ${runId} expected FAILED before repair, got ${initial.resultState}`
|
|
2161
|
+
);
|
|
2162
|
+
}
|
|
2163
|
+
const retried = assertTask(initial.tasks, "retry", "SUCCESS");
|
|
2164
|
+
if ((retried.attemptNumber ?? 0) < 1) {
|
|
2165
|
+
throw new Error("Certification retry task did not record a second attempt");
|
|
2166
|
+
}
|
|
2167
|
+
assertTask(initial.tasks, "seed", "SUCCESS");
|
|
2168
|
+
assertTask(initial.tasks, "branch", "SUCCESS");
|
|
2169
|
+
assertTask(initial.tasks, "foreach", "SUCCESS");
|
|
2170
|
+
assertTask(initial.tasks, "repair", "FAILED");
|
|
2171
|
+
const repairId = await adapter2.repair(runId, {
|
|
2172
|
+
rerunTasks: ["repair"],
|
|
2173
|
+
rerunDependentTasks: true
|
|
2174
|
+
});
|
|
2175
|
+
const repaired = await adapter2.waitForRepair(runId, repairId, waitOptions(env));
|
|
2176
|
+
if (repaired.resultState !== "SUCCESS") {
|
|
2177
|
+
throw new Error(`Certification repair ${repairId} ended in ${repaired.resultState}`);
|
|
2178
|
+
}
|
|
2179
|
+
assertTask(repaired.tasks, "repair", "SUCCESS");
|
|
2180
|
+
const cancellationRunId = await adapter2.submit(cancellationRequest(paths));
|
|
2181
|
+
await adapter2.waitUntilActive(cancellationRunId, waitOptions(env));
|
|
2182
|
+
await adapter2.cancel(cancellationRunId);
|
|
2183
|
+
const canceled = await adapter2.wait(cancellationRunId, waitOptions(env));
|
|
2184
|
+
if (canceled.resultState !== "CANCELED") {
|
|
2185
|
+
throw new Error(`Certification cancellation ended in ${canceled.resultState}`);
|
|
2186
|
+
}
|
|
2187
|
+
return {
|
|
2188
|
+
runId,
|
|
2189
|
+
repairId,
|
|
2190
|
+
cancellationRunId,
|
|
2191
|
+
taskValues: true,
|
|
2192
|
+
retry: true,
|
|
2193
|
+
condition: true,
|
|
2194
|
+
forEach: true,
|
|
2195
|
+
repair: true,
|
|
2196
|
+
cancellation: true,
|
|
2197
|
+
fixtureRoot: root
|
|
2198
|
+
};
|
|
2199
|
+
} finally {
|
|
2200
|
+
await client.post("/api/2.0/workspace/delete", { path: root, recursive: true });
|
|
2201
|
+
}
|
|
2202
|
+
}
|
|
2203
|
+
};
|
|
2204
|
+
}
|
|
2205
|
+
async function importNotebooks(client, paths) {
|
|
2206
|
+
await client.post("/api/2.0/workspace/mkdirs", {
|
|
2207
|
+
path: paths.seed.slice(0, paths.seed.lastIndexOf("/"))
|
|
2208
|
+
});
|
|
2209
|
+
for (const [name, source] of Object.entries(NOTEBOOKS)) {
|
|
2210
|
+
await client.post("/api/2.0/workspace/import", {
|
|
2211
|
+
path: paths[name],
|
|
2212
|
+
format: "SOURCE",
|
|
2213
|
+
language: "PYTHON",
|
|
2214
|
+
overwrite: true,
|
|
2215
|
+
content: Buffer.from(source).toString("base64")
|
|
2216
|
+
});
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
function certificationRequest(paths) {
|
|
2220
|
+
return {
|
|
2221
|
+
run_name: "fabric-experiments-jobs-certification",
|
|
2222
|
+
tasks: [
|
|
2223
|
+
{
|
|
2224
|
+
task_key: "retry",
|
|
2225
|
+
notebook_task: {
|
|
2226
|
+
notebook_path: paths.retry,
|
|
2227
|
+
base_parameters: { execution_count: "{{task.execution_count}}" }
|
|
2228
|
+
},
|
|
2229
|
+
max_retries: 1
|
|
2230
|
+
},
|
|
2231
|
+
{
|
|
2232
|
+
task_key: "seed",
|
|
2233
|
+
depends_on: [{ task_key: "retry" }],
|
|
2234
|
+
notebook_task: { notebook_path: paths.seed }
|
|
2235
|
+
},
|
|
2236
|
+
{
|
|
2237
|
+
task_key: "branch",
|
|
2238
|
+
depends_on: [{ task_key: "seed" }],
|
|
2239
|
+
condition_task: {
|
|
2240
|
+
left: "{{tasks.seed.values.branch}}",
|
|
2241
|
+
op: "EQUAL_TO",
|
|
2242
|
+
right: "run"
|
|
2243
|
+
}
|
|
2244
|
+
},
|
|
2245
|
+
{
|
|
2246
|
+
task_key: "foreach",
|
|
2247
|
+
depends_on: [{ task_key: "branch", outcome: "true" }],
|
|
2248
|
+
for_each_task: {
|
|
2249
|
+
inputs: "{{tasks.seed.values.items}}",
|
|
2250
|
+
concurrency: 2,
|
|
2251
|
+
task: {
|
|
2252
|
+
task_key: "process_item",
|
|
2253
|
+
notebook_task: {
|
|
2254
|
+
notebook_path: paths.process,
|
|
2255
|
+
base_parameters: { item: "{{input}}" }
|
|
2256
|
+
}
|
|
2257
|
+
}
|
|
2258
|
+
}
|
|
2259
|
+
},
|
|
2260
|
+
{
|
|
2261
|
+
task_key: "repair",
|
|
2262
|
+
depends_on: [{ task_key: "foreach" }],
|
|
2263
|
+
notebook_task: {
|
|
2264
|
+
notebook_path: paths.repair,
|
|
2265
|
+
base_parameters: { repair_count: "{{job.repair_count}}" }
|
|
2266
|
+
}
|
|
2267
|
+
}
|
|
2268
|
+
]
|
|
2269
|
+
};
|
|
2270
|
+
}
|
|
2271
|
+
function cancellationRequest(paths) {
|
|
2272
|
+
return {
|
|
2273
|
+
run_name: "fabric-experiments-jobs-cancellation-certification",
|
|
2274
|
+
tasks: [{ task_key: "sleep", notebook_task: { notebook_path: paths.sleep } }]
|
|
2275
|
+
};
|
|
2276
|
+
}
|
|
2277
|
+
function waitOptions(env) {
|
|
2278
|
+
return {
|
|
2279
|
+
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
2280
|
+
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 5e3)
|
|
2281
|
+
};
|
|
2282
|
+
}
|
|
2283
|
+
function assertTask(tasks, taskKey, resultState) {
|
|
2284
|
+
const task = tasks.find((candidate) => candidate.taskKey === taskKey);
|
|
2285
|
+
if (!task) throw new Error(`Certification result did not contain task '${taskKey}'`);
|
|
2286
|
+
if (task.resultState !== resultState) {
|
|
2287
|
+
throw new Error(
|
|
2288
|
+
`Certification task '${taskKey}' ended in ${task.resultState}, expected ${resultState}`
|
|
2289
|
+
);
|
|
2290
|
+
}
|
|
2291
|
+
return task;
|
|
2292
|
+
}
|
|
2006
2293
|
|
|
2007
2294
|
// src/jobs-target-pack.ts
|
|
2008
2295
|
function createLakeflowJobsTargetPack() {
|
|
@@ -2010,8 +2297,8 @@ function createLakeflowJobsTargetPack() {
|
|
|
2010
2297
|
id: "lakeflow-jobs",
|
|
2011
2298
|
name: "Lakeflow Jobs orchestration",
|
|
2012
2299
|
description: "Typed multi-task DAG, branch/loop, retry, cancellation, repair and task-outcome validation.",
|
|
2013
|
-
version: "0.
|
|
2014
|
-
maturity: "
|
|
2300
|
+
version: "0.2.0",
|
|
2301
|
+
maturity: "live-gated",
|
|
2015
2302
|
capabilities: [
|
|
2016
2303
|
"jobs.multi-task-dag",
|
|
2017
2304
|
"jobs.dependencies",
|
|
@@ -2022,7 +2309,12 @@ function createLakeflowJobsTargetPack() {
|
|
|
2022
2309
|
"jobs.repair",
|
|
2023
2310
|
"jobs.task-outcomes"
|
|
2024
2311
|
],
|
|
2025
|
-
checks: () => [
|
|
2312
|
+
checks: () => [
|
|
2313
|
+
restrictedPrincipalCheck(),
|
|
2314
|
+
jobRunCheck(),
|
|
2315
|
+
jobsOrchestrationCheck(),
|
|
2316
|
+
jobsCertificationCheck()
|
|
2317
|
+
],
|
|
2026
2318
|
requiredChecks: ["jobs-orchestration"],
|
|
2027
2319
|
requirements: [
|
|
2028
2320
|
{
|
|
@@ -2045,17 +2337,812 @@ function createLakeflowJobsTargetPack() {
|
|
|
2045
2337
|
{
|
|
2046
2338
|
id: "orchestration-spec",
|
|
2047
2339
|
description: "JSON multi-task orchestration scenario",
|
|
2048
|
-
anyOf: ["DBX_TEST_JOBS_ORCHESTRATION_JSON"]
|
|
2340
|
+
anyOf: ["DBX_TEST_JOBS_ORCHESTRATION_JSON"],
|
|
2341
|
+
required: false
|
|
2049
2342
|
}
|
|
2050
2343
|
],
|
|
2051
|
-
docsUrl: "https://experiments.fabric.pro/docs/testing/target-packs/#
|
|
2052
|
-
certificationScopes: [
|
|
2344
|
+
docsUrl: "https://experiments.fabric.pro/docs/testing/target-packs/#lakeflow-jobs-orchestration",
|
|
2345
|
+
certificationScopes: [
|
|
2346
|
+
"Azure Databricks serverless Jobs, West US, workspace OAuth, public network path"
|
|
2347
|
+
]
|
|
2053
2348
|
});
|
|
2054
2349
|
}
|
|
2055
2350
|
|
|
2351
|
+
// src/advanced-workload-checks.ts
|
|
2352
|
+
init_workspace_context();
|
|
2353
|
+
var DatabricksAdvancedWorkloadsAdapter = class {
|
|
2354
|
+
constructor(client) {
|
|
2355
|
+
this.client = client;
|
|
2356
|
+
}
|
|
2357
|
+
client;
|
|
2358
|
+
pipeline(id) {
|
|
2359
|
+
return this.client.get(`/api/2.0/pipelines/${segment(id)}`);
|
|
2360
|
+
}
|
|
2361
|
+
table(fullName) {
|
|
2362
|
+
return this.client.get(`/api/2.1/unity-catalog/tables/${segment(fullName)}`);
|
|
2363
|
+
}
|
|
2364
|
+
dashboard(id) {
|
|
2365
|
+
return this.client.get(`/api/2.0/lakeview/dashboards/${segment(id)}`);
|
|
2366
|
+
}
|
|
2367
|
+
genieSpace(id) {
|
|
2368
|
+
return this.client.get(`/api/2.0/genie/spaces/${segment(id)}`);
|
|
2369
|
+
}
|
|
2370
|
+
startGenieConversation(spaceId, question) {
|
|
2371
|
+
return this.client.post(`/api/2.0/genie/spaces/${segment(spaceId)}/start-conversation`, {
|
|
2372
|
+
content: question
|
|
2373
|
+
});
|
|
2374
|
+
}
|
|
2375
|
+
experiment(id) {
|
|
2376
|
+
return this.client.get("/api/2.0/mlflow/experiments/get", { experiment_id: id });
|
|
2377
|
+
}
|
|
2378
|
+
registeredModel(fullName) {
|
|
2379
|
+
return this.client.get(`/api/2.1/unity-catalog/models/${segment(fullName)}`);
|
|
2380
|
+
}
|
|
2381
|
+
modelVersion(fullName, version) {
|
|
2382
|
+
return this.client.get(
|
|
2383
|
+
`/api/2.1/unity-catalog/models/${segment(fullName)}/versions/${segment(version)}`
|
|
2384
|
+
);
|
|
2385
|
+
}
|
|
2386
|
+
onlineTable(fullName) {
|
|
2387
|
+
return this.client.get(`/api/2.0/online-tables/${segment(fullName)}`);
|
|
2388
|
+
}
|
|
2389
|
+
servingEndpoint(name) {
|
|
2390
|
+
return this.client.get(`/api/2.0/serving-endpoints/${segment(name)}`);
|
|
2391
|
+
}
|
|
2392
|
+
invokeServingEndpoint(name, request) {
|
|
2393
|
+
return this.client.post(`/serving-endpoints/${segment(name)}/invocations`, request);
|
|
2394
|
+
}
|
|
2395
|
+
vectorSearchEndpoint(name) {
|
|
2396
|
+
return this.client.get(`/api/2.0/vector-search/endpoints/${segment(name)}`);
|
|
2397
|
+
}
|
|
2398
|
+
vectorSearchIndex(name) {
|
|
2399
|
+
return this.client.get(`/api/2.0/vector-search/indexes/${segment(name)}`);
|
|
2400
|
+
}
|
|
2401
|
+
queryVectorSearchIndex(name, request) {
|
|
2402
|
+
return this.client.post(`/api/2.0/vector-search/indexes/${segment(name)}/query`, request);
|
|
2403
|
+
}
|
|
2404
|
+
connection(name) {
|
|
2405
|
+
return this.client.get(`/api/2.1/unity-catalog/connections/${segment(name)}`);
|
|
2406
|
+
}
|
|
2407
|
+
share(name) {
|
|
2408
|
+
return this.client.get(`/api/2.1/unity-catalog/shares/${segment(name)}`);
|
|
2409
|
+
}
|
|
2410
|
+
cleanRoom(name) {
|
|
2411
|
+
return this.client.get(`/api/2.0/clean-rooms/${segment(name)}`);
|
|
2412
|
+
}
|
|
2413
|
+
ipAccessList(id) {
|
|
2414
|
+
return this.client.get(`/api/2.0/ip-access-lists/${segment(id)}`);
|
|
2415
|
+
}
|
|
2416
|
+
clusterPolicy(id) {
|
|
2417
|
+
return this.client.get("/api/2.0/policies/clusters/get", { policy_id: id });
|
|
2418
|
+
}
|
|
2419
|
+
warehouse(id) {
|
|
2420
|
+
return this.client.get(`/api/2.0/sql/warehouses/${segment(id)}`);
|
|
2421
|
+
}
|
|
2422
|
+
};
|
|
2423
|
+
function segment(value) {
|
|
2424
|
+
return encodeURIComponent(value);
|
|
2425
|
+
}
|
|
2426
|
+
function adapter(client) {
|
|
2427
|
+
return new DatabricksAdvancedWorkloadsAdapter(client);
|
|
2428
|
+
}
|
|
2429
|
+
function required2(env, names) {
|
|
2430
|
+
return names.filter((name) => !env[name]?.trim());
|
|
2431
|
+
}
|
|
2432
|
+
function requireEnv(env, names) {
|
|
2433
|
+
const missing = required2(env, names);
|
|
2434
|
+
if (missing.length > 0) throw new Error(`Missing required configuration: ${missing.join(", ")}`);
|
|
2435
|
+
}
|
|
2436
|
+
function parseJson(value, name) {
|
|
2437
|
+
if (!value) return void 0;
|
|
2438
|
+
try {
|
|
2439
|
+
return JSON.parse(value);
|
|
2440
|
+
} catch {
|
|
2441
|
+
throw new Error(`${name} must contain valid JSON`);
|
|
2442
|
+
}
|
|
2443
|
+
}
|
|
2444
|
+
function assertNotFailed(label, state) {
|
|
2445
|
+
if (typeof state === "string" && /FAIL|ERROR|DELET/i.test(state)) {
|
|
2446
|
+
throw new Error(`${label} is ${state}`);
|
|
2447
|
+
}
|
|
2448
|
+
}
|
|
2449
|
+
function assertReady(label, state, accepted) {
|
|
2450
|
+
if (typeof state !== "string" || !accepted.includes(state.toUpperCase())) {
|
|
2451
|
+
throw new Error(`${label} is ${String(state ?? "UNKNOWN")}; expected ${accepted.join(" or ")}`);
|
|
2452
|
+
}
|
|
2453
|
+
}
|
|
2454
|
+
async function booleanSql(label, sql, query) {
|
|
2455
|
+
if (!sql) return;
|
|
2456
|
+
const rows = await query(sql);
|
|
2457
|
+
const first = rows[0] ? Object.values(rows[0])[0] : void 0;
|
|
2458
|
+
if (!(first === true || first === 1 || first === "1" || first === "true")) {
|
|
2459
|
+
throw new Error(`${label} assertion did not return a truthy first column`);
|
|
2460
|
+
}
|
|
2461
|
+
}
|
|
2462
|
+
function advancedStreamingCdcCheck() {
|
|
2463
|
+
const names = [
|
|
2464
|
+
"DBX_TEST_STREAMING_PIPELINE_ID",
|
|
2465
|
+
"DBX_TEST_STREAMING_TABLE",
|
|
2466
|
+
"DBX_TEST_CDC_ASSERTION_SQL",
|
|
2467
|
+
"DBX_TEST_STREAMING_ASSERTION_SQL"
|
|
2468
|
+
];
|
|
2469
|
+
return {
|
|
2470
|
+
id: "streaming-cdc",
|
|
2471
|
+
description: "Streaming pipeline, streaming table and CDC assertions are healthy",
|
|
2472
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2473
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2474
|
+
requireEnv(env, names);
|
|
2475
|
+
const api = adapter(client);
|
|
2476
|
+
const pipeline = await api.pipeline(env.DBX_TEST_STREAMING_PIPELINE_ID);
|
|
2477
|
+
assertNotFailed("Streaming pipeline", pipeline.state);
|
|
2478
|
+
const table = await api.table(env.DBX_TEST_STREAMING_TABLE);
|
|
2479
|
+
const tableType = String(table.table_type ?? table.data_source_format ?? "").toUpperCase();
|
|
2480
|
+
if (!["STREAMING_TABLE", "MATERIALIZED_VIEW", "DELTA"].includes(tableType)) {
|
|
2481
|
+
throw new Error(`Streaming target has unsupported table type ${tableType || "UNKNOWN"}`);
|
|
2482
|
+
}
|
|
2483
|
+
await booleanSql("CDC", env.DBX_TEST_CDC_ASSERTION_SQL, warehouse2.query);
|
|
2484
|
+
await booleanSql(
|
|
2485
|
+
"Streaming freshness",
|
|
2486
|
+
env.DBX_TEST_STREAMING_ASSERTION_SQL,
|
|
2487
|
+
warehouse2.query
|
|
2488
|
+
);
|
|
2489
|
+
return { pipelineId: pipeline.pipeline_id, table: env.DBX_TEST_STREAMING_TABLE, tableType };
|
|
2490
|
+
}
|
|
2491
|
+
};
|
|
2492
|
+
}
|
|
2493
|
+
function lakeflowConnectCheck() {
|
|
2494
|
+
const names = ["DBX_TEST_CONNECT_PIPELINE_ID", "DBX_TEST_CONNECT_ASSERTION_SQL"];
|
|
2495
|
+
return {
|
|
2496
|
+
id: "lakeflow-connect",
|
|
2497
|
+
description: "Lakeflow Connect ingestion pipeline exists and is not failed",
|
|
2498
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2499
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2500
|
+
requireEnv(env, names);
|
|
2501
|
+
const id = env.DBX_TEST_CONNECT_PIPELINE_ID;
|
|
2502
|
+
const pipeline = await adapter(client).pipeline(id);
|
|
2503
|
+
assertNotFailed("Lakeflow Connect pipeline", pipeline.state);
|
|
2504
|
+
await booleanSql(
|
|
2505
|
+
"Lakeflow Connect replication",
|
|
2506
|
+
env.DBX_TEST_CONNECT_ASSERTION_SQL,
|
|
2507
|
+
warehouse2.query
|
|
2508
|
+
);
|
|
2509
|
+
return { pipelineId: pipeline.pipeline_id ?? id, state: pipeline.state ?? "IDLE" };
|
|
2510
|
+
}
|
|
2511
|
+
};
|
|
2512
|
+
}
|
|
2513
|
+
function aiBiDashboardCheck() {
|
|
2514
|
+
return {
|
|
2515
|
+
id: "aibi-dashboard",
|
|
2516
|
+
description: "Published AI/BI dashboard is accessible",
|
|
2517
|
+
configured: (env) => Boolean(env.DBX_TEST_DASHBOARD_ID),
|
|
2518
|
+
async run({ env, client }) {
|
|
2519
|
+
const dashboard = await adapter(client).dashboard(env.DBX_TEST_DASHBOARD_ID);
|
|
2520
|
+
assertNotFailed("AI/BI dashboard", dashboard.lifecycle_state);
|
|
2521
|
+
return {
|
|
2522
|
+
dashboardId: dashboard.dashboard_id ?? env.DBX_TEST_DASHBOARD_ID,
|
|
2523
|
+
name: dashboard.display_name
|
|
2524
|
+
};
|
|
2525
|
+
}
|
|
2526
|
+
};
|
|
2527
|
+
}
|
|
2528
|
+
function genieCheck() {
|
|
2529
|
+
const names = ["DBX_TEST_GENIE_SPACE_ID", "DBX_TEST_GENIE_QUESTION"];
|
|
2530
|
+
return {
|
|
2531
|
+
id: "genie",
|
|
2532
|
+
description: "Genie space is accessible and can optionally start a grounded conversation",
|
|
2533
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2534
|
+
async run({ env, client }) {
|
|
2535
|
+
requireEnv(env, names);
|
|
2536
|
+
const api = adapter(client);
|
|
2537
|
+
const id = env.DBX_TEST_GENIE_SPACE_ID;
|
|
2538
|
+
const space = await api.genieSpace(id);
|
|
2539
|
+
let conversation;
|
|
2540
|
+
if (env.DBX_TEST_GENIE_QUESTION) {
|
|
2541
|
+
conversation = await api.startGenieConversation(id, env.DBX_TEST_GENIE_QUESTION);
|
|
2542
|
+
if (!conversation.conversation_id && !conversation.message_id) {
|
|
2543
|
+
throw new Error("Genie did not return a conversation or message identifier");
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
return {
|
|
2547
|
+
spaceId: space.space_id ?? id,
|
|
2548
|
+
title: space.title,
|
|
2549
|
+
conversationStarted: Boolean(conversation)
|
|
2550
|
+
};
|
|
2551
|
+
}
|
|
2552
|
+
};
|
|
2553
|
+
}
|
|
2554
|
+
function mlflowLifecycleCheck() {
|
|
2555
|
+
const names = [
|
|
2556
|
+
"DBX_TEST_MLFLOW_EXPERIMENT_ID",
|
|
2557
|
+
"DBX_TEST_REGISTERED_MODEL",
|
|
2558
|
+
"DBX_TEST_MODEL_VERSION"
|
|
2559
|
+
];
|
|
2560
|
+
return {
|
|
2561
|
+
id: "mlflow-lifecycle",
|
|
2562
|
+
description: "MLflow experiment and Unity Catalog model version are accessible",
|
|
2563
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2564
|
+
async run({ env, client }) {
|
|
2565
|
+
requireEnv(env, names);
|
|
2566
|
+
const api = adapter(client);
|
|
2567
|
+
const experiment = await api.experiment(env.DBX_TEST_MLFLOW_EXPERIMENT_ID);
|
|
2568
|
+
const model = await api.registeredModel(env.DBX_TEST_REGISTERED_MODEL);
|
|
2569
|
+
const version = await api.modelVersion(
|
|
2570
|
+
env.DBX_TEST_REGISTERED_MODEL,
|
|
2571
|
+
env.DBX_TEST_MODEL_VERSION
|
|
2572
|
+
);
|
|
2573
|
+
assertNotFailed("Model version", version.status);
|
|
2574
|
+
return {
|
|
2575
|
+
experimentId: experiment.experiment?.experiment_id ?? env.DBX_TEST_MLFLOW_EXPERIMENT_ID,
|
|
2576
|
+
model: model.full_name ?? model.name ?? env.DBX_TEST_REGISTERED_MODEL,
|
|
2577
|
+
version: version.version ?? env.DBX_TEST_MODEL_VERSION
|
|
2578
|
+
};
|
|
2579
|
+
}
|
|
2580
|
+
};
|
|
2581
|
+
}
|
|
2582
|
+
function featureEngineeringServingCheck() {
|
|
2583
|
+
return {
|
|
2584
|
+
id: "feature-engineering",
|
|
2585
|
+
description: "Feature table and online materialization are healthy",
|
|
2586
|
+
configured: (env) => Boolean(
|
|
2587
|
+
env.DBX_TEST_FEATURE_TABLE && env.DBX_TEST_FEATURE_ASSERTION_SQL && (env.DBX_TEST_ONLINE_TABLE || env.DBX_TEST_FEATURE_SERVING_PIPELINE_ID)
|
|
2588
|
+
),
|
|
2589
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2590
|
+
requireEnv(env, ["DBX_TEST_FEATURE_TABLE"]);
|
|
2591
|
+
if (!env.DBX_TEST_ONLINE_TABLE && !env.DBX_TEST_FEATURE_SERVING_PIPELINE_ID) {
|
|
2592
|
+
throw new Error(
|
|
2593
|
+
"Feature serving requires DBX_TEST_ONLINE_TABLE or DBX_TEST_FEATURE_SERVING_PIPELINE_ID"
|
|
2594
|
+
);
|
|
2595
|
+
}
|
|
2596
|
+
const api = adapter(client);
|
|
2597
|
+
const source = await api.table(env.DBX_TEST_FEATURE_TABLE);
|
|
2598
|
+
let state;
|
|
2599
|
+
let servingResource;
|
|
2600
|
+
if (env.DBX_TEST_ONLINE_TABLE) {
|
|
2601
|
+
const online = await api.onlineTable(env.DBX_TEST_ONLINE_TABLE);
|
|
2602
|
+
const status = online.status;
|
|
2603
|
+
state = status?.state ?? status?.detailed_state ?? online.state;
|
|
2604
|
+
assertReady("Online table", state, ["ONLINE", "ACTIVE", "READY", "PROVISIONED"]);
|
|
2605
|
+
servingResource = env.DBX_TEST_ONLINE_TABLE;
|
|
2606
|
+
} else {
|
|
2607
|
+
const pipelineId = env.DBX_TEST_FEATURE_SERVING_PIPELINE_ID;
|
|
2608
|
+
const pipeline = await api.pipeline(pipelineId);
|
|
2609
|
+
state = pipeline.state ?? "IDLE";
|
|
2610
|
+
assertNotFailed("Synced Table pipeline", state);
|
|
2611
|
+
servingResource = pipeline.pipeline_id ?? pipelineId;
|
|
2612
|
+
}
|
|
2613
|
+
await booleanSql("Feature freshness", env.DBX_TEST_FEATURE_ASSERTION_SQL, warehouse2.query);
|
|
2614
|
+
return {
|
|
2615
|
+
featureTable: source.full_name ?? env.DBX_TEST_FEATURE_TABLE,
|
|
2616
|
+
servingResource,
|
|
2617
|
+
state
|
|
2618
|
+
};
|
|
2619
|
+
}
|
|
2620
|
+
};
|
|
2621
|
+
}
|
|
2622
|
+
function modelServingCheck() {
|
|
2623
|
+
const names = [
|
|
2624
|
+
"DBX_TEST_SERVING_ENDPOINT",
|
|
2625
|
+
"DBX_TEST_SERVING_REQUEST_JSON",
|
|
2626
|
+
"DBX_TEST_AI_GATEWAY_REQUIRED"
|
|
2627
|
+
];
|
|
2628
|
+
return {
|
|
2629
|
+
id: "model-serving",
|
|
2630
|
+
description: "Model Serving endpoint is ready and can optionally serve an inference request",
|
|
2631
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2632
|
+
async run({ env, client }) {
|
|
2633
|
+
requireEnv(env, names);
|
|
2634
|
+
if (env.DBX_TEST_AI_GATEWAY_REQUIRED !== "1") {
|
|
2635
|
+
throw new Error("DBX_TEST_AI_GATEWAY_REQUIRED must be 1 for the Model Serving target pack");
|
|
2636
|
+
}
|
|
2637
|
+
const api = adapter(client);
|
|
2638
|
+
const name = env.DBX_TEST_SERVING_ENDPOINT;
|
|
2639
|
+
const endpoint = await api.servingEndpoint(name);
|
|
2640
|
+
assertReady("Serving endpoint", endpoint.state?.ready, ["READY"]);
|
|
2641
|
+
if (env.DBX_TEST_AI_GATEWAY_REQUIRED === "1" && !endpoint.ai_gateway) {
|
|
2642
|
+
throw new Error(`Serving endpoint ${name} has no AI Gateway configuration`);
|
|
2643
|
+
}
|
|
2644
|
+
let response;
|
|
2645
|
+
if (env.DBX_TEST_SERVING_REQUEST_JSON) {
|
|
2646
|
+
response = await api.invokeServingEndpoint(
|
|
2647
|
+
name,
|
|
2648
|
+
parseJson(env.DBX_TEST_SERVING_REQUEST_JSON, "DBX_TEST_SERVING_REQUEST_JSON")
|
|
2649
|
+
);
|
|
2650
|
+
if (Object.keys(response).length === 0)
|
|
2651
|
+
throw new Error("Serving invocation returned an empty response");
|
|
2652
|
+
}
|
|
2653
|
+
return {
|
|
2654
|
+
endpoint: endpoint.name ?? name,
|
|
2655
|
+
ready: endpoint.state?.ready,
|
|
2656
|
+
aiGateway: Boolean(endpoint.ai_gateway),
|
|
2657
|
+
invoked: Boolean(response)
|
|
2658
|
+
};
|
|
2659
|
+
}
|
|
2660
|
+
};
|
|
2661
|
+
}
|
|
2662
|
+
function vectorSearchCheck() {
|
|
2663
|
+
const names = ["DBX_TEST_VECTOR_ENDPOINT", "DBX_TEST_VECTOR_INDEX"];
|
|
2664
|
+
return {
|
|
2665
|
+
id: "vector-search",
|
|
2666
|
+
description: "Vector Search endpoint and index are online",
|
|
2667
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2668
|
+
async run({ env, client }) {
|
|
2669
|
+
requireEnv(env, names);
|
|
2670
|
+
const api = adapter(client);
|
|
2671
|
+
const endpoint = await api.vectorSearchEndpoint(env.DBX_TEST_VECTOR_ENDPOINT);
|
|
2672
|
+
assertReady("Vector Search endpoint", endpoint.endpoint_status?.state, ["ONLINE", "READY"]);
|
|
2673
|
+
const index = await api.vectorSearchIndex(env.DBX_TEST_VECTOR_INDEX);
|
|
2674
|
+
if (index.status?.ready !== true)
|
|
2675
|
+
throw new Error(`Vector Search index is not ready: ${index.status?.message ?? "UNKNOWN"}`);
|
|
2676
|
+
return {
|
|
2677
|
+
endpoint: endpoint.name ?? env.DBX_TEST_VECTOR_ENDPOINT,
|
|
2678
|
+
index: index.name ?? env.DBX_TEST_VECTOR_INDEX,
|
|
2679
|
+
indexedRows: index.status.indexed_row_count
|
|
2680
|
+
};
|
|
2681
|
+
}
|
|
2682
|
+
};
|
|
2683
|
+
}
|
|
2684
|
+
function ragAgentCheck() {
|
|
2685
|
+
const names = [
|
|
2686
|
+
"DBX_TEST_VECTOR_INDEX",
|
|
2687
|
+
"DBX_TEST_VECTOR_QUERY_JSON",
|
|
2688
|
+
"DBX_TEST_AGENT_ENDPOINT",
|
|
2689
|
+
"DBX_TEST_AGENT_REQUEST_JSON"
|
|
2690
|
+
];
|
|
2691
|
+
return {
|
|
2692
|
+
id: "rag-agent",
|
|
2693
|
+
description: "RAG retrieval and agent serving endpoint return non-empty responses",
|
|
2694
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2695
|
+
async run({ env, client }) {
|
|
2696
|
+
requireEnv(env, names);
|
|
2697
|
+
const api = adapter(client);
|
|
2698
|
+
let retrieval;
|
|
2699
|
+
if (env.DBX_TEST_VECTOR_QUERY_JSON) {
|
|
2700
|
+
retrieval = await api.queryVectorSearchIndex(
|
|
2701
|
+
env.DBX_TEST_VECTOR_INDEX,
|
|
2702
|
+
parseJson(env.DBX_TEST_VECTOR_QUERY_JSON, "DBX_TEST_VECTOR_QUERY_JSON")
|
|
2703
|
+
);
|
|
2704
|
+
if (!retrieval.result && !retrieval.data_array && !retrieval.manifest) {
|
|
2705
|
+
throw new Error("Vector query returned no result payload");
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
let agent;
|
|
2709
|
+
if (env.DBX_TEST_AGENT_ENDPOINT) {
|
|
2710
|
+
const endpoint = await api.servingEndpoint(env.DBX_TEST_AGENT_ENDPOINT);
|
|
2711
|
+
assertReady("Agent endpoint", endpoint.state?.ready, ["READY"]);
|
|
2712
|
+
if (env.DBX_TEST_AGENT_REQUEST_JSON) {
|
|
2713
|
+
agent = await api.invokeServingEndpoint(
|
|
2714
|
+
env.DBX_TEST_AGENT_ENDPOINT,
|
|
2715
|
+
parseJson(env.DBX_TEST_AGENT_REQUEST_JSON, "DBX_TEST_AGENT_REQUEST_JSON")
|
|
2716
|
+
);
|
|
2717
|
+
if (Object.keys(agent).length === 0)
|
|
2718
|
+
throw new Error("Agent invocation returned an empty response");
|
|
2719
|
+
}
|
|
2720
|
+
}
|
|
2721
|
+
return {
|
|
2722
|
+
retrieval: Boolean(retrieval),
|
|
2723
|
+
agentEndpoint: env.DBX_TEST_AGENT_ENDPOINT,
|
|
2724
|
+
agentInvoked: Boolean(agent)
|
|
2725
|
+
};
|
|
2726
|
+
}
|
|
2727
|
+
};
|
|
2728
|
+
}
|
|
2729
|
+
function federationSharingCleanRoomsCheck() {
|
|
2730
|
+
const names = [
|
|
2731
|
+
"DBX_TEST_CONNECTION",
|
|
2732
|
+
"DBX_TEST_SHARE",
|
|
2733
|
+
"DBX_TEST_CLEAN_ROOM",
|
|
2734
|
+
"DBX_TEST_FEDERATION_ASSERTION_SQL"
|
|
2735
|
+
];
|
|
2736
|
+
return {
|
|
2737
|
+
id: "federation-sharing-cleanrooms",
|
|
2738
|
+
description: "Lakehouse Federation connection, Delta Share and Clean Room are accessible",
|
|
2739
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2740
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2741
|
+
requireEnv(env, names);
|
|
2742
|
+
const api = adapter(client);
|
|
2743
|
+
const connection = await api.connection(env.DBX_TEST_CONNECTION);
|
|
2744
|
+
const share = await api.share(env.DBX_TEST_SHARE);
|
|
2745
|
+
const room = await api.cleanRoom(env.DBX_TEST_CLEAN_ROOM);
|
|
2746
|
+
await booleanSql("Federated query", env.DBX_TEST_FEDERATION_ASSERTION_SQL, warehouse2.query);
|
|
2747
|
+
return {
|
|
2748
|
+
connection: connection.name ?? env.DBX_TEST_CONNECTION,
|
|
2749
|
+
share: share.name ?? env.DBX_TEST_SHARE,
|
|
2750
|
+
cleanRoom: room.name ?? env.DBX_TEST_CLEAN_ROOM
|
|
2751
|
+
};
|
|
2752
|
+
}
|
|
2753
|
+
};
|
|
2754
|
+
}
|
|
2755
|
+
function securityPostureCostCheck() {
|
|
2756
|
+
const names = [
|
|
2757
|
+
"DBX_TEST_IP_ACCESS_LIST_ID",
|
|
2758
|
+
"DBX_TEST_CLUSTER_POLICY_ID",
|
|
2759
|
+
"DBX_TEST_COST_ASSERTION_SQL"
|
|
2760
|
+
];
|
|
2761
|
+
return {
|
|
2762
|
+
id: "security-cost",
|
|
2763
|
+
description: "Network access controls, compute policy and cost guardrail are enforced",
|
|
2764
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2765
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2766
|
+
requireEnv(env, names);
|
|
2767
|
+
const api = adapter(client);
|
|
2768
|
+
const access = await api.ipAccessList(env.DBX_TEST_IP_ACCESS_LIST_ID);
|
|
2769
|
+
if (access.enabled === false) throw new Error("Configured IP access list is disabled");
|
|
2770
|
+
const policy = await api.clusterPolicy(env.DBX_TEST_CLUSTER_POLICY_ID);
|
|
2771
|
+
if (!policy.policy_id && !policy.name)
|
|
2772
|
+
throw new Error("Cluster policy response has no identity");
|
|
2773
|
+
if (env.DBX_TEST_EXPECT_HOST_SUFFIX && !env.DATABRICKS_HOST?.endsWith(env.DBX_TEST_EXPECT_HOST_SUFFIX)) {
|
|
2774
|
+
throw new Error(`Workspace host does not end with ${env.DBX_TEST_EXPECT_HOST_SUFFIX}`);
|
|
2775
|
+
}
|
|
2776
|
+
await booleanSql("Cost guardrail", env.DBX_TEST_COST_ASSERTION_SQL, warehouse2.query);
|
|
2777
|
+
return {
|
|
2778
|
+
ipAccessList: access.list_id ?? env.DBX_TEST_IP_ACCESS_LIST_ID,
|
|
2779
|
+
policy: policy.policy_id ?? policy.name,
|
|
2780
|
+
costGuardrail: "pass"
|
|
2781
|
+
};
|
|
2782
|
+
}
|
|
2783
|
+
};
|
|
2784
|
+
}
|
|
2785
|
+
function regionalDrCheck(options = {}) {
|
|
2786
|
+
const names = [
|
|
2787
|
+
"DBX_TEST_DR_HOST",
|
|
2788
|
+
"DBX_TEST_DR_WAREHOUSE_ID",
|
|
2789
|
+
"DBX_TEST_DR_ASSERTION_SQL"
|
|
2790
|
+
];
|
|
2791
|
+
return {
|
|
2792
|
+
id: "regional-dr",
|
|
2793
|
+
description: "Secondary-region workspace authentication and SQL warehouse are reachable",
|
|
2794
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2795
|
+
async run({ env }) {
|
|
2796
|
+
requireEnv(env, names);
|
|
2797
|
+
const drEnv = {
|
|
2798
|
+
...env,
|
|
2799
|
+
DATABRICKS_HOST: env.DBX_TEST_DR_HOST,
|
|
2800
|
+
DATABRICKS_WAREHOUSE_ID: env.DBX_TEST_DR_WAREHOUSE_ID,
|
|
2801
|
+
DATABRICKS_HTTP_PATH: void 0,
|
|
2802
|
+
DATABRICKS_BEARER: env.DBX_TEST_DR_BEARER,
|
|
2803
|
+
DATABRICKS_TOKEN: env.DBX_TEST_DR_TOKEN,
|
|
2804
|
+
DATABRICKS_CLIENT_ID: env.DBX_TEST_DR_CLIENT_ID ?? env.DATABRICKS_CLIENT_ID,
|
|
2805
|
+
DATABRICKS_CLIENT_SECRET: env.DBX_TEST_DR_CLIENT_SECRET ?? env.DATABRICKS_CLIENT_SECRET
|
|
2806
|
+
};
|
|
2807
|
+
const client = options.clientFactory?.(drEnv) ?? createClientFromEnv(drEnv);
|
|
2808
|
+
const secondary = await adapter(client).warehouse(env.DBX_TEST_DR_WAREHOUSE_ID);
|
|
2809
|
+
assertReady("DR SQL warehouse", secondary.state, ["RUNNING"]);
|
|
2810
|
+
const drWarehouse = await createWorkspaceContext({ env: drEnv, client });
|
|
2811
|
+
try {
|
|
2812
|
+
await booleanSql("Regional DR workload", env.DBX_TEST_DR_ASSERTION_SQL, drWarehouse.query);
|
|
2813
|
+
} finally {
|
|
2814
|
+
await drWarehouse.close();
|
|
2815
|
+
}
|
|
2816
|
+
return {
|
|
2817
|
+
secondaryWorkspace: "reachable",
|
|
2818
|
+
warehouseId: secondary.id ?? env.DBX_TEST_DR_WAREHOUSE_ID,
|
|
2819
|
+
state: secondary.state
|
|
2820
|
+
};
|
|
2821
|
+
}
|
|
2822
|
+
};
|
|
2823
|
+
}
|
|
2824
|
+
|
|
2825
|
+
// src/advanced-target-packs.ts
|
|
2826
|
+
var DOCS2 = "https://experiments.fabric.pro/docs/testing/target-packs/";
|
|
2827
|
+
var SHIPPED_SCOPE = "Typed adapter and offline contract tests shipped; live certification requires customer-owned test resources";
|
|
2828
|
+
var AZURE_USER_SCOPE = "Azure Databricks, West US, interactive user OAuth, public network path; existing certification fixtures";
|
|
2829
|
+
var workspace = {
|
|
2830
|
+
id: "workspace",
|
|
2831
|
+
description: "Databricks workspace host",
|
|
2832
|
+
anyOf: ["DATABRICKS_HOST"]
|
|
2833
|
+
};
|
|
2834
|
+
var auth = {
|
|
2835
|
+
id: "authentication",
|
|
2836
|
+
description: "Databricks OAuth, OIDC, bearer, or PAT authentication",
|
|
2837
|
+
anyOf: [
|
|
2838
|
+
"DATABRICKS_BEARER",
|
|
2839
|
+
"DATABRICKS_TOKEN",
|
|
2840
|
+
"DATABRICKS_CLIENT_ID",
|
|
2841
|
+
"DATABRICKS_OIDC_TOKEN",
|
|
2842
|
+
"DATABRICKS_OIDC_TOKEN_FILEPATH"
|
|
2843
|
+
],
|
|
2844
|
+
secret: true
|
|
2845
|
+
};
|
|
2846
|
+
var warehouse = {
|
|
2847
|
+
id: "warehouse",
|
|
2848
|
+
description: "SQL warehouse for data-plane assertions",
|
|
2849
|
+
anyOf: ["DATABRICKS_WAREHOUSE_ID", "DATABRICKS_HTTP_PATH"]
|
|
2850
|
+
};
|
|
2851
|
+
var requirement = (id, description, ...anyOf) => ({
|
|
2852
|
+
id,
|
|
2853
|
+
description,
|
|
2854
|
+
anyOf
|
|
2855
|
+
});
|
|
2856
|
+
function createAdvancedStreamingTargetPack() {
|
|
2857
|
+
return defineTargetPack({
|
|
2858
|
+
id: "streaming-cdc-connect",
|
|
2859
|
+
name: "Advanced streaming, CDC and Lakeflow Connect",
|
|
2860
|
+
description: "Streaming tables, pipeline health, CDC correctness, freshness and managed connector replication.",
|
|
2861
|
+
version: "0.1.0",
|
|
2862
|
+
maturity: "shipped",
|
|
2863
|
+
capabilities: [
|
|
2864
|
+
"streaming.tables",
|
|
2865
|
+
"streaming.freshness",
|
|
2866
|
+
"cdc.correctness",
|
|
2867
|
+
"lakeflow-connect.replication"
|
|
2868
|
+
],
|
|
2869
|
+
checks: () => [restrictedPrincipalCheck(), advancedStreamingCdcCheck(), lakeflowConnectCheck()],
|
|
2870
|
+
requiredChecks: ["streaming-cdc", "lakeflow-connect"],
|
|
2871
|
+
requirements: [
|
|
2872
|
+
workspace,
|
|
2873
|
+
auth,
|
|
2874
|
+
warehouse,
|
|
2875
|
+
requirement("streaming-pipeline", "Streaming pipeline ID", "DBX_TEST_STREAMING_PIPELINE_ID"),
|
|
2876
|
+
requirement("streaming-table", "Streaming target table", "DBX_TEST_STREAMING_TABLE"),
|
|
2877
|
+
requirement(
|
|
2878
|
+
"connect-pipeline",
|
|
2879
|
+
"Lakeflow Connect ingestion pipeline",
|
|
2880
|
+
"DBX_TEST_CONNECT_PIPELINE_ID"
|
|
2881
|
+
),
|
|
2882
|
+
requirement(
|
|
2883
|
+
"streaming-assertion",
|
|
2884
|
+
"Streaming freshness SQL",
|
|
2885
|
+
"DBX_TEST_STREAMING_ASSERTION_SQL"
|
|
2886
|
+
),
|
|
2887
|
+
requirement("cdc-assertion", "CDC correctness SQL", "DBX_TEST_CDC_ASSERTION_SQL"),
|
|
2888
|
+
requirement(
|
|
2889
|
+
"connect-assertion",
|
|
2890
|
+
"Lakeflow Connect replication SQL",
|
|
2891
|
+
"DBX_TEST_CONNECT_ASSERTION_SQL"
|
|
2892
|
+
)
|
|
2893
|
+
],
|
|
2894
|
+
docsUrl: `${DOCS2}#advanced-streaming-cdc-and-lakeflow-connect`,
|
|
2895
|
+
certificationScopes: [SHIPPED_SCOPE]
|
|
2896
|
+
});
|
|
2897
|
+
}
|
|
2898
|
+
function createAiBiGenieTargetPack() {
|
|
2899
|
+
return defineTargetPack({
|
|
2900
|
+
id: "aibi-genie",
|
|
2901
|
+
name: "AI/BI dashboards and Genie",
|
|
2902
|
+
description: "Dashboard accessibility and authenticated Genie space/conversation behavior.",
|
|
2903
|
+
version: "0.1.0",
|
|
2904
|
+
maturity: "live-gated",
|
|
2905
|
+
capabilities: ["aibi.dashboard", "genie.space", "genie.conversation"],
|
|
2906
|
+
checks: () => [restrictedPrincipalCheck(), aiBiDashboardCheck(), genieCheck()],
|
|
2907
|
+
requiredChecks: ["aibi-dashboard", "genie"],
|
|
2908
|
+
requirements: [
|
|
2909
|
+
workspace,
|
|
2910
|
+
auth,
|
|
2911
|
+
requirement("dashboard", "AI/BI dashboard ID", "DBX_TEST_DASHBOARD_ID"),
|
|
2912
|
+
requirement("genie-space", "Genie space ID", "DBX_TEST_GENIE_SPACE_ID"),
|
|
2913
|
+
requirement(
|
|
2914
|
+
"genie-question",
|
|
2915
|
+
"Grounded Genie certification question",
|
|
2916
|
+
"DBX_TEST_GENIE_QUESTION"
|
|
2917
|
+
)
|
|
2918
|
+
],
|
|
2919
|
+
docsUrl: `${DOCS2}#aibi-dashboards-and-genie`,
|
|
2920
|
+
certificationScopes: [
|
|
2921
|
+
`${AZURE_USER_SCOPE}; AI/BI dashboard metadata and Genie conversation start`
|
|
2922
|
+
]
|
|
2923
|
+
});
|
|
2924
|
+
}
|
|
2925
|
+
function createMlflowLifecycleTargetPack() {
|
|
2926
|
+
return defineTargetPack({
|
|
2927
|
+
id: "mlflow-lifecycle",
|
|
2928
|
+
name: "MLflow and model lifecycle",
|
|
2929
|
+
description: "MLflow experiments plus Unity Catalog registered-model and model-version lifecycle evidence.",
|
|
2930
|
+
version: "0.1.0",
|
|
2931
|
+
maturity: "live-gated",
|
|
2932
|
+
capabilities: [
|
|
2933
|
+
"mlflow.experiments",
|
|
2934
|
+
"models.unity-catalog",
|
|
2935
|
+
"models.versions",
|
|
2936
|
+
"models.lifecycle"
|
|
2937
|
+
],
|
|
2938
|
+
checks: () => [restrictedPrincipalCheck(), mlflowLifecycleCheck()],
|
|
2939
|
+
requiredChecks: ["mlflow-lifecycle"],
|
|
2940
|
+
requirements: [
|
|
2941
|
+
workspace,
|
|
2942
|
+
auth,
|
|
2943
|
+
requirement("experiment", "MLflow experiment ID", "DBX_TEST_MLFLOW_EXPERIMENT_ID"),
|
|
2944
|
+
requirement(
|
|
2945
|
+
"registered-model",
|
|
2946
|
+
"Unity Catalog registered model",
|
|
2947
|
+
"DBX_TEST_REGISTERED_MODEL"
|
|
2948
|
+
),
|
|
2949
|
+
requirement("model-version", "Registered model version", "DBX_TEST_MODEL_VERSION")
|
|
2950
|
+
],
|
|
2951
|
+
docsUrl: `${DOCS2}#mlflow-and-model-lifecycle`,
|
|
2952
|
+
certificationScopes: [
|
|
2953
|
+
`${AZURE_USER_SCOPE}; MLflow experiment and Unity Catalog model version 20`
|
|
2954
|
+
]
|
|
2955
|
+
});
|
|
2956
|
+
}
|
|
2957
|
+
function createFeatureEngineeringTargetPack() {
|
|
2958
|
+
return defineTargetPack({
|
|
2959
|
+
id: "feature-engineering-serving",
|
|
2960
|
+
name: "Feature engineering and serving",
|
|
2961
|
+
description: "Unity Catalog feature tables, online materialization state and freshness assertions.",
|
|
2962
|
+
version: "0.1.0",
|
|
2963
|
+
maturity: "live-gated",
|
|
2964
|
+
capabilities: [
|
|
2965
|
+
"features.tables",
|
|
2966
|
+
"features.online-tables",
|
|
2967
|
+
"features.freshness",
|
|
2968
|
+
"features.serving"
|
|
2969
|
+
],
|
|
2970
|
+
checks: () => [restrictedPrincipalCheck(), featureEngineeringServingCheck()],
|
|
2971
|
+
requiredChecks: ["feature-engineering"],
|
|
2972
|
+
requirements: [
|
|
2973
|
+
workspace,
|
|
2974
|
+
auth,
|
|
2975
|
+
warehouse,
|
|
2976
|
+
requirement("feature-table", "Unity Catalog feature table", "DBX_TEST_FEATURE_TABLE"),
|
|
2977
|
+
requirement(
|
|
2978
|
+
"online-materialization",
|
|
2979
|
+
"Databricks Online Table or Synced Table pipeline",
|
|
2980
|
+
"DBX_TEST_ONLINE_TABLE",
|
|
2981
|
+
"DBX_TEST_FEATURE_SERVING_PIPELINE_ID"
|
|
2982
|
+
),
|
|
2983
|
+
requirement(
|
|
2984
|
+
"feature-assertion",
|
|
2985
|
+
"Feature freshness/correctness SQL",
|
|
2986
|
+
"DBX_TEST_FEATURE_ASSERTION_SQL"
|
|
2987
|
+
)
|
|
2988
|
+
],
|
|
2989
|
+
docsUrl: `${DOCS2}#feature-engineering-and-serving`,
|
|
2990
|
+
certificationScopes: [
|
|
2991
|
+
`${AZURE_USER_SCOPE}; Delta feature table, PostgreSQL-backed Synced Table, and SQL freshness`
|
|
2992
|
+
]
|
|
2993
|
+
});
|
|
2994
|
+
}
|
|
2995
|
+
function createModelServingTargetPack() {
|
|
2996
|
+
return defineTargetPack({
|
|
2997
|
+
id: "model-serving-gateway",
|
|
2998
|
+
name: "Model Serving and AI Gateway",
|
|
2999
|
+
description: "Serving readiness, inference behavior and optional AI Gateway policy presence.",
|
|
3000
|
+
version: "0.1.0",
|
|
3001
|
+
maturity: "live-gated",
|
|
3002
|
+
capabilities: [
|
|
3003
|
+
"model-serving.readiness",
|
|
3004
|
+
"model-serving.inference",
|
|
3005
|
+
"ai-gateway.configuration"
|
|
3006
|
+
],
|
|
3007
|
+
checks: () => [restrictedPrincipalCheck(), modelServingCheck()],
|
|
3008
|
+
requiredChecks: ["model-serving"],
|
|
3009
|
+
requirements: [
|
|
3010
|
+
workspace,
|
|
3011
|
+
auth,
|
|
3012
|
+
requirement("serving-endpoint", "Model Serving endpoint name", "DBX_TEST_SERVING_ENDPOINT"),
|
|
3013
|
+
requirement(
|
|
3014
|
+
"serving-request",
|
|
3015
|
+
"Safe inference request JSON",
|
|
3016
|
+
"DBX_TEST_SERVING_REQUEST_JSON"
|
|
3017
|
+
),
|
|
3018
|
+
requirement(
|
|
3019
|
+
"ai-gateway",
|
|
3020
|
+
"AI Gateway enforcement flag set to 1",
|
|
3021
|
+
"DBX_TEST_AI_GATEWAY_REQUIRED"
|
|
3022
|
+
)
|
|
3023
|
+
],
|
|
3024
|
+
docsUrl: `${DOCS2}#model-serving-and-ai-gateway`,
|
|
3025
|
+
certificationScopes: [
|
|
3026
|
+
`${AZURE_USER_SCOPE}; custom-model inference and AI Gateway inference-table configuration`
|
|
3027
|
+
]
|
|
3028
|
+
});
|
|
3029
|
+
}
|
|
3030
|
+
function createVectorRagAgentsTargetPack() {
|
|
3031
|
+
return defineTargetPack({
|
|
3032
|
+
id: "vector-rag-agents",
|
|
3033
|
+
name: "Vector Search, RAG and agents",
|
|
3034
|
+
description: "Vector endpoint/index readiness, retrieval behavior and agent endpoint inference.",
|
|
3035
|
+
version: "0.1.0",
|
|
3036
|
+
maturity: "live-gated",
|
|
3037
|
+
capabilities: [
|
|
3038
|
+
"vector-search.endpoint",
|
|
3039
|
+
"vector-search.index",
|
|
3040
|
+
"rag.retrieval",
|
|
3041
|
+
"agents.serving"
|
|
3042
|
+
],
|
|
3043
|
+
checks: () => [restrictedPrincipalCheck(), vectorSearchCheck(), ragAgentCheck()],
|
|
3044
|
+
requiredChecks: ["vector-search", "rag-agent"],
|
|
3045
|
+
requirements: [
|
|
3046
|
+
workspace,
|
|
3047
|
+
auth,
|
|
3048
|
+
requirement("vector-endpoint", "Vector Search endpoint", "DBX_TEST_VECTOR_ENDPOINT"),
|
|
3049
|
+
requirement("vector-index", "Vector Search index", "DBX_TEST_VECTOR_INDEX"),
|
|
3050
|
+
requirement("vector-query", "Vector retrieval request JSON", "DBX_TEST_VECTOR_QUERY_JSON"),
|
|
3051
|
+
requirement("agent-endpoint", "Agent serving endpoint", "DBX_TEST_AGENT_ENDPOINT"),
|
|
3052
|
+
requirement("agent-request", "Safe agent request JSON", "DBX_TEST_AGENT_REQUEST_JSON")
|
|
3053
|
+
],
|
|
3054
|
+
docsUrl: `${DOCS2}#vector-search-rag-and-agents`,
|
|
3055
|
+
certificationScopes: [
|
|
3056
|
+
`${AZURE_USER_SCOPE}; Delta Sync vector retrieval and custom PyFunc agent invocation`
|
|
3057
|
+
]
|
|
3058
|
+
});
|
|
3059
|
+
}
|
|
3060
|
+
function createFederationSharingTargetPack() {
|
|
3061
|
+
return defineTargetPack({
|
|
3062
|
+
id: "federation-sharing-cleanrooms",
|
|
3063
|
+
name: "Federation, Delta Sharing and Clean Rooms",
|
|
3064
|
+
description: "Federated connection, data-plane query, Delta Share and Clean Room governance access.",
|
|
3065
|
+
version: "0.1.0",
|
|
3066
|
+
maturity: "shipped",
|
|
3067
|
+
capabilities: [
|
|
3068
|
+
"federation.connections",
|
|
3069
|
+
"federation.query",
|
|
3070
|
+
"delta-sharing.shares",
|
|
3071
|
+
"clean-rooms.access"
|
|
3072
|
+
],
|
|
3073
|
+
checks: () => [restrictedPrincipalCheck(), federationSharingCleanRoomsCheck()],
|
|
3074
|
+
requiredChecks: ["federation-sharing-cleanrooms"],
|
|
3075
|
+
requirements: [
|
|
3076
|
+
workspace,
|
|
3077
|
+
auth,
|
|
3078
|
+
warehouse,
|
|
3079
|
+
requirement("connection", "Lakehouse Federation connection", "DBX_TEST_CONNECTION"),
|
|
3080
|
+
requirement("share", "Delta Sharing share", "DBX_TEST_SHARE"),
|
|
3081
|
+
requirement("clean-room", "Clean Room name", "DBX_TEST_CLEAN_ROOM"),
|
|
3082
|
+
requirement(
|
|
3083
|
+
"federation-assertion",
|
|
3084
|
+
"Federated data-plane SQL",
|
|
3085
|
+
"DBX_TEST_FEDERATION_ASSERTION_SQL"
|
|
3086
|
+
)
|
|
3087
|
+
],
|
|
3088
|
+
docsUrl: `${DOCS2}#federation-delta-sharing-and-clean-rooms`,
|
|
3089
|
+
certificationScopes: [SHIPPED_SCOPE]
|
|
3090
|
+
});
|
|
3091
|
+
}
|
|
3092
|
+
function createSecurityCostDrTargetPack() {
|
|
3093
|
+
return defineTargetPack({
|
|
3094
|
+
id: "security-cost-dr",
|
|
3095
|
+
name: "Networking, security posture, cost and regional DR",
|
|
3096
|
+
description: "IP controls, compute policy, cost guardrail and authenticated secondary-region readiness.",
|
|
3097
|
+
version: "0.1.0",
|
|
3098
|
+
maturity: "shipped",
|
|
3099
|
+
capabilities: [
|
|
3100
|
+
"network.ip-access",
|
|
3101
|
+
"security.compute-policy",
|
|
3102
|
+
"cost.system-tables",
|
|
3103
|
+
"dr.secondary-region"
|
|
3104
|
+
],
|
|
3105
|
+
checks: () => [restrictedPrincipalCheck(), securityPostureCostCheck(), regionalDrCheck()],
|
|
3106
|
+
requiredChecks: ["security-cost", "regional-dr"],
|
|
3107
|
+
requirements: [
|
|
3108
|
+
workspace,
|
|
3109
|
+
auth,
|
|
3110
|
+
warehouse,
|
|
3111
|
+
requirement(
|
|
3112
|
+
"ip-access-list",
|
|
3113
|
+
"Enabled workspace IP access list",
|
|
3114
|
+
"DBX_TEST_IP_ACCESS_LIST_ID"
|
|
3115
|
+
),
|
|
3116
|
+
requirement("cluster-policy", "Compute policy ID", "DBX_TEST_CLUSTER_POLICY_ID"),
|
|
3117
|
+
requirement("cost-assertion", "Boolean SQL cost guardrail", "DBX_TEST_COST_ASSERTION_SQL"),
|
|
3118
|
+
requirement("dr-workspace", "Secondary workspace host", "DBX_TEST_DR_HOST"),
|
|
3119
|
+
requirement("dr-warehouse", "Running secondary-region warehouse", "DBX_TEST_DR_WAREHOUSE_ID"),
|
|
3120
|
+
requirement("dr-assertion", "Secondary-region workload SQL", "DBX_TEST_DR_ASSERTION_SQL")
|
|
3121
|
+
],
|
|
3122
|
+
docsUrl: `${DOCS2}#networking-security-cost-and-regional-dr`,
|
|
3123
|
+
certificationScopes: [SHIPPED_SCOPE]
|
|
3124
|
+
});
|
|
3125
|
+
}
|
|
3126
|
+
function createAdvancedTargetPacks() {
|
|
3127
|
+
return [
|
|
3128
|
+
createAdvancedStreamingTargetPack(),
|
|
3129
|
+
createAiBiGenieTargetPack(),
|
|
3130
|
+
createMlflowLifecycleTargetPack(),
|
|
3131
|
+
createFeatureEngineeringTargetPack(),
|
|
3132
|
+
createModelServingTargetPack(),
|
|
3133
|
+
createVectorRagAgentsTargetPack(),
|
|
3134
|
+
createFederationSharingTargetPack(),
|
|
3135
|
+
createSecurityCostDrTargetPack()
|
|
3136
|
+
];
|
|
3137
|
+
}
|
|
3138
|
+
|
|
2056
3139
|
// src/builtin-packs.ts
|
|
2057
3140
|
function createBuiltinTargetPacks() {
|
|
2058
|
-
return [
|
|
3141
|
+
return [
|
|
3142
|
+
...createFoundationTargetPacks(),
|
|
3143
|
+
createLakeflowJobsTargetPack(),
|
|
3144
|
+
...createAdvancedTargetPacks()
|
|
3145
|
+
];
|
|
2059
3146
|
}
|
|
2060
3147
|
var builtinTargetPacks = createBuiltinTargetPacks();
|
|
2061
3148
|
|
|
@@ -2067,7 +3154,10 @@ Object.defineProperty(exports, "waitForDatabricksStatement", {
|
|
|
2067
3154
|
enumerable: true,
|
|
2068
3155
|
get: function () { return databricks.waitForDatabricksStatement; }
|
|
2069
3156
|
});
|
|
3157
|
+
exports.DatabricksAdvancedWorkloadsAdapter = DatabricksAdvancedWorkloadsAdapter;
|
|
2070
3158
|
exports.DatabricksJobsAdapter = DatabricksJobsAdapter;
|
|
3159
|
+
exports.advancedStreamingCdcCheck = advancedStreamingCdcCheck;
|
|
3160
|
+
exports.aiBiDashboardCheck = aiBiDashboardCheck;
|
|
2071
3161
|
exports.appHealthCheck = appHealthCheck;
|
|
2072
3162
|
exports.assertDeltaContract = assertDeltaContract;
|
|
2073
3163
|
exports.autoLoaderIngestionCheck = autoLoaderIngestionCheck;
|
|
@@ -2075,21 +3165,30 @@ exports.backupRestoreCheck = backupRestoreCheck;
|
|
|
2075
3165
|
exports.builtinTargetPacks = builtinTargetPacks;
|
|
2076
3166
|
exports.classicComputeCheck = classicComputeCheck;
|
|
2077
3167
|
exports.compareToGolden = compareToGolden;
|
|
3168
|
+
exports.createAdvancedStreamingTargetPack = createAdvancedStreamingTargetPack;
|
|
3169
|
+
exports.createAdvancedTargetPacks = createAdvancedTargetPacks;
|
|
3170
|
+
exports.createAiBiGenieTargetPack = createAiBiGenieTargetPack;
|
|
2078
3171
|
exports.createAppsOperationalFoundationPack = createAppsOperationalFoundationPack;
|
|
2079
3172
|
exports.createBuiltinTargetPacks = createBuiltinTargetPacks;
|
|
2080
3173
|
exports.createClientFromEnv = createClientFromEnv;
|
|
2081
3174
|
exports.createDuckDbContext = createDuckDbContext;
|
|
2082
3175
|
exports.createEphemeralLakebaseBranch = createEphemeralLakebaseBranch;
|
|
2083
3176
|
exports.createExecutionContext = createExecutionContext;
|
|
3177
|
+
exports.createFeatureEngineeringTargetPack = createFeatureEngineeringTargetPack;
|
|
3178
|
+
exports.createFederationSharingTargetPack = createFederationSharingTargetPack;
|
|
2084
3179
|
exports.createFoundationTargetPacks = createFoundationTargetPacks;
|
|
2085
3180
|
exports.createJobsCodeFoundationPack = createJobsCodeFoundationPack;
|
|
2086
3181
|
exports.createLakebaseTestProvider = createLakebaseTestProvider;
|
|
2087
3182
|
exports.createLakeflowFoundationPack = createLakeflowFoundationPack;
|
|
2088
3183
|
exports.createLakeflowJobsTargetPack = createLakeflowJobsTargetPack;
|
|
3184
|
+
exports.createMlflowLifecycleTargetPack = createMlflowLifecycleTargetPack;
|
|
2089
3185
|
exports.createMockDatabricksClient = createMockDatabricksClient;
|
|
3186
|
+
exports.createModelServingTargetPack = createModelServingTargetPack;
|
|
3187
|
+
exports.createSecurityCostDrTargetPack = createSecurityCostDrTargetPack;
|
|
2090
3188
|
exports.createSqlDeltaFoundationPack = createSqlDeltaFoundationPack;
|
|
2091
3189
|
exports.createTargetPackRegistry = createTargetPackRegistry;
|
|
2092
3190
|
exports.createUnityStorageFoundationPack = createUnityStorageFoundationPack;
|
|
3191
|
+
exports.createVectorRagAgentsTargetPack = createVectorRagAgentsTargetPack;
|
|
2093
3192
|
exports.createWorkspaceContext = createWorkspaceContext;
|
|
2094
3193
|
exports.customLiveCheck = customLiveCheck;
|
|
2095
3194
|
exports.dbtBuildCheck = dbtBuildCheck;
|
|
@@ -2104,11 +3203,18 @@ exports.ensureVolumeDirectory = ensureVolumeDirectory;
|
|
|
2104
3203
|
exports.expectQueryToMatchGolden = expectQueryToMatchGolden;
|
|
2105
3204
|
exports.expectTable = expectTable;
|
|
2106
3205
|
exports.failureRecoveryCheck = failureRecoveryCheck;
|
|
3206
|
+
exports.featureEngineeringServingCheck = featureEngineeringServingCheck;
|
|
3207
|
+
exports.federationSharingCleanRoomsCheck = federationSharingCleanRoomsCheck;
|
|
2107
3208
|
exports.foundationTargetPacks = foundationTargetPacks;
|
|
3209
|
+
exports.genieCheck = genieCheck;
|
|
2108
3210
|
exports.jobRunCheck = jobRunCheck;
|
|
3211
|
+
exports.jobsCertificationCheck = jobsCertificationCheck;
|
|
2109
3212
|
exports.jobsOrchestrationCheck = jobsOrchestrationCheck;
|
|
2110
3213
|
exports.lakebaseCredentialCheck = lakebaseCredentialCheck;
|
|
2111
3214
|
exports.lakebaseRoundTripCheck = lakebaseRoundTripCheck;
|
|
3215
|
+
exports.lakeflowConnectCheck = lakeflowConnectCheck;
|
|
3216
|
+
exports.mlflowLifecycleCheck = mlflowLifecycleCheck;
|
|
3217
|
+
exports.modelServingCheck = modelServingCheck;
|
|
2112
3218
|
exports.normalizeJobRun = normalizeJobRun;
|
|
2113
3219
|
exports.normalizeRow = normalizeRow;
|
|
2114
3220
|
exports.normalizeVolumeRoot = normalizeVolumeRoot;
|
|
@@ -2118,6 +3224,8 @@ exports.parseJobOrchestrationLiveSpec = parseJobOrchestrationLiveSpec;
|
|
|
2118
3224
|
exports.parseStatementRows = parseStatementRows;
|
|
2119
3225
|
exports.performanceBudgetCheck = performanceBudgetCheck;
|
|
2120
3226
|
exports.pipelineRefreshCheck = pipelineRefreshCheck;
|
|
3227
|
+
exports.ragAgentCheck = ragAgentCheck;
|
|
3228
|
+
exports.regionalDrCheck = regionalDrCheck;
|
|
2121
3229
|
exports.renderJUnit = renderJUnit;
|
|
2122
3230
|
exports.resolveFixtureRows = resolveFixtureRows;
|
|
2123
3231
|
exports.resolveLakebaseProject = resolveLakebaseProject;
|
|
@@ -2129,6 +3237,7 @@ exports.runLiveSuite = runLiveSuite;
|
|
|
2129
3237
|
exports.runTargetPack = runTargetPack;
|
|
2130
3238
|
exports.secretRotationCheck = secretRotationCheck;
|
|
2131
3239
|
exports.secretScopeCheck = secretScopeCheck;
|
|
3240
|
+
exports.securityPostureCostCheck = securityPostureCostCheck;
|
|
2132
3241
|
exports.serverlessComputeCheck = serverlessComputeCheck;
|
|
2133
3242
|
exports.sqlRoundTripCheck = sqlRoundTripCheck;
|
|
2134
3243
|
exports.toLakebaseBranchId = toLakebaseBranchId;
|
|
@@ -2136,6 +3245,7 @@ exports.toNamedParameters = toNamedParameters;
|
|
|
2136
3245
|
exports.unityCatalogAccessCheck = unityCatalogAccessCheck;
|
|
2137
3246
|
exports.uploadVolumeFile = uploadVolumeFile;
|
|
2138
3247
|
exports.validateJobGraph = validateJobGraph;
|
|
3248
|
+
exports.vectorSearchCheck = vectorSearchCheck;
|
|
2139
3249
|
exports.volumeIOCheck = volumeIOCheck;
|
|
2140
3250
|
exports.waitForPipelineUpdate = waitForPipelineUpdate;
|
|
2141
3251
|
//# sourceMappingURL=index.cjs.map
|