@fabricorg/databricks-testkit 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/dist/{chunk-FBADA7LQ.js → chunk-SCVVMUAF.js} +50 -3
- package/dist/chunk-SCVVMUAF.js.map +1 -0
- package/dist/index.cjs +593 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +241 -2
- package/dist/index.d.ts +241 -2
- package/dist/index.js +529 -5
- package/dist/index.js.map +1 -1
- package/dist/parity.cjs +48 -1
- package/dist/parity.cjs.map +1 -1
- package/dist/parity.js +1 -1
- package/dist/{workspace-context-AIUYISUS.js → workspace-context-VLBKMX3V.js} +3 -3
- package/dist/{workspace-context-AIUYISUS.js.map → workspace-context-VLBKMX3V.js.map} +1 -1
- package/package.json +2 -2
- package/dist/chunk-FBADA7LQ.js.map +0 -1
package/dist/index.cjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
var databricks = require('@fabric-harness/databricks');
|
|
4
3
|
var promises = require('fs/promises');
|
|
4
|
+
var databricks = require('@fabric-harness/databricks');
|
|
5
5
|
var path = require('path');
|
|
6
6
|
var experimentsDbPool = require('@fabricorg/experiments-db-pool');
|
|
7
7
|
var crypto = require('crypto');
|
|
@@ -340,6 +340,39 @@ function createClientFromEnv(env, fetchImpl) {
|
|
|
340
340
|
}
|
|
341
341
|
function resolveTokenProvider(host, env, fetchImpl = fetch) {
|
|
342
342
|
if (env.DATABRICKS_BEARER) return env.DATABRICKS_BEARER;
|
|
343
|
+
const authType = env.DATABRICKS_AUTH_TYPE;
|
|
344
|
+
if (authType === "env-oidc" || authType === "file-oidc") {
|
|
345
|
+
let cached = null;
|
|
346
|
+
return async () => {
|
|
347
|
+
if (cached && Date.now() < cached.expiresAt) return cached.token;
|
|
348
|
+
const subjectToken = await resolveFederatedSubjectToken(authType, env);
|
|
349
|
+
const params = new URLSearchParams({
|
|
350
|
+
grant_type: "urn:ietf:params:oauth:grant-type:token-exchange",
|
|
351
|
+
subject_token: subjectToken,
|
|
352
|
+
subject_token_type: "urn:ietf:params:oauth:token-type:jwt",
|
|
353
|
+
scope: env.DATABRICKS_SCOPE ?? "all-apis"
|
|
354
|
+
});
|
|
355
|
+
if (env.DATABRICKS_CLIENT_ID) params.set("client_id", env.DATABRICKS_CLIENT_ID);
|
|
356
|
+
const response = await fetchImpl(`${host.replace(/\/$/, "")}/oidc/v1/token`, {
|
|
357
|
+
method: "POST",
|
|
358
|
+
headers: { "content-type": "application/x-www-form-urlencoded" },
|
|
359
|
+
body: params.toString()
|
|
360
|
+
});
|
|
361
|
+
if (!response.ok) {
|
|
362
|
+
throw new Error(`OAuth federated token exchange failed: ${response.status}`);
|
|
363
|
+
}
|
|
364
|
+
const body = await response.json();
|
|
365
|
+
if (typeof body.access_token !== "string" || !body.access_token) {
|
|
366
|
+
throw new Error("OAuth federated token exchange returned no access_token");
|
|
367
|
+
}
|
|
368
|
+
const expiresIn = typeof body.expires_in === "number" ? body.expires_in : 3600;
|
|
369
|
+
cached = {
|
|
370
|
+
token: body.access_token,
|
|
371
|
+
expiresAt: Date.now() + Math.max(1, expiresIn - 300) * 1e3
|
|
372
|
+
};
|
|
373
|
+
return body.access_token;
|
|
374
|
+
};
|
|
375
|
+
}
|
|
343
376
|
const clientId = env.DATABRICKS_CLIENT_ID;
|
|
344
377
|
const clientSecret = env.DATABRICKS_CLIENT_SECRET;
|
|
345
378
|
if (clientId && clientSecret) {
|
|
@@ -370,9 +403,22 @@ function resolveTokenProvider(host, env, fetchImpl = fetch) {
|
|
|
370
403
|
}
|
|
371
404
|
if (env.DATABRICKS_TOKEN) return env.DATABRICKS_TOKEN;
|
|
372
405
|
throw new Error(
|
|
373
|
-
"Live profile requires DATABRICKS_BEARER, DATABRICKS_CLIENT_ID/SECRET, or DATABRICKS_TOKEN"
|
|
406
|
+
"Live profile requires DATABRICKS_BEARER, env/file OIDC federation, DATABRICKS_CLIENT_ID/SECRET, or DATABRICKS_TOKEN"
|
|
374
407
|
);
|
|
375
408
|
}
|
|
409
|
+
async function resolveFederatedSubjectToken(authType, env) {
|
|
410
|
+
if (authType === "env-oidc") {
|
|
411
|
+
const tokenEnv = env.DATABRICKS_OIDC_TOKEN_ENV ?? "DATABRICKS_OIDC_TOKEN";
|
|
412
|
+
const token2 = env[tokenEnv]?.trim();
|
|
413
|
+
if (!token2) throw new Error(`Federated OIDC token environment variable ${tokenEnv} is empty`);
|
|
414
|
+
return token2;
|
|
415
|
+
}
|
|
416
|
+
const path = env.DATABRICKS_OIDC_TOKEN_FILEPATH;
|
|
417
|
+
if (!path) throw new Error("DATABRICKS_OIDC_TOKEN_FILEPATH is required for file-oidc");
|
|
418
|
+
const token = (await promises.readFile(path, "utf8")).trim();
|
|
419
|
+
if (!token) throw new Error(`Federated OIDC token file ${path} is empty`);
|
|
420
|
+
return token;
|
|
421
|
+
}
|
|
376
422
|
var NUMERIC_TYPES;
|
|
377
423
|
var init_workspace_context = __esm({
|
|
378
424
|
"src/workspace-context.ts"() {
|
|
@@ -608,7 +654,8 @@ async function runLiveSuite(spec, runtime = {}) {
|
|
|
608
654
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
609
655
|
success,
|
|
610
656
|
required: required2,
|
|
611
|
-
results
|
|
657
|
+
results,
|
|
658
|
+
...spec.targetPack ? { targetPack: spec.targetPack } : {}
|
|
612
659
|
};
|
|
613
660
|
if (spec.evidencePath)
|
|
614
661
|
await writeReport(spec.evidencePath, `${JSON.stringify(evidence, null, 2)}
|
|
@@ -1489,6 +1536,529 @@ function classicComputeCheck() {
|
|
|
1489
1536
|
};
|
|
1490
1537
|
}
|
|
1491
1538
|
|
|
1539
|
+
// src/target-packs.ts
|
|
1540
|
+
var TARGET_PACK_ID = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/;
|
|
1541
|
+
function defineTargetPack(spec) {
|
|
1542
|
+
if (!TARGET_PACK_ID.test(spec.id)) {
|
|
1543
|
+
throw new Error(`Target pack id '${spec.id}' must be lowercase kebab-case`);
|
|
1544
|
+
}
|
|
1545
|
+
if (!spec.name.trim()) throw new Error(`Target pack '${spec.id}' requires a name`);
|
|
1546
|
+
if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(spec.version)) {
|
|
1547
|
+
throw new Error(
|
|
1548
|
+
`Target pack '${spec.id}' version '${spec.version}' is not semantic versioning`
|
|
1549
|
+
);
|
|
1550
|
+
}
|
|
1551
|
+
if (spec.capabilities.length === 0) {
|
|
1552
|
+
throw new Error(`Target pack '${spec.id}' must declare at least one capability`);
|
|
1553
|
+
}
|
|
1554
|
+
const createChecks = typeof spec.checks === "function" ? spec.checks : () => spec.checks;
|
|
1555
|
+
const validateChecks = () => {
|
|
1556
|
+
const checks = [...createChecks()];
|
|
1557
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1558
|
+
for (const check of checks) {
|
|
1559
|
+
if (!check.id.trim())
|
|
1560
|
+
throw new Error(`Target pack '${spec.id}' contains a check without an id`);
|
|
1561
|
+
if (ids.has(check.id)) {
|
|
1562
|
+
throw new Error(`Target pack '${spec.id}' contains duplicate check '${check.id}'`);
|
|
1563
|
+
}
|
|
1564
|
+
ids.add(check.id);
|
|
1565
|
+
}
|
|
1566
|
+
for (const required2 of spec.requiredChecks ?? []) {
|
|
1567
|
+
if (!ids.has(required2)) {
|
|
1568
|
+
throw new Error(`Target pack '${spec.id}' requires unknown check '${required2}'`);
|
|
1569
|
+
}
|
|
1570
|
+
}
|
|
1571
|
+
return checks;
|
|
1572
|
+
};
|
|
1573
|
+
validateChecks();
|
|
1574
|
+
return Object.freeze({
|
|
1575
|
+
...spec,
|
|
1576
|
+
capabilities: Object.freeze([...spec.capabilities]),
|
|
1577
|
+
requiredChecks: Object.freeze([...spec.requiredChecks ?? []]),
|
|
1578
|
+
requirements: Object.freeze([...spec.requirements ?? []]),
|
|
1579
|
+
certificationScopes: Object.freeze([...spec.certificationScopes ?? []]),
|
|
1580
|
+
checks: validateChecks
|
|
1581
|
+
});
|
|
1582
|
+
}
|
|
1583
|
+
function createTargetPackRegistry(packs) {
|
|
1584
|
+
const registry = /* @__PURE__ */ new Map();
|
|
1585
|
+
for (const pack of packs) {
|
|
1586
|
+
if (registry.has(pack.id)) throw new Error(`Duplicate target pack '${pack.id}'`);
|
|
1587
|
+
registry.set(pack.id, pack);
|
|
1588
|
+
}
|
|
1589
|
+
return registry;
|
|
1590
|
+
}
|
|
1591
|
+
function doctorTargetPack(pack, env = process.env, requiredChecks = pack.requiredChecks ?? []) {
|
|
1592
|
+
const requirements = (pack.requirements ?? []).map((requirement) => ({
|
|
1593
|
+
id: requirement.id,
|
|
1594
|
+
description: requirement.description,
|
|
1595
|
+
satisfied: requirement.anyOf.some((name) => Boolean(env[name]?.trim())),
|
|
1596
|
+
required: requirement.required !== false,
|
|
1597
|
+
variables: [...requirement.anyOf]
|
|
1598
|
+
}));
|
|
1599
|
+
const required2 = new Set(requiredChecks);
|
|
1600
|
+
const checks = pack.checks().map((check) => ({
|
|
1601
|
+
id: check.id,
|
|
1602
|
+
description: check.description,
|
|
1603
|
+
configured: check.configured ? check.configured(env) : true,
|
|
1604
|
+
required: required2.has(check.id)
|
|
1605
|
+
}));
|
|
1606
|
+
const known = new Set(checks.map((check) => check.id));
|
|
1607
|
+
const missingRequirements = requirements.filter((requirement) => requirement.required && !requirement.satisfied).map((requirement) => requirement.id);
|
|
1608
|
+
const missingRequiredChecks = [
|
|
1609
|
+
...checks.filter((check) => check.required && !check.configured).map((check) => check.id),
|
|
1610
|
+
...[...required2].filter((id) => !known.has(id))
|
|
1611
|
+
];
|
|
1612
|
+
return {
|
|
1613
|
+
pack: {
|
|
1614
|
+
id: pack.id,
|
|
1615
|
+
name: pack.name,
|
|
1616
|
+
version: pack.version,
|
|
1617
|
+
maturity: pack.maturity
|
|
1618
|
+
},
|
|
1619
|
+
ready: missingRequirements.length === 0 && missingRequiredChecks.length === 0,
|
|
1620
|
+
requirements,
|
|
1621
|
+
checks,
|
|
1622
|
+
missingRequirements,
|
|
1623
|
+
missingRequiredChecks
|
|
1624
|
+
};
|
|
1625
|
+
}
|
|
1626
|
+
function defineTargetPackRun(pack, options = {}) {
|
|
1627
|
+
const env = options.env ?? process.env;
|
|
1628
|
+
const requiredChecks = [...options.requiredChecks ?? pack.requiredChecks ?? []];
|
|
1629
|
+
const doctor = doctorTargetPack(pack, env, requiredChecks);
|
|
1630
|
+
if (!doctor.ready) {
|
|
1631
|
+
const missing = [...doctor.missingRequirements, ...doctor.missingRequiredChecks].join(", ");
|
|
1632
|
+
throw new Error(`Target pack '${pack.id}' is not configured: ${missing}`);
|
|
1633
|
+
}
|
|
1634
|
+
const targetPack = evidenceMetadata(pack, env);
|
|
1635
|
+
const suite = defineLiveSuite({
|
|
1636
|
+
checks: pack.checks(),
|
|
1637
|
+
required: requiredChecks,
|
|
1638
|
+
env,
|
|
1639
|
+
evidencePath: options.evidencePath,
|
|
1640
|
+
junitPath: options.junitPath,
|
|
1641
|
+
targetPack
|
|
1642
|
+
});
|
|
1643
|
+
return { pack, run: (runtime) => suite.run(runtime) };
|
|
1644
|
+
}
|
|
1645
|
+
async function runTargetPack(pack, options = {}, runtime) {
|
|
1646
|
+
return defineTargetPackRun(pack, options).run(runtime);
|
|
1647
|
+
}
|
|
1648
|
+
function evidenceMetadata(pack, env) {
|
|
1649
|
+
return {
|
|
1650
|
+
id: pack.id,
|
|
1651
|
+
name: pack.name,
|
|
1652
|
+
version: pack.version,
|
|
1653
|
+
maturity: pack.maturity,
|
|
1654
|
+
capabilities: [...pack.capabilities],
|
|
1655
|
+
...pack.docsUrl ? { docsUrl: pack.docsUrl } : {},
|
|
1656
|
+
certificationScopes: [...pack.certificationScopes ?? []],
|
|
1657
|
+
target: compact({
|
|
1658
|
+
cloud: env.DBX_TEST_CLOUD,
|
|
1659
|
+
region: env.DBX_TEST_REGION,
|
|
1660
|
+
workspace: env.DATABRICKS_HOST,
|
|
1661
|
+
runtime: env.DBX_TEST_RUNTIME_VERSION,
|
|
1662
|
+
compute: env.DBX_TEST_COMPUTE
|
|
1663
|
+
})
|
|
1664
|
+
};
|
|
1665
|
+
}
|
|
1666
|
+
function compact(value) {
|
|
1667
|
+
return Object.fromEntries(
|
|
1668
|
+
Object.entries(value).filter((entry) => Boolean(entry[1]))
|
|
1669
|
+
);
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
// src/foundation-packs.ts
|
|
1673
|
+
var DOCS = "https://experiments.fabric.pro/docs/testing/target-packs/";
|
|
1674
|
+
var AZURE_SCOPE = "Azure Databricks live-gated; see certification evidence for exact runtime";
|
|
1675
|
+
var workspaceRequirement = {
|
|
1676
|
+
id: "workspace",
|
|
1677
|
+
description: "Databricks workspace host",
|
|
1678
|
+
anyOf: ["DATABRICKS_HOST"]
|
|
1679
|
+
};
|
|
1680
|
+
var authRequirement = {
|
|
1681
|
+
id: "authentication",
|
|
1682
|
+
description: "Databricks OAuth, OIDC, bearer, or PAT authentication",
|
|
1683
|
+
anyOf: [
|
|
1684
|
+
"DATABRICKS_BEARER",
|
|
1685
|
+
"DATABRICKS_TOKEN",
|
|
1686
|
+
"DATABRICKS_CLIENT_ID",
|
|
1687
|
+
"DATABRICKS_OIDC_TOKEN",
|
|
1688
|
+
"DATABRICKS_OIDC_TOKEN_FILEPATH"
|
|
1689
|
+
],
|
|
1690
|
+
secret: true
|
|
1691
|
+
};
|
|
1692
|
+
var warehouseRequirement = {
|
|
1693
|
+
id: "warehouse",
|
|
1694
|
+
description: "SQL warehouse ID or HTTP path",
|
|
1695
|
+
anyOf: ["DATABRICKS_WAREHOUSE_ID", "DATABRICKS_HTTP_PATH"]
|
|
1696
|
+
};
|
|
1697
|
+
function createSqlDeltaFoundationPack() {
|
|
1698
|
+
return defineTargetPack({
|
|
1699
|
+
id: "sql-delta",
|
|
1700
|
+
name: "SQL and Delta foundation",
|
|
1701
|
+
description: "SQL Statement Execution, Delta contracts, performance, recovery, backup and rollback.",
|
|
1702
|
+
version: "1.0.0",
|
|
1703
|
+
maturity: "live-gated",
|
|
1704
|
+
capabilities: [
|
|
1705
|
+
"sql.statement-execution",
|
|
1706
|
+
"delta.contract",
|
|
1707
|
+
"delta.time-travel",
|
|
1708
|
+
"performance.sql",
|
|
1709
|
+
"recovery.sql",
|
|
1710
|
+
"backup-restore.delta",
|
|
1711
|
+
"rollback.delta"
|
|
1712
|
+
],
|
|
1713
|
+
checks: () => [
|
|
1714
|
+
restrictedPrincipalCheck(),
|
|
1715
|
+
sqlRoundTripCheck(),
|
|
1716
|
+
deltaContractCheck(),
|
|
1717
|
+
serverlessComputeCheck(),
|
|
1718
|
+
performanceBudgetCheck(),
|
|
1719
|
+
failureRecoveryCheck(),
|
|
1720
|
+
backupRestoreCheck(),
|
|
1721
|
+
rollbackCheck()
|
|
1722
|
+
],
|
|
1723
|
+
requiredChecks: ["sql"],
|
|
1724
|
+
requirements: [workspaceRequirement, authRequirement, warehouseRequirement],
|
|
1725
|
+
docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
|
|
1726
|
+
certificationScopes: [AZURE_SCOPE]
|
|
1727
|
+
});
|
|
1728
|
+
}
|
|
1729
|
+
function createLakeflowFoundationPack() {
|
|
1730
|
+
return defineTargetPack({
|
|
1731
|
+
id: "lakeflow-ingestion",
|
|
1732
|
+
name: "Lakeflow ingestion foundation",
|
|
1733
|
+
description: "Lakeflow refresh, Volumes, Auto Loader ingestion and rescued-row evidence.",
|
|
1734
|
+
version: "1.0.0",
|
|
1735
|
+
maturity: "live-gated",
|
|
1736
|
+
capabilities: [
|
|
1737
|
+
"lakeflow.refresh",
|
|
1738
|
+
"autoloader.ingestion",
|
|
1739
|
+
"volumes.files",
|
|
1740
|
+
"delta.rescued-data"
|
|
1741
|
+
],
|
|
1742
|
+
checks: () => [
|
|
1743
|
+
restrictedPrincipalCheck(),
|
|
1744
|
+
pipelineRefreshCheck(),
|
|
1745
|
+
volumeIOCheck(),
|
|
1746
|
+
autoLoaderIngestionCheck()
|
|
1747
|
+
],
|
|
1748
|
+
requiredChecks: ["pipeline"],
|
|
1749
|
+
requirements: [workspaceRequirement, authRequirement],
|
|
1750
|
+
docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
|
|
1751
|
+
certificationScopes: [AZURE_SCOPE]
|
|
1752
|
+
});
|
|
1753
|
+
}
|
|
1754
|
+
function createJobsCodeFoundationPack() {
|
|
1755
|
+
return defineTargetPack({
|
|
1756
|
+
id: "jobs-code",
|
|
1757
|
+
name: "Jobs and code foundation",
|
|
1758
|
+
description: "Jobs, notebooks, dbt, serverless compute and classic compute probes.",
|
|
1759
|
+
version: "1.0.0",
|
|
1760
|
+
maturity: "live-gated",
|
|
1761
|
+
capabilities: [
|
|
1762
|
+
"jobs.run-now",
|
|
1763
|
+
"jobs.submit.notebook",
|
|
1764
|
+
"jobs.submit.dbt",
|
|
1765
|
+
"compute.serverless",
|
|
1766
|
+
"compute.classic"
|
|
1767
|
+
],
|
|
1768
|
+
checks: () => [
|
|
1769
|
+
restrictedPrincipalCheck(),
|
|
1770
|
+
jobRunCheck(),
|
|
1771
|
+
notebookSubmitCheck(),
|
|
1772
|
+
dbtBuildCheck(),
|
|
1773
|
+
serverlessComputeCheck(),
|
|
1774
|
+
classicComputeCheck()
|
|
1775
|
+
],
|
|
1776
|
+
requiredChecks: ["jobs"],
|
|
1777
|
+
requirements: [workspaceRequirement, authRequirement],
|
|
1778
|
+
docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
|
|
1779
|
+
certificationScopes: [AZURE_SCOPE]
|
|
1780
|
+
});
|
|
1781
|
+
}
|
|
1782
|
+
function createUnityStorageFoundationPack() {
|
|
1783
|
+
return defineTargetPack({
|
|
1784
|
+
id: "unity-storage",
|
|
1785
|
+
name: "Unity Catalog and storage foundation",
|
|
1786
|
+
description: "Unity Catalog allow/deny, Volumes, secret metadata and secret rotation.",
|
|
1787
|
+
version: "1.0.0",
|
|
1788
|
+
maturity: "live-gated",
|
|
1789
|
+
capabilities: [
|
|
1790
|
+
"unity-catalog.permissions",
|
|
1791
|
+
"volumes.files",
|
|
1792
|
+
"secrets.metadata",
|
|
1793
|
+
"secrets.rotation"
|
|
1794
|
+
],
|
|
1795
|
+
checks: () => [
|
|
1796
|
+
restrictedPrincipalCheck(),
|
|
1797
|
+
unityCatalogAccessCheck(),
|
|
1798
|
+
volumeIOCheck(),
|
|
1799
|
+
secretScopeCheck(),
|
|
1800
|
+
secretRotationCheck()
|
|
1801
|
+
],
|
|
1802
|
+
requiredChecks: ["uc-grants"],
|
|
1803
|
+
requirements: [workspaceRequirement, authRequirement, warehouseRequirement],
|
|
1804
|
+
docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
|
|
1805
|
+
certificationScopes: [AZURE_SCOPE]
|
|
1806
|
+
});
|
|
1807
|
+
}
|
|
1808
|
+
function createAppsOperationalFoundationPack() {
|
|
1809
|
+
return defineTargetPack({
|
|
1810
|
+
id: "apps-operational",
|
|
1811
|
+
name: "Apps and operational data foundation",
|
|
1812
|
+
description: "Databricks Apps resources and health plus Lakebase OAuth and pool refresh.",
|
|
1813
|
+
version: "1.0.0",
|
|
1814
|
+
maturity: "live-gated",
|
|
1815
|
+
capabilities: [
|
|
1816
|
+
"apps.health",
|
|
1817
|
+
"apps.resources",
|
|
1818
|
+
"lakebase.credentials",
|
|
1819
|
+
"lakebase.round-trip",
|
|
1820
|
+
"lakebase.pool-refresh"
|
|
1821
|
+
],
|
|
1822
|
+
checks: () => [
|
|
1823
|
+
restrictedPrincipalCheck(),
|
|
1824
|
+
appHealthCheck(),
|
|
1825
|
+
lakebaseCredentialCheck(),
|
|
1826
|
+
lakebaseRoundTripCheck()
|
|
1827
|
+
],
|
|
1828
|
+
requiredChecks: ["app"],
|
|
1829
|
+
requirements: [workspaceRequirement, authRequirement],
|
|
1830
|
+
docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
|
|
1831
|
+
certificationScopes: [AZURE_SCOPE]
|
|
1832
|
+
});
|
|
1833
|
+
}
|
|
1834
|
+
function createFoundationTargetPacks() {
|
|
1835
|
+
return [
|
|
1836
|
+
createSqlDeltaFoundationPack(),
|
|
1837
|
+
createLakeflowFoundationPack(),
|
|
1838
|
+
createJobsCodeFoundationPack(),
|
|
1839
|
+
createUnityStorageFoundationPack(),
|
|
1840
|
+
createAppsOperationalFoundationPack()
|
|
1841
|
+
];
|
|
1842
|
+
}
|
|
1843
|
+
var foundationTargetPacks = createFoundationTargetPacks();
|
|
1844
|
+
var DatabricksJobsAdapter = class {
|
|
1845
|
+
constructor(client) {
|
|
1846
|
+
this.client = client;
|
|
1847
|
+
}
|
|
1848
|
+
client;
|
|
1849
|
+
async submit(request) {
|
|
1850
|
+
validateJobGraph(request.tasks);
|
|
1851
|
+
const response = await this.client.post(
|
|
1852
|
+
"/api/2.1/jobs/runs/submit",
|
|
1853
|
+
request
|
|
1854
|
+
);
|
|
1855
|
+
if (!response.run_id) throw new Error("Databricks Jobs submit returned no run_id");
|
|
1856
|
+
return response.run_id;
|
|
1857
|
+
}
|
|
1858
|
+
async get(runId) {
|
|
1859
|
+
const raw = await this.client.get("/api/2.1/jobs/runs/get", {
|
|
1860
|
+
run_id: runId,
|
|
1861
|
+
include_history: true
|
|
1862
|
+
});
|
|
1863
|
+
return normalizeJobRun(runId, raw);
|
|
1864
|
+
}
|
|
1865
|
+
async wait(runId, options = {}) {
|
|
1866
|
+
const raw = await databricks.waitForDatabricksRun(this.client, runId, {
|
|
1867
|
+
timeoutMs: options.timeoutMs,
|
|
1868
|
+
pollIntervalMs: options.pollIntervalMs
|
|
1869
|
+
});
|
|
1870
|
+
return normalizeJobRun(runId, raw);
|
|
1871
|
+
}
|
|
1872
|
+
async cancel(runId) {
|
|
1873
|
+
await this.client.post("/api/2.1/jobs/runs/cancel", { run_id: runId });
|
|
1874
|
+
}
|
|
1875
|
+
async repair(runId, options = {}) {
|
|
1876
|
+
const response = await this.client.post("/api/2.1/jobs/runs/repair", {
|
|
1877
|
+
run_id: runId,
|
|
1878
|
+
...options.latestRepairId ? { latest_repair_id: options.latestRepairId } : {},
|
|
1879
|
+
...options.rerunTasks?.length ? { rerun_tasks: options.rerunTasks } : { rerun_all_failed_tasks: true }
|
|
1880
|
+
});
|
|
1881
|
+
if (!response.repair_id) throw new Error("Databricks Jobs repair returned no repair_id");
|
|
1882
|
+
return response.repair_id;
|
|
1883
|
+
}
|
|
1884
|
+
};
|
|
1885
|
+
function validateJobGraph(tasks) {
|
|
1886
|
+
if (tasks.length === 0) throw new Error("Jobs orchestration requires at least one task");
|
|
1887
|
+
const keys = /* @__PURE__ */ new Set();
|
|
1888
|
+
for (const task of tasks) {
|
|
1889
|
+
if (!/^[A-Za-z0-9_-]{1,100}$/.test(task.task_key)) {
|
|
1890
|
+
throw new Error(`Invalid task key '${task.task_key}'`);
|
|
1891
|
+
}
|
|
1892
|
+
if (keys.has(task.task_key)) throw new Error(`Duplicate task key '${task.task_key}'`);
|
|
1893
|
+
keys.add(task.task_key);
|
|
1894
|
+
}
|
|
1895
|
+
const edges = /* @__PURE__ */ new Map();
|
|
1896
|
+
for (const task of tasks) {
|
|
1897
|
+
const dependencies = (task.depends_on ?? []).map((dependency) => dependency.task_key);
|
|
1898
|
+
for (const dependency of dependencies) {
|
|
1899
|
+
if (!keys.has(dependency)) {
|
|
1900
|
+
throw new Error(`Task '${task.task_key}' depends on unknown task '${dependency}'`);
|
|
1901
|
+
}
|
|
1902
|
+
if (dependency === task.task_key) {
|
|
1903
|
+
throw new Error(`Task '${task.task_key}' cannot depend on itself`);
|
|
1904
|
+
}
|
|
1905
|
+
}
|
|
1906
|
+
edges.set(task.task_key, dependencies);
|
|
1907
|
+
}
|
|
1908
|
+
const visiting = /* @__PURE__ */ new Set();
|
|
1909
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1910
|
+
const visit = (key) => {
|
|
1911
|
+
if (visiting.has(key)) throw new Error(`Jobs task graph contains a cycle at '${key}'`);
|
|
1912
|
+
if (visited.has(key)) return;
|
|
1913
|
+
visiting.add(key);
|
|
1914
|
+
for (const dependency of edges.get(key) ?? []) visit(dependency);
|
|
1915
|
+
visiting.delete(key);
|
|
1916
|
+
visited.add(key);
|
|
1917
|
+
};
|
|
1918
|
+
for (const key of keys) visit(key);
|
|
1919
|
+
}
|
|
1920
|
+
function jobsOrchestrationCheck() {
|
|
1921
|
+
return {
|
|
1922
|
+
id: "jobs-orchestration",
|
|
1923
|
+
description: "Lakeflow Jobs multi-task orchestration reaches expected task outcomes",
|
|
1924
|
+
configured: (env) => Boolean(env.DBX_TEST_JOBS_ORCHESTRATION_JSON),
|
|
1925
|
+
async run({ env, client }) {
|
|
1926
|
+
const configured = env.DBX_TEST_JOBS_ORCHESTRATION_JSON;
|
|
1927
|
+
if (!configured) throw new Error("DBX_TEST_JOBS_ORCHESTRATION_JSON is required");
|
|
1928
|
+
const spec = parseJobOrchestrationLiveSpec(configured);
|
|
1929
|
+
const adapter = new DatabricksJobsAdapter(client);
|
|
1930
|
+
const runId = await adapter.submit(spec.request);
|
|
1931
|
+
let run = await adapter.wait(runId, {
|
|
1932
|
+
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
1933
|
+
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
1934
|
+
});
|
|
1935
|
+
if (spec.repairFailedTasks && run.resultState !== "SUCCESS") {
|
|
1936
|
+
const failed = run.tasks.filter((task) => task.resultState === "FAILED").map((task) => task.taskKey);
|
|
1937
|
+
await adapter.repair(runId, { rerunTasks: failed.length > 0 ? failed : void 0 });
|
|
1938
|
+
run = await adapter.wait(runId, {
|
|
1939
|
+
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
1940
|
+
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
1941
|
+
});
|
|
1942
|
+
}
|
|
1943
|
+
const expectedResult = spec.expectedResult ?? "SUCCESS";
|
|
1944
|
+
if (run.resultState !== expectedResult) {
|
|
1945
|
+
throw new Error(
|
|
1946
|
+
`Job orchestration run ${runId} ended in ${run.resultState}, expected ${expectedResult}`
|
|
1947
|
+
);
|
|
1948
|
+
}
|
|
1949
|
+
for (const [taskKey, expected] of Object.entries(spec.expectedTasks ?? {})) {
|
|
1950
|
+
const task = run.tasks.find((candidate) => candidate.taskKey === taskKey);
|
|
1951
|
+
if (!task) throw new Error(`Job orchestration result did not include task '${taskKey}'`);
|
|
1952
|
+
if (task.resultState !== expected) {
|
|
1953
|
+
throw new Error(`Task '${taskKey}' ended in ${task.resultState}, expected ${expected}`);
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
return {
|
|
1957
|
+
runId,
|
|
1958
|
+
resultState: run.resultState,
|
|
1959
|
+
tasks: run.tasks
|
|
1960
|
+
};
|
|
1961
|
+
}
|
|
1962
|
+
};
|
|
1963
|
+
}
|
|
1964
|
+
function parseJobOrchestrationLiveSpec(value) {
|
|
1965
|
+
let parsed;
|
|
1966
|
+
try {
|
|
1967
|
+
parsed = JSON.parse(value);
|
|
1968
|
+
} catch (error) {
|
|
1969
|
+
throw new Error(`DBX_TEST_JOBS_ORCHESTRATION_JSON is invalid JSON: ${String(error)}`);
|
|
1970
|
+
}
|
|
1971
|
+
if (!parsed || typeof parsed !== "object") {
|
|
1972
|
+
throw new Error("Jobs orchestration spec must be an object");
|
|
1973
|
+
}
|
|
1974
|
+
const spec = parsed;
|
|
1975
|
+
if (!spec.request || typeof spec.request !== "object" || !Array.isArray(spec.request.tasks)) {
|
|
1976
|
+
throw new Error("Jobs orchestration spec requires request.tasks");
|
|
1977
|
+
}
|
|
1978
|
+
validateJobGraph(spec.request.tasks);
|
|
1979
|
+
return spec;
|
|
1980
|
+
}
|
|
1981
|
+
function normalizeJobRun(fallbackRunId, raw) {
|
|
1982
|
+
const state = objectOf(raw.state);
|
|
1983
|
+
const tasks = Array.isArray(raw.tasks) ? raw.tasks.filter(
|
|
1984
|
+
(task) => Boolean(task && typeof task === "object")
|
|
1985
|
+
).map((task) => {
|
|
1986
|
+
const taskState = objectOf(task.state);
|
|
1987
|
+
return {
|
|
1988
|
+
taskKey: String(task.task_key ?? "unknown"),
|
|
1989
|
+
...Number.isFinite(Number(task.run_id)) ? { runId: Number(task.run_id) } : {},
|
|
1990
|
+
lifeCycleState: String(taskState.life_cycle_state ?? "UNKNOWN"),
|
|
1991
|
+
resultState: String(taskState.result_state ?? "UNKNOWN"),
|
|
1992
|
+
...typeof taskState.state_message === "string" ? { stateMessage: taskState.state_message } : {}
|
|
1993
|
+
};
|
|
1994
|
+
}) : [];
|
|
1995
|
+
return {
|
|
1996
|
+
runId: Number(raw.run_id ?? fallbackRunId),
|
|
1997
|
+
lifeCycleState: String(state.life_cycle_state ?? "UNKNOWN"),
|
|
1998
|
+
resultState: String(state.result_state ?? "UNKNOWN"),
|
|
1999
|
+
tasks,
|
|
2000
|
+
raw
|
|
2001
|
+
};
|
|
2002
|
+
}
|
|
2003
|
+
function objectOf(value) {
|
|
2004
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
2005
|
+
}
|
|
2006
|
+
|
|
2007
|
+
// src/jobs-target-pack.ts
|
|
2008
|
+
function createLakeflowJobsTargetPack() {
|
|
2009
|
+
return defineTargetPack({
|
|
2010
|
+
id: "lakeflow-jobs",
|
|
2011
|
+
name: "Lakeflow Jobs orchestration",
|
|
2012
|
+
description: "Typed multi-task DAG, branch/loop, retry, cancellation, repair and task-outcome validation.",
|
|
2013
|
+
version: "0.1.0",
|
|
2014
|
+
maturity: "shipped",
|
|
2015
|
+
capabilities: [
|
|
2016
|
+
"jobs.multi-task-dag",
|
|
2017
|
+
"jobs.dependencies",
|
|
2018
|
+
"jobs.condition-task",
|
|
2019
|
+
"jobs.for-each-task",
|
|
2020
|
+
"jobs.retry",
|
|
2021
|
+
"jobs.cancel",
|
|
2022
|
+
"jobs.repair",
|
|
2023
|
+
"jobs.task-outcomes"
|
|
2024
|
+
],
|
|
2025
|
+
checks: () => [restrictedPrincipalCheck(), jobRunCheck(), jobsOrchestrationCheck()],
|
|
2026
|
+
requiredChecks: ["jobs-orchestration"],
|
|
2027
|
+
requirements: [
|
|
2028
|
+
{
|
|
2029
|
+
id: "workspace",
|
|
2030
|
+
description: "Databricks workspace host",
|
|
2031
|
+
anyOf: ["DATABRICKS_HOST"]
|
|
2032
|
+
},
|
|
2033
|
+
{
|
|
2034
|
+
id: "authentication",
|
|
2035
|
+
description: "Databricks OAuth, OIDC, bearer, or PAT authentication",
|
|
2036
|
+
anyOf: [
|
|
2037
|
+
"DATABRICKS_BEARER",
|
|
2038
|
+
"DATABRICKS_TOKEN",
|
|
2039
|
+
"DATABRICKS_CLIENT_ID",
|
|
2040
|
+
"DATABRICKS_OIDC_TOKEN",
|
|
2041
|
+
"DATABRICKS_OIDC_TOKEN_FILEPATH"
|
|
2042
|
+
],
|
|
2043
|
+
secret: true
|
|
2044
|
+
},
|
|
2045
|
+
{
|
|
2046
|
+
id: "orchestration-spec",
|
|
2047
|
+
description: "JSON multi-task orchestration scenario",
|
|
2048
|
+
anyOf: ["DBX_TEST_JOBS_ORCHESTRATION_JSON"]
|
|
2049
|
+
}
|
|
2050
|
+
],
|
|
2051
|
+
docsUrl: "https://experiments.fabric.pro/docs/testing/target-packs/#pack-1--lakeflow-jobs-orchestration",
|
|
2052
|
+
certificationScopes: []
|
|
2053
|
+
});
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
// src/builtin-packs.ts
|
|
2057
|
+
function createBuiltinTargetPacks() {
|
|
2058
|
+
return [...createFoundationTargetPacks(), createLakeflowJobsTargetPack()];
|
|
2059
|
+
}
|
|
2060
|
+
var builtinTargetPacks = createBuiltinTargetPacks();
|
|
2061
|
+
|
|
1492
2062
|
Object.defineProperty(exports, "waitForDatabricksRun", {
|
|
1493
2063
|
enumerable: true,
|
|
1494
2064
|
get: function () { return databricks.waitForDatabricksRun; }
|
|
@@ -1497,36 +2067,54 @@ Object.defineProperty(exports, "waitForDatabricksStatement", {
|
|
|
1497
2067
|
enumerable: true,
|
|
1498
2068
|
get: function () { return databricks.waitForDatabricksStatement; }
|
|
1499
2069
|
});
|
|
2070
|
+
exports.DatabricksJobsAdapter = DatabricksJobsAdapter;
|
|
1500
2071
|
exports.appHealthCheck = appHealthCheck;
|
|
1501
2072
|
exports.assertDeltaContract = assertDeltaContract;
|
|
1502
2073
|
exports.autoLoaderIngestionCheck = autoLoaderIngestionCheck;
|
|
1503
2074
|
exports.backupRestoreCheck = backupRestoreCheck;
|
|
2075
|
+
exports.builtinTargetPacks = builtinTargetPacks;
|
|
1504
2076
|
exports.classicComputeCheck = classicComputeCheck;
|
|
1505
2077
|
exports.compareToGolden = compareToGolden;
|
|
2078
|
+
exports.createAppsOperationalFoundationPack = createAppsOperationalFoundationPack;
|
|
2079
|
+
exports.createBuiltinTargetPacks = createBuiltinTargetPacks;
|
|
1506
2080
|
exports.createClientFromEnv = createClientFromEnv;
|
|
1507
2081
|
exports.createDuckDbContext = createDuckDbContext;
|
|
1508
2082
|
exports.createEphemeralLakebaseBranch = createEphemeralLakebaseBranch;
|
|
1509
2083
|
exports.createExecutionContext = createExecutionContext;
|
|
2084
|
+
exports.createFoundationTargetPacks = createFoundationTargetPacks;
|
|
2085
|
+
exports.createJobsCodeFoundationPack = createJobsCodeFoundationPack;
|
|
1510
2086
|
exports.createLakebaseTestProvider = createLakebaseTestProvider;
|
|
2087
|
+
exports.createLakeflowFoundationPack = createLakeflowFoundationPack;
|
|
2088
|
+
exports.createLakeflowJobsTargetPack = createLakeflowJobsTargetPack;
|
|
1511
2089
|
exports.createMockDatabricksClient = createMockDatabricksClient;
|
|
2090
|
+
exports.createSqlDeltaFoundationPack = createSqlDeltaFoundationPack;
|
|
2091
|
+
exports.createTargetPackRegistry = createTargetPackRegistry;
|
|
2092
|
+
exports.createUnityStorageFoundationPack = createUnityStorageFoundationPack;
|
|
1512
2093
|
exports.createWorkspaceContext = createWorkspaceContext;
|
|
1513
2094
|
exports.customLiveCheck = customLiveCheck;
|
|
1514
2095
|
exports.dbtBuildCheck = dbtBuildCheck;
|
|
1515
2096
|
exports.defineLiveSuite = defineLiveSuite;
|
|
2097
|
+
exports.defineTargetPack = defineTargetPack;
|
|
2098
|
+
exports.defineTargetPackRun = defineTargetPackRun;
|
|
1516
2099
|
exports.deleteEphemeralLakebaseBranch = deleteEphemeralLakebaseBranch;
|
|
1517
2100
|
exports.deleteVolumeFile = deleteVolumeFile;
|
|
1518
2101
|
exports.deltaContractCheck = deltaContractCheck;
|
|
2102
|
+
exports.doctorTargetPack = doctorTargetPack;
|
|
1519
2103
|
exports.ensureVolumeDirectory = ensureVolumeDirectory;
|
|
1520
2104
|
exports.expectQueryToMatchGolden = expectQueryToMatchGolden;
|
|
1521
2105
|
exports.expectTable = expectTable;
|
|
1522
2106
|
exports.failureRecoveryCheck = failureRecoveryCheck;
|
|
2107
|
+
exports.foundationTargetPacks = foundationTargetPacks;
|
|
1523
2108
|
exports.jobRunCheck = jobRunCheck;
|
|
2109
|
+
exports.jobsOrchestrationCheck = jobsOrchestrationCheck;
|
|
1524
2110
|
exports.lakebaseCredentialCheck = lakebaseCredentialCheck;
|
|
1525
2111
|
exports.lakebaseRoundTripCheck = lakebaseRoundTripCheck;
|
|
2112
|
+
exports.normalizeJobRun = normalizeJobRun;
|
|
1526
2113
|
exports.normalizeRow = normalizeRow;
|
|
1527
2114
|
exports.normalizeVolumeRoot = normalizeVolumeRoot;
|
|
1528
2115
|
exports.notebookSubmitCheck = notebookSubmitCheck;
|
|
1529
2116
|
exports.parseFixtureColumns = parseFixtureColumns;
|
|
2117
|
+
exports.parseJobOrchestrationLiveSpec = parseJobOrchestrationLiveSpec;
|
|
1530
2118
|
exports.parseStatementRows = parseStatementRows;
|
|
1531
2119
|
exports.performanceBudgetCheck = performanceBudgetCheck;
|
|
1532
2120
|
exports.pipelineRefreshCheck = pipelineRefreshCheck;
|
|
@@ -1538,6 +2126,7 @@ exports.resolveTokenProvider = resolveTokenProvider;
|
|
|
1538
2126
|
exports.restrictedPrincipalCheck = restrictedPrincipalCheck;
|
|
1539
2127
|
exports.rollbackCheck = rollbackCheck;
|
|
1540
2128
|
exports.runLiveSuite = runLiveSuite;
|
|
2129
|
+
exports.runTargetPack = runTargetPack;
|
|
1541
2130
|
exports.secretRotationCheck = secretRotationCheck;
|
|
1542
2131
|
exports.secretScopeCheck = secretScopeCheck;
|
|
1543
2132
|
exports.serverlessComputeCheck = serverlessComputeCheck;
|
|
@@ -1546,6 +2135,7 @@ exports.toLakebaseBranchId = toLakebaseBranchId;
|
|
|
1546
2135
|
exports.toNamedParameters = toNamedParameters;
|
|
1547
2136
|
exports.unityCatalogAccessCheck = unityCatalogAccessCheck;
|
|
1548
2137
|
exports.uploadVolumeFile = uploadVolumeFile;
|
|
2138
|
+
exports.validateJobGraph = validateJobGraph;
|
|
1549
2139
|
exports.volumeIOCheck = volumeIOCheck;
|
|
1550
2140
|
exports.waitForPipelineUpdate = waitForPipelineUpdate;
|
|
1551
2141
|
//# sourceMappingURL=index.cjs.map
|