@fabricorg/databricks-testkit 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/dist/index.cjs +545 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +239 -1
- package/dist/index.d.ts +239 -1
- package/dist/index.js +526 -2
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -77,6 +77,37 @@ configured and report `pass | fail | not-configured`; only IDs listed in
|
|
|
77
77
|
Auto Loader full refreshes require a distinct `DBX_TEST_AUTOLOADER_PIPELINE_ID`.
|
|
78
78
|
The check refuses to use `DBX_TEST_PIPELINE_ID`, protecting production telemetry.
|
|
79
79
|
|
|
80
|
+
## Typed target packs
|
|
81
|
+
|
|
82
|
+
Target packs bind a workload adapter, offline unit tests, live checks,
|
|
83
|
+
documentation, certification scope, and Quality Center metadata into one
|
|
84
|
+
versioned contract. Built-in packs cover SQL/Delta, Lakeflow ingestion,
|
|
85
|
+
Jobs/code, Unity Catalog storage, Apps operations, and Lakeflow Jobs
|
|
86
|
+
orchestration.
|
|
87
|
+
|
|
88
|
+
```ts
|
|
89
|
+
import {
|
|
90
|
+
builtinTargetPacks,
|
|
91
|
+
createTargetPackRegistry,
|
|
92
|
+
doctorTargetPack,
|
|
93
|
+
runTargetPack,
|
|
94
|
+
} from '@fabricorg/databricks-testkit';
|
|
95
|
+
|
|
96
|
+
const pack = createTargetPackRegistry(builtinTargetPacks).get('lakeflow-jobs');
|
|
97
|
+
if (!pack) throw new Error('lakeflow-jobs target pack is not installed');
|
|
98
|
+
const diagnosis = doctorTargetPack(pack, process.env);
|
|
99
|
+
if (!diagnosis.ready) throw new Error(diagnosis.missing.join(', '));
|
|
100
|
+
|
|
101
|
+
const evidence = await runTargetPack(pack, {
|
|
102
|
+
env: process.env,
|
|
103
|
+
live: true,
|
|
104
|
+
});
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
The Lakeflow Jobs adapter models notebook, wheel, JAR, Spark Python, submit,
|
|
108
|
+
pipeline, SQL, dbt, nested-job, condition, and for-each tasks, including graph
|
|
109
|
+
validation, cancellation, repair, and task-level terminal-state assertions.
|
|
110
|
+
|
|
80
111
|
Docs: `apps/docs/content/docs/testing/` · Plan:
|
|
81
112
|
`.weave/plans/databricks-testing.md` · BDD layer: `@fabricorg/databricks-bdd`
|
|
82
113
|
(Phase 2).
|
package/dist/index.cjs
CHANGED
|
@@ -654,7 +654,8 @@ async function runLiveSuite(spec, runtime = {}) {
|
|
|
654
654
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
655
655
|
success,
|
|
656
656
|
required: required2,
|
|
657
|
-
results
|
|
657
|
+
results,
|
|
658
|
+
...spec.targetPack ? { targetPack: spec.targetPack } : {}
|
|
658
659
|
};
|
|
659
660
|
if (spec.evidencePath)
|
|
660
661
|
await writeReport(spec.evidencePath, `${JSON.stringify(evidence, null, 2)}
|
|
@@ -1535,6 +1536,529 @@ function classicComputeCheck() {
|
|
|
1535
1536
|
};
|
|
1536
1537
|
}
|
|
1537
1538
|
|
|
1539
|
+
// src/target-packs.ts
|
|
1540
|
+
var TARGET_PACK_ID = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/;
|
|
1541
|
+
function defineTargetPack(spec) {
|
|
1542
|
+
if (!TARGET_PACK_ID.test(spec.id)) {
|
|
1543
|
+
throw new Error(`Target pack id '${spec.id}' must be lowercase kebab-case`);
|
|
1544
|
+
}
|
|
1545
|
+
if (!spec.name.trim()) throw new Error(`Target pack '${spec.id}' requires a name`);
|
|
1546
|
+
if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(spec.version)) {
|
|
1547
|
+
throw new Error(
|
|
1548
|
+
`Target pack '${spec.id}' version '${spec.version}' is not semantic versioning`
|
|
1549
|
+
);
|
|
1550
|
+
}
|
|
1551
|
+
if (spec.capabilities.length === 0) {
|
|
1552
|
+
throw new Error(`Target pack '${spec.id}' must declare at least one capability`);
|
|
1553
|
+
}
|
|
1554
|
+
const createChecks = typeof spec.checks === "function" ? spec.checks : () => spec.checks;
|
|
1555
|
+
const validateChecks = () => {
|
|
1556
|
+
const checks = [...createChecks()];
|
|
1557
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1558
|
+
for (const check of checks) {
|
|
1559
|
+
if (!check.id.trim())
|
|
1560
|
+
throw new Error(`Target pack '${spec.id}' contains a check without an id`);
|
|
1561
|
+
if (ids.has(check.id)) {
|
|
1562
|
+
throw new Error(`Target pack '${spec.id}' contains duplicate check '${check.id}'`);
|
|
1563
|
+
}
|
|
1564
|
+
ids.add(check.id);
|
|
1565
|
+
}
|
|
1566
|
+
for (const required2 of spec.requiredChecks ?? []) {
|
|
1567
|
+
if (!ids.has(required2)) {
|
|
1568
|
+
throw new Error(`Target pack '${spec.id}' requires unknown check '${required2}'`);
|
|
1569
|
+
}
|
|
1570
|
+
}
|
|
1571
|
+
return checks;
|
|
1572
|
+
};
|
|
1573
|
+
validateChecks();
|
|
1574
|
+
return Object.freeze({
|
|
1575
|
+
...spec,
|
|
1576
|
+
capabilities: Object.freeze([...spec.capabilities]),
|
|
1577
|
+
requiredChecks: Object.freeze([...spec.requiredChecks ?? []]),
|
|
1578
|
+
requirements: Object.freeze([...spec.requirements ?? []]),
|
|
1579
|
+
certificationScopes: Object.freeze([...spec.certificationScopes ?? []]),
|
|
1580
|
+
checks: validateChecks
|
|
1581
|
+
});
|
|
1582
|
+
}
|
|
1583
|
+
function createTargetPackRegistry(packs) {
|
|
1584
|
+
const registry = /* @__PURE__ */ new Map();
|
|
1585
|
+
for (const pack of packs) {
|
|
1586
|
+
if (registry.has(pack.id)) throw new Error(`Duplicate target pack '${pack.id}'`);
|
|
1587
|
+
registry.set(pack.id, pack);
|
|
1588
|
+
}
|
|
1589
|
+
return registry;
|
|
1590
|
+
}
|
|
1591
|
+
function doctorTargetPack(pack, env = process.env, requiredChecks = pack.requiredChecks ?? []) {
|
|
1592
|
+
const requirements = (pack.requirements ?? []).map((requirement) => ({
|
|
1593
|
+
id: requirement.id,
|
|
1594
|
+
description: requirement.description,
|
|
1595
|
+
satisfied: requirement.anyOf.some((name) => Boolean(env[name]?.trim())),
|
|
1596
|
+
required: requirement.required !== false,
|
|
1597
|
+
variables: [...requirement.anyOf]
|
|
1598
|
+
}));
|
|
1599
|
+
const required2 = new Set(requiredChecks);
|
|
1600
|
+
const checks = pack.checks().map((check) => ({
|
|
1601
|
+
id: check.id,
|
|
1602
|
+
description: check.description,
|
|
1603
|
+
configured: check.configured ? check.configured(env) : true,
|
|
1604
|
+
required: required2.has(check.id)
|
|
1605
|
+
}));
|
|
1606
|
+
const known = new Set(checks.map((check) => check.id));
|
|
1607
|
+
const missingRequirements = requirements.filter((requirement) => requirement.required && !requirement.satisfied).map((requirement) => requirement.id);
|
|
1608
|
+
const missingRequiredChecks = [
|
|
1609
|
+
...checks.filter((check) => check.required && !check.configured).map((check) => check.id),
|
|
1610
|
+
...[...required2].filter((id) => !known.has(id))
|
|
1611
|
+
];
|
|
1612
|
+
return {
|
|
1613
|
+
pack: {
|
|
1614
|
+
id: pack.id,
|
|
1615
|
+
name: pack.name,
|
|
1616
|
+
version: pack.version,
|
|
1617
|
+
maturity: pack.maturity
|
|
1618
|
+
},
|
|
1619
|
+
ready: missingRequirements.length === 0 && missingRequiredChecks.length === 0,
|
|
1620
|
+
requirements,
|
|
1621
|
+
checks,
|
|
1622
|
+
missingRequirements,
|
|
1623
|
+
missingRequiredChecks
|
|
1624
|
+
};
|
|
1625
|
+
}
|
|
1626
|
+
function defineTargetPackRun(pack, options = {}) {
|
|
1627
|
+
const env = options.env ?? process.env;
|
|
1628
|
+
const requiredChecks = [...options.requiredChecks ?? pack.requiredChecks ?? []];
|
|
1629
|
+
const doctor = doctorTargetPack(pack, env, requiredChecks);
|
|
1630
|
+
if (!doctor.ready) {
|
|
1631
|
+
const missing = [...doctor.missingRequirements, ...doctor.missingRequiredChecks].join(", ");
|
|
1632
|
+
throw new Error(`Target pack '${pack.id}' is not configured: ${missing}`);
|
|
1633
|
+
}
|
|
1634
|
+
const targetPack = evidenceMetadata(pack, env);
|
|
1635
|
+
const suite = defineLiveSuite({
|
|
1636
|
+
checks: pack.checks(),
|
|
1637
|
+
required: requiredChecks,
|
|
1638
|
+
env,
|
|
1639
|
+
evidencePath: options.evidencePath,
|
|
1640
|
+
junitPath: options.junitPath,
|
|
1641
|
+
targetPack
|
|
1642
|
+
});
|
|
1643
|
+
return { pack, run: (runtime) => suite.run(runtime) };
|
|
1644
|
+
}
|
|
1645
|
+
async function runTargetPack(pack, options = {}, runtime) {
|
|
1646
|
+
return defineTargetPackRun(pack, options).run(runtime);
|
|
1647
|
+
}
|
|
1648
|
+
function evidenceMetadata(pack, env) {
|
|
1649
|
+
return {
|
|
1650
|
+
id: pack.id,
|
|
1651
|
+
name: pack.name,
|
|
1652
|
+
version: pack.version,
|
|
1653
|
+
maturity: pack.maturity,
|
|
1654
|
+
capabilities: [...pack.capabilities],
|
|
1655
|
+
...pack.docsUrl ? { docsUrl: pack.docsUrl } : {},
|
|
1656
|
+
certificationScopes: [...pack.certificationScopes ?? []],
|
|
1657
|
+
target: compact({
|
|
1658
|
+
cloud: env.DBX_TEST_CLOUD,
|
|
1659
|
+
region: env.DBX_TEST_REGION,
|
|
1660
|
+
workspace: env.DATABRICKS_HOST,
|
|
1661
|
+
runtime: env.DBX_TEST_RUNTIME_VERSION,
|
|
1662
|
+
compute: env.DBX_TEST_COMPUTE
|
|
1663
|
+
})
|
|
1664
|
+
};
|
|
1665
|
+
}
|
|
1666
|
+
function compact(value) {
|
|
1667
|
+
return Object.fromEntries(
|
|
1668
|
+
Object.entries(value).filter((entry) => Boolean(entry[1]))
|
|
1669
|
+
);
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
// src/foundation-packs.ts
|
|
1673
|
+
var DOCS = "https://experiments.fabric.pro/docs/testing/target-packs/";
|
|
1674
|
+
var AZURE_SCOPE = "Azure Databricks live-gated; see certification evidence for exact runtime";
|
|
1675
|
+
var workspaceRequirement = {
|
|
1676
|
+
id: "workspace",
|
|
1677
|
+
description: "Databricks workspace host",
|
|
1678
|
+
anyOf: ["DATABRICKS_HOST"]
|
|
1679
|
+
};
|
|
1680
|
+
var authRequirement = {
|
|
1681
|
+
id: "authentication",
|
|
1682
|
+
description: "Databricks OAuth, OIDC, bearer, or PAT authentication",
|
|
1683
|
+
anyOf: [
|
|
1684
|
+
"DATABRICKS_BEARER",
|
|
1685
|
+
"DATABRICKS_TOKEN",
|
|
1686
|
+
"DATABRICKS_CLIENT_ID",
|
|
1687
|
+
"DATABRICKS_OIDC_TOKEN",
|
|
1688
|
+
"DATABRICKS_OIDC_TOKEN_FILEPATH"
|
|
1689
|
+
],
|
|
1690
|
+
secret: true
|
|
1691
|
+
};
|
|
1692
|
+
var warehouseRequirement = {
|
|
1693
|
+
id: "warehouse",
|
|
1694
|
+
description: "SQL warehouse ID or HTTP path",
|
|
1695
|
+
anyOf: ["DATABRICKS_WAREHOUSE_ID", "DATABRICKS_HTTP_PATH"]
|
|
1696
|
+
};
|
|
1697
|
+
function createSqlDeltaFoundationPack() {
|
|
1698
|
+
return defineTargetPack({
|
|
1699
|
+
id: "sql-delta",
|
|
1700
|
+
name: "SQL and Delta foundation",
|
|
1701
|
+
description: "SQL Statement Execution, Delta contracts, performance, recovery, backup and rollback.",
|
|
1702
|
+
version: "1.0.0",
|
|
1703
|
+
maturity: "live-gated",
|
|
1704
|
+
capabilities: [
|
|
1705
|
+
"sql.statement-execution",
|
|
1706
|
+
"delta.contract",
|
|
1707
|
+
"delta.time-travel",
|
|
1708
|
+
"performance.sql",
|
|
1709
|
+
"recovery.sql",
|
|
1710
|
+
"backup-restore.delta",
|
|
1711
|
+
"rollback.delta"
|
|
1712
|
+
],
|
|
1713
|
+
checks: () => [
|
|
1714
|
+
restrictedPrincipalCheck(),
|
|
1715
|
+
sqlRoundTripCheck(),
|
|
1716
|
+
deltaContractCheck(),
|
|
1717
|
+
serverlessComputeCheck(),
|
|
1718
|
+
performanceBudgetCheck(),
|
|
1719
|
+
failureRecoveryCheck(),
|
|
1720
|
+
backupRestoreCheck(),
|
|
1721
|
+
rollbackCheck()
|
|
1722
|
+
],
|
|
1723
|
+
requiredChecks: ["sql"],
|
|
1724
|
+
requirements: [workspaceRequirement, authRequirement, warehouseRequirement],
|
|
1725
|
+
docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
|
|
1726
|
+
certificationScopes: [AZURE_SCOPE]
|
|
1727
|
+
});
|
|
1728
|
+
}
|
|
1729
|
+
function createLakeflowFoundationPack() {
|
|
1730
|
+
return defineTargetPack({
|
|
1731
|
+
id: "lakeflow-ingestion",
|
|
1732
|
+
name: "Lakeflow ingestion foundation",
|
|
1733
|
+
description: "Lakeflow refresh, Volumes, Auto Loader ingestion and rescued-row evidence.",
|
|
1734
|
+
version: "1.0.0",
|
|
1735
|
+
maturity: "live-gated",
|
|
1736
|
+
capabilities: [
|
|
1737
|
+
"lakeflow.refresh",
|
|
1738
|
+
"autoloader.ingestion",
|
|
1739
|
+
"volumes.files",
|
|
1740
|
+
"delta.rescued-data"
|
|
1741
|
+
],
|
|
1742
|
+
checks: () => [
|
|
1743
|
+
restrictedPrincipalCheck(),
|
|
1744
|
+
pipelineRefreshCheck(),
|
|
1745
|
+
volumeIOCheck(),
|
|
1746
|
+
autoLoaderIngestionCheck()
|
|
1747
|
+
],
|
|
1748
|
+
requiredChecks: ["pipeline"],
|
|
1749
|
+
requirements: [workspaceRequirement, authRequirement],
|
|
1750
|
+
docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
|
|
1751
|
+
certificationScopes: [AZURE_SCOPE]
|
|
1752
|
+
});
|
|
1753
|
+
}
|
|
1754
|
+
function createJobsCodeFoundationPack() {
|
|
1755
|
+
return defineTargetPack({
|
|
1756
|
+
id: "jobs-code",
|
|
1757
|
+
name: "Jobs and code foundation",
|
|
1758
|
+
description: "Jobs, notebooks, dbt, serverless compute and classic compute probes.",
|
|
1759
|
+
version: "1.0.0",
|
|
1760
|
+
maturity: "live-gated",
|
|
1761
|
+
capabilities: [
|
|
1762
|
+
"jobs.run-now",
|
|
1763
|
+
"jobs.submit.notebook",
|
|
1764
|
+
"jobs.submit.dbt",
|
|
1765
|
+
"compute.serverless",
|
|
1766
|
+
"compute.classic"
|
|
1767
|
+
],
|
|
1768
|
+
checks: () => [
|
|
1769
|
+
restrictedPrincipalCheck(),
|
|
1770
|
+
jobRunCheck(),
|
|
1771
|
+
notebookSubmitCheck(),
|
|
1772
|
+
dbtBuildCheck(),
|
|
1773
|
+
serverlessComputeCheck(),
|
|
1774
|
+
classicComputeCheck()
|
|
1775
|
+
],
|
|
1776
|
+
requiredChecks: ["jobs"],
|
|
1777
|
+
requirements: [workspaceRequirement, authRequirement],
|
|
1778
|
+
docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
|
|
1779
|
+
certificationScopes: [AZURE_SCOPE]
|
|
1780
|
+
});
|
|
1781
|
+
}
|
|
1782
|
+
function createUnityStorageFoundationPack() {
|
|
1783
|
+
return defineTargetPack({
|
|
1784
|
+
id: "unity-storage",
|
|
1785
|
+
name: "Unity Catalog and storage foundation",
|
|
1786
|
+
description: "Unity Catalog allow/deny, Volumes, secret metadata and secret rotation.",
|
|
1787
|
+
version: "1.0.0",
|
|
1788
|
+
maturity: "live-gated",
|
|
1789
|
+
capabilities: [
|
|
1790
|
+
"unity-catalog.permissions",
|
|
1791
|
+
"volumes.files",
|
|
1792
|
+
"secrets.metadata",
|
|
1793
|
+
"secrets.rotation"
|
|
1794
|
+
],
|
|
1795
|
+
checks: () => [
|
|
1796
|
+
restrictedPrincipalCheck(),
|
|
1797
|
+
unityCatalogAccessCheck(),
|
|
1798
|
+
volumeIOCheck(),
|
|
1799
|
+
secretScopeCheck(),
|
|
1800
|
+
secretRotationCheck()
|
|
1801
|
+
],
|
|
1802
|
+
requiredChecks: ["uc-grants"],
|
|
1803
|
+
requirements: [workspaceRequirement, authRequirement, warehouseRequirement],
|
|
1804
|
+
docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
|
|
1805
|
+
certificationScopes: [AZURE_SCOPE]
|
|
1806
|
+
});
|
|
1807
|
+
}
|
|
1808
|
+
function createAppsOperationalFoundationPack() {
|
|
1809
|
+
return defineTargetPack({
|
|
1810
|
+
id: "apps-operational",
|
|
1811
|
+
name: "Apps and operational data foundation",
|
|
1812
|
+
description: "Databricks Apps resources and health plus Lakebase OAuth and pool refresh.",
|
|
1813
|
+
version: "1.0.0",
|
|
1814
|
+
maturity: "live-gated",
|
|
1815
|
+
capabilities: [
|
|
1816
|
+
"apps.health",
|
|
1817
|
+
"apps.resources",
|
|
1818
|
+
"lakebase.credentials",
|
|
1819
|
+
"lakebase.round-trip",
|
|
1820
|
+
"lakebase.pool-refresh"
|
|
1821
|
+
],
|
|
1822
|
+
checks: () => [
|
|
1823
|
+
restrictedPrincipalCheck(),
|
|
1824
|
+
appHealthCheck(),
|
|
1825
|
+
lakebaseCredentialCheck(),
|
|
1826
|
+
lakebaseRoundTripCheck()
|
|
1827
|
+
],
|
|
1828
|
+
requiredChecks: ["app"],
|
|
1829
|
+
requirements: [workspaceRequirement, authRequirement],
|
|
1830
|
+
docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
|
|
1831
|
+
certificationScopes: [AZURE_SCOPE]
|
|
1832
|
+
});
|
|
1833
|
+
}
|
|
1834
|
+
function createFoundationTargetPacks() {
|
|
1835
|
+
return [
|
|
1836
|
+
createSqlDeltaFoundationPack(),
|
|
1837
|
+
createLakeflowFoundationPack(),
|
|
1838
|
+
createJobsCodeFoundationPack(),
|
|
1839
|
+
createUnityStorageFoundationPack(),
|
|
1840
|
+
createAppsOperationalFoundationPack()
|
|
1841
|
+
];
|
|
1842
|
+
}
|
|
1843
|
+
var foundationTargetPacks = createFoundationTargetPacks();
|
|
1844
|
+
var DatabricksJobsAdapter = class {
|
|
1845
|
+
constructor(client) {
|
|
1846
|
+
this.client = client;
|
|
1847
|
+
}
|
|
1848
|
+
client;
|
|
1849
|
+
async submit(request) {
|
|
1850
|
+
validateJobGraph(request.tasks);
|
|
1851
|
+
const response = await this.client.post(
|
|
1852
|
+
"/api/2.1/jobs/runs/submit",
|
|
1853
|
+
request
|
|
1854
|
+
);
|
|
1855
|
+
if (!response.run_id) throw new Error("Databricks Jobs submit returned no run_id");
|
|
1856
|
+
return response.run_id;
|
|
1857
|
+
}
|
|
1858
|
+
async get(runId) {
|
|
1859
|
+
const raw = await this.client.get("/api/2.1/jobs/runs/get", {
|
|
1860
|
+
run_id: runId,
|
|
1861
|
+
include_history: true
|
|
1862
|
+
});
|
|
1863
|
+
return normalizeJobRun(runId, raw);
|
|
1864
|
+
}
|
|
1865
|
+
async wait(runId, options = {}) {
|
|
1866
|
+
const raw = await databricks.waitForDatabricksRun(this.client, runId, {
|
|
1867
|
+
timeoutMs: options.timeoutMs,
|
|
1868
|
+
pollIntervalMs: options.pollIntervalMs
|
|
1869
|
+
});
|
|
1870
|
+
return normalizeJobRun(runId, raw);
|
|
1871
|
+
}
|
|
1872
|
+
async cancel(runId) {
|
|
1873
|
+
await this.client.post("/api/2.1/jobs/runs/cancel", { run_id: runId });
|
|
1874
|
+
}
|
|
1875
|
+
async repair(runId, options = {}) {
|
|
1876
|
+
const response = await this.client.post("/api/2.1/jobs/runs/repair", {
|
|
1877
|
+
run_id: runId,
|
|
1878
|
+
...options.latestRepairId ? { latest_repair_id: options.latestRepairId } : {},
|
|
1879
|
+
...options.rerunTasks?.length ? { rerun_tasks: options.rerunTasks } : { rerun_all_failed_tasks: true }
|
|
1880
|
+
});
|
|
1881
|
+
if (!response.repair_id) throw new Error("Databricks Jobs repair returned no repair_id");
|
|
1882
|
+
return response.repair_id;
|
|
1883
|
+
}
|
|
1884
|
+
};
|
|
1885
|
+
function validateJobGraph(tasks) {
|
|
1886
|
+
if (tasks.length === 0) throw new Error("Jobs orchestration requires at least one task");
|
|
1887
|
+
const keys = /* @__PURE__ */ new Set();
|
|
1888
|
+
for (const task of tasks) {
|
|
1889
|
+
if (!/^[A-Za-z0-9_-]{1,100}$/.test(task.task_key)) {
|
|
1890
|
+
throw new Error(`Invalid task key '${task.task_key}'`);
|
|
1891
|
+
}
|
|
1892
|
+
if (keys.has(task.task_key)) throw new Error(`Duplicate task key '${task.task_key}'`);
|
|
1893
|
+
keys.add(task.task_key);
|
|
1894
|
+
}
|
|
1895
|
+
const edges = /* @__PURE__ */ new Map();
|
|
1896
|
+
for (const task of tasks) {
|
|
1897
|
+
const dependencies = (task.depends_on ?? []).map((dependency) => dependency.task_key);
|
|
1898
|
+
for (const dependency of dependencies) {
|
|
1899
|
+
if (!keys.has(dependency)) {
|
|
1900
|
+
throw new Error(`Task '${task.task_key}' depends on unknown task '${dependency}'`);
|
|
1901
|
+
}
|
|
1902
|
+
if (dependency === task.task_key) {
|
|
1903
|
+
throw new Error(`Task '${task.task_key}' cannot depend on itself`);
|
|
1904
|
+
}
|
|
1905
|
+
}
|
|
1906
|
+
edges.set(task.task_key, dependencies);
|
|
1907
|
+
}
|
|
1908
|
+
const visiting = /* @__PURE__ */ new Set();
|
|
1909
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1910
|
+
const visit = (key) => {
|
|
1911
|
+
if (visiting.has(key)) throw new Error(`Jobs task graph contains a cycle at '${key}'`);
|
|
1912
|
+
if (visited.has(key)) return;
|
|
1913
|
+
visiting.add(key);
|
|
1914
|
+
for (const dependency of edges.get(key) ?? []) visit(dependency);
|
|
1915
|
+
visiting.delete(key);
|
|
1916
|
+
visited.add(key);
|
|
1917
|
+
};
|
|
1918
|
+
for (const key of keys) visit(key);
|
|
1919
|
+
}
|
|
1920
|
+
function jobsOrchestrationCheck() {
|
|
1921
|
+
return {
|
|
1922
|
+
id: "jobs-orchestration",
|
|
1923
|
+
description: "Lakeflow Jobs multi-task orchestration reaches expected task outcomes",
|
|
1924
|
+
configured: (env) => Boolean(env.DBX_TEST_JOBS_ORCHESTRATION_JSON),
|
|
1925
|
+
async run({ env, client }) {
|
|
1926
|
+
const configured = env.DBX_TEST_JOBS_ORCHESTRATION_JSON;
|
|
1927
|
+
if (!configured) throw new Error("DBX_TEST_JOBS_ORCHESTRATION_JSON is required");
|
|
1928
|
+
const spec = parseJobOrchestrationLiveSpec(configured);
|
|
1929
|
+
const adapter = new DatabricksJobsAdapter(client);
|
|
1930
|
+
const runId = await adapter.submit(spec.request);
|
|
1931
|
+
let run = await adapter.wait(runId, {
|
|
1932
|
+
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
1933
|
+
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
1934
|
+
});
|
|
1935
|
+
if (spec.repairFailedTasks && run.resultState !== "SUCCESS") {
|
|
1936
|
+
const failed = run.tasks.filter((task) => task.resultState === "FAILED").map((task) => task.taskKey);
|
|
1937
|
+
await adapter.repair(runId, { rerunTasks: failed.length > 0 ? failed : void 0 });
|
|
1938
|
+
run = await adapter.wait(runId, {
|
|
1939
|
+
timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
1940
|
+
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
1941
|
+
});
|
|
1942
|
+
}
|
|
1943
|
+
const expectedResult = spec.expectedResult ?? "SUCCESS";
|
|
1944
|
+
if (run.resultState !== expectedResult) {
|
|
1945
|
+
throw new Error(
|
|
1946
|
+
`Job orchestration run ${runId} ended in ${run.resultState}, expected ${expectedResult}`
|
|
1947
|
+
);
|
|
1948
|
+
}
|
|
1949
|
+
for (const [taskKey, expected] of Object.entries(spec.expectedTasks ?? {})) {
|
|
1950
|
+
const task = run.tasks.find((candidate) => candidate.taskKey === taskKey);
|
|
1951
|
+
if (!task) throw new Error(`Job orchestration result did not include task '${taskKey}'`);
|
|
1952
|
+
if (task.resultState !== expected) {
|
|
1953
|
+
throw new Error(`Task '${taskKey}' ended in ${task.resultState}, expected ${expected}`);
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
return {
|
|
1957
|
+
runId,
|
|
1958
|
+
resultState: run.resultState,
|
|
1959
|
+
tasks: run.tasks
|
|
1960
|
+
};
|
|
1961
|
+
}
|
|
1962
|
+
};
|
|
1963
|
+
}
|
|
1964
|
+
function parseJobOrchestrationLiveSpec(value) {
|
|
1965
|
+
let parsed;
|
|
1966
|
+
try {
|
|
1967
|
+
parsed = JSON.parse(value);
|
|
1968
|
+
} catch (error) {
|
|
1969
|
+
throw new Error(`DBX_TEST_JOBS_ORCHESTRATION_JSON is invalid JSON: ${String(error)}`);
|
|
1970
|
+
}
|
|
1971
|
+
if (!parsed || typeof parsed !== "object") {
|
|
1972
|
+
throw new Error("Jobs orchestration spec must be an object");
|
|
1973
|
+
}
|
|
1974
|
+
const spec = parsed;
|
|
1975
|
+
if (!spec.request || typeof spec.request !== "object" || !Array.isArray(spec.request.tasks)) {
|
|
1976
|
+
throw new Error("Jobs orchestration spec requires request.tasks");
|
|
1977
|
+
}
|
|
1978
|
+
validateJobGraph(spec.request.tasks);
|
|
1979
|
+
return spec;
|
|
1980
|
+
}
|
|
1981
|
+
function normalizeJobRun(fallbackRunId, raw) {
|
|
1982
|
+
const state = objectOf(raw.state);
|
|
1983
|
+
const tasks = Array.isArray(raw.tasks) ? raw.tasks.filter(
|
|
1984
|
+
(task) => Boolean(task && typeof task === "object")
|
|
1985
|
+
).map((task) => {
|
|
1986
|
+
const taskState = objectOf(task.state);
|
|
1987
|
+
return {
|
|
1988
|
+
taskKey: String(task.task_key ?? "unknown"),
|
|
1989
|
+
...Number.isFinite(Number(task.run_id)) ? { runId: Number(task.run_id) } : {},
|
|
1990
|
+
lifeCycleState: String(taskState.life_cycle_state ?? "UNKNOWN"),
|
|
1991
|
+
resultState: String(taskState.result_state ?? "UNKNOWN"),
|
|
1992
|
+
...typeof taskState.state_message === "string" ? { stateMessage: taskState.state_message } : {}
|
|
1993
|
+
};
|
|
1994
|
+
}) : [];
|
|
1995
|
+
return {
|
|
1996
|
+
runId: Number(raw.run_id ?? fallbackRunId),
|
|
1997
|
+
lifeCycleState: String(state.life_cycle_state ?? "UNKNOWN"),
|
|
1998
|
+
resultState: String(state.result_state ?? "UNKNOWN"),
|
|
1999
|
+
tasks,
|
|
2000
|
+
raw
|
|
2001
|
+
};
|
|
2002
|
+
}
|
|
2003
|
+
function objectOf(value) {
|
|
2004
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
2005
|
+
}
|
|
2006
|
+
|
|
2007
|
+
// src/jobs-target-pack.ts
|
|
2008
|
+
function createLakeflowJobsTargetPack() {
|
|
2009
|
+
return defineTargetPack({
|
|
2010
|
+
id: "lakeflow-jobs",
|
|
2011
|
+
name: "Lakeflow Jobs orchestration",
|
|
2012
|
+
description: "Typed multi-task DAG, branch/loop, retry, cancellation, repair and task-outcome validation.",
|
|
2013
|
+
version: "0.1.0",
|
|
2014
|
+
maturity: "shipped",
|
|
2015
|
+
capabilities: [
|
|
2016
|
+
"jobs.multi-task-dag",
|
|
2017
|
+
"jobs.dependencies",
|
|
2018
|
+
"jobs.condition-task",
|
|
2019
|
+
"jobs.for-each-task",
|
|
2020
|
+
"jobs.retry",
|
|
2021
|
+
"jobs.cancel",
|
|
2022
|
+
"jobs.repair",
|
|
2023
|
+
"jobs.task-outcomes"
|
|
2024
|
+
],
|
|
2025
|
+
checks: () => [restrictedPrincipalCheck(), jobRunCheck(), jobsOrchestrationCheck()],
|
|
2026
|
+
requiredChecks: ["jobs-orchestration"],
|
|
2027
|
+
requirements: [
|
|
2028
|
+
{
|
|
2029
|
+
id: "workspace",
|
|
2030
|
+
description: "Databricks workspace host",
|
|
2031
|
+
anyOf: ["DATABRICKS_HOST"]
|
|
2032
|
+
},
|
|
2033
|
+
{
|
|
2034
|
+
id: "authentication",
|
|
2035
|
+
description: "Databricks OAuth, OIDC, bearer, or PAT authentication",
|
|
2036
|
+
anyOf: [
|
|
2037
|
+
"DATABRICKS_BEARER",
|
|
2038
|
+
"DATABRICKS_TOKEN",
|
|
2039
|
+
"DATABRICKS_CLIENT_ID",
|
|
2040
|
+
"DATABRICKS_OIDC_TOKEN",
|
|
2041
|
+
"DATABRICKS_OIDC_TOKEN_FILEPATH"
|
|
2042
|
+
],
|
|
2043
|
+
secret: true
|
|
2044
|
+
},
|
|
2045
|
+
{
|
|
2046
|
+
id: "orchestration-spec",
|
|
2047
|
+
description: "JSON multi-task orchestration scenario",
|
|
2048
|
+
anyOf: ["DBX_TEST_JOBS_ORCHESTRATION_JSON"]
|
|
2049
|
+
}
|
|
2050
|
+
],
|
|
2051
|
+
docsUrl: "https://experiments.fabric.pro/docs/testing/target-packs/#pack-1--lakeflow-jobs-orchestration",
|
|
2052
|
+
certificationScopes: []
|
|
2053
|
+
});
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
// src/builtin-packs.ts
|
|
2057
|
+
function createBuiltinTargetPacks() {
|
|
2058
|
+
return [...createFoundationTargetPacks(), createLakeflowJobsTargetPack()];
|
|
2059
|
+
}
|
|
2060
|
+
var builtinTargetPacks = createBuiltinTargetPacks();
|
|
2061
|
+
|
|
1538
2062
|
Object.defineProperty(exports, "waitForDatabricksRun", {
|
|
1539
2063
|
enumerable: true,
|
|
1540
2064
|
get: function () { return databricks.waitForDatabricksRun; }
|
|
@@ -1543,36 +2067,54 @@ Object.defineProperty(exports, "waitForDatabricksStatement", {
|
|
|
1543
2067
|
enumerable: true,
|
|
1544
2068
|
get: function () { return databricks.waitForDatabricksStatement; }
|
|
1545
2069
|
});
|
|
2070
|
+
exports.DatabricksJobsAdapter = DatabricksJobsAdapter;
|
|
1546
2071
|
exports.appHealthCheck = appHealthCheck;
|
|
1547
2072
|
exports.assertDeltaContract = assertDeltaContract;
|
|
1548
2073
|
exports.autoLoaderIngestionCheck = autoLoaderIngestionCheck;
|
|
1549
2074
|
exports.backupRestoreCheck = backupRestoreCheck;
|
|
2075
|
+
exports.builtinTargetPacks = builtinTargetPacks;
|
|
1550
2076
|
exports.classicComputeCheck = classicComputeCheck;
|
|
1551
2077
|
exports.compareToGolden = compareToGolden;
|
|
2078
|
+
exports.createAppsOperationalFoundationPack = createAppsOperationalFoundationPack;
|
|
2079
|
+
exports.createBuiltinTargetPacks = createBuiltinTargetPacks;
|
|
1552
2080
|
exports.createClientFromEnv = createClientFromEnv;
|
|
1553
2081
|
exports.createDuckDbContext = createDuckDbContext;
|
|
1554
2082
|
exports.createEphemeralLakebaseBranch = createEphemeralLakebaseBranch;
|
|
1555
2083
|
exports.createExecutionContext = createExecutionContext;
|
|
2084
|
+
exports.createFoundationTargetPacks = createFoundationTargetPacks;
|
|
2085
|
+
exports.createJobsCodeFoundationPack = createJobsCodeFoundationPack;
|
|
1556
2086
|
exports.createLakebaseTestProvider = createLakebaseTestProvider;
|
|
2087
|
+
exports.createLakeflowFoundationPack = createLakeflowFoundationPack;
|
|
2088
|
+
exports.createLakeflowJobsTargetPack = createLakeflowJobsTargetPack;
|
|
1557
2089
|
exports.createMockDatabricksClient = createMockDatabricksClient;
|
|
2090
|
+
exports.createSqlDeltaFoundationPack = createSqlDeltaFoundationPack;
|
|
2091
|
+
exports.createTargetPackRegistry = createTargetPackRegistry;
|
|
2092
|
+
exports.createUnityStorageFoundationPack = createUnityStorageFoundationPack;
|
|
1558
2093
|
exports.createWorkspaceContext = createWorkspaceContext;
|
|
1559
2094
|
exports.customLiveCheck = customLiveCheck;
|
|
1560
2095
|
exports.dbtBuildCheck = dbtBuildCheck;
|
|
1561
2096
|
exports.defineLiveSuite = defineLiveSuite;
|
|
2097
|
+
exports.defineTargetPack = defineTargetPack;
|
|
2098
|
+
exports.defineTargetPackRun = defineTargetPackRun;
|
|
1562
2099
|
exports.deleteEphemeralLakebaseBranch = deleteEphemeralLakebaseBranch;
|
|
1563
2100
|
exports.deleteVolumeFile = deleteVolumeFile;
|
|
1564
2101
|
exports.deltaContractCheck = deltaContractCheck;
|
|
2102
|
+
exports.doctorTargetPack = doctorTargetPack;
|
|
1565
2103
|
exports.ensureVolumeDirectory = ensureVolumeDirectory;
|
|
1566
2104
|
exports.expectQueryToMatchGolden = expectQueryToMatchGolden;
|
|
1567
2105
|
exports.expectTable = expectTable;
|
|
1568
2106
|
exports.failureRecoveryCheck = failureRecoveryCheck;
|
|
2107
|
+
exports.foundationTargetPacks = foundationTargetPacks;
|
|
1569
2108
|
exports.jobRunCheck = jobRunCheck;
|
|
2109
|
+
exports.jobsOrchestrationCheck = jobsOrchestrationCheck;
|
|
1570
2110
|
exports.lakebaseCredentialCheck = lakebaseCredentialCheck;
|
|
1571
2111
|
exports.lakebaseRoundTripCheck = lakebaseRoundTripCheck;
|
|
2112
|
+
exports.normalizeJobRun = normalizeJobRun;
|
|
1572
2113
|
exports.normalizeRow = normalizeRow;
|
|
1573
2114
|
exports.normalizeVolumeRoot = normalizeVolumeRoot;
|
|
1574
2115
|
exports.notebookSubmitCheck = notebookSubmitCheck;
|
|
1575
2116
|
exports.parseFixtureColumns = parseFixtureColumns;
|
|
2117
|
+
exports.parseJobOrchestrationLiveSpec = parseJobOrchestrationLiveSpec;
|
|
1576
2118
|
exports.parseStatementRows = parseStatementRows;
|
|
1577
2119
|
exports.performanceBudgetCheck = performanceBudgetCheck;
|
|
1578
2120
|
exports.pipelineRefreshCheck = pipelineRefreshCheck;
|
|
@@ -1584,6 +2126,7 @@ exports.resolveTokenProvider = resolveTokenProvider;
|
|
|
1584
2126
|
exports.restrictedPrincipalCheck = restrictedPrincipalCheck;
|
|
1585
2127
|
exports.rollbackCheck = rollbackCheck;
|
|
1586
2128
|
exports.runLiveSuite = runLiveSuite;
|
|
2129
|
+
exports.runTargetPack = runTargetPack;
|
|
1587
2130
|
exports.secretRotationCheck = secretRotationCheck;
|
|
1588
2131
|
exports.secretScopeCheck = secretScopeCheck;
|
|
1589
2132
|
exports.serverlessComputeCheck = serverlessComputeCheck;
|
|
@@ -1592,6 +2135,7 @@ exports.toLakebaseBranchId = toLakebaseBranchId;
|
|
|
1592
2135
|
exports.toNamedParameters = toNamedParameters;
|
|
1593
2136
|
exports.unityCatalogAccessCheck = unityCatalogAccessCheck;
|
|
1594
2137
|
exports.uploadVolumeFile = uploadVolumeFile;
|
|
2138
|
+
exports.validateJobGraph = validateJobGraph;
|
|
1595
2139
|
exports.volumeIOCheck = volumeIOCheck;
|
|
1596
2140
|
exports.waitForPipelineUpdate = waitForPipelineUpdate;
|
|
1597
2141
|
//# sourceMappingURL=index.cjs.map
|