@ingestron/pack-databricks 1.1.0 → 2.0.0

package/README.md

@@ -4,7 +4,7 @@ The Databricks pack: from resolved Ingestron definitions it generates a
 [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/)
 (formerly Databricks Asset Bundle) with one
 [Lakeflow Spark Declarative Pipeline](https://docs.databricks.com/aws/en/ldp/concepts/)
-per project. Name it in your standard: `packs: [databricks@1]`.
+per project. Name it in your standard: `packs: [databricks@2]`.
 
 What it uses on Databricks:
 
@@ -23,24 +23,33 @@ What it uses on Databricks:
 - **Serverless pipelines**, a scheduled refresh job, Unity Catalog schemas
   and grants when you ask for them, permissions, tags and notifications.
 
-Every value is resolved when generating, so the bundle has no variables.
-Each environment gets its own bundle folder.
+One bundle covers every environment, with a target per environment, as
+Databricks teams lay out bundles by hand. A value is a bundle variable only
+where environments differ; everything the same in every environment is written
+as it is. Each target sets its own variables, so the `targets` block shows
+exactly what changes between dev and prod.
 
 ## Example
 
-Most people use this through the [`ingestron`](https://www.npmjs.com/package/ingestron) command. In code:
+Most people use this through the [`ingestron`](https://www.npmjs.com/package/ingestron) command, which writes the bundle to `build/databricks/`. In code:
 
 ```ts
-import { loadDefinitions } from "@ingestron/definitions";
+import { loadDefinitions, type ResolvedModel } from "@ingestron/definitions";
 import { databricks } from "@ingestron/pack-databricks";
 
-const result = loadDefinitions("definitions", {
-  packs: [databricks],
-  environment: "dev",
-});
-if (result.model) {
-  for (const path of Object.keys(databricks.build(result.model))) console.log(path);
+const models: Record<string, ResolvedModel> = {};
+for (const environment of ["dev", "prod"]) {
+  const result = loadDefinitions("definitions", {
+    packs: [databricks],
+    environment,
+    deployTime: true,
+    environmentsTogether: true,
+  });
+  if (result.model) models[environment] = result.model;
 }
+const { files, problems } = databricks.buildEnvironments!(models);
+for (const problem of problems) console.log(problem.code, problem.message);
+for (const path of Object.keys(files)) console.log(path);
 ```
 
 For the retail example this prints:
@@ -54,10 +63,114 @@ src/retail/silver/daily_sales.sql
 resources/retail_refresh.job.yml
 ```
 
+The values that differ between dev and prod are in `databricks.yml`:
+
+```yaml
+variables:
+  catalog:
+    description: Catalog of the retail tables.
+  shop_path:
+    description: Folder the files of connection "shop" are read from.
+targets:
+  dev:
+    mode: development
+    default: true
+    workspace:
+      host: https://adb-1111111111111111.1.azuredatabricks.net
+    # run_as and permissions, as the platform sets them
+    presets:
+      tags:
+        environment: dev
+    variables:
+      catalog: dev_sales
+      shop_path: abfss://landing@retaildev.dfs.core.windows.net/shop
+  prod:
+    mode: production
+    # prod's workspace, run_as, permissions, presets and variables
+```
+
+The pipeline reads them as `catalog: ${var.catalog}` and as the configuration
+value `ingestron.shop_path`, which its SQL uses as `${ingestron.shop_path}`.
+Tables in the pipeline's catalog are named with two parts, such as
+`bronze.orders`.
+
+## Deploy
+
+From the bundle folder, deploy one environment at a time, the same files each
+time:
+
+```sh
+cd build/databricks
+databricks bundle validate -t prod
+databricks bundle deploy -t prod
+```
+
+A value written as `${envvar:NAME}` in the definitions is supplied when
+deploying: the bundle declares it as a variable that the targets leave unset,
+and the deploying CI pipeline sets `BUNDLE_VAR_<name>` (`name` is NAME in lower
+case). A workspace host supplied this way is `DATABRICKS_HOST`.
+
+`prerequisites/<environment>.sql` holds the grants an administrator runs once
+in that environment. It is not deployed, and neither are `.generated.json` and
+`.generated.lock`, which `ingestron` keeps in the folder.
+
+The after-refresh task (tags, column comments and table checks) is
+`tasks/<pipeline>/apply_contract.py`, outside the pipeline's `src/<pipeline>/`
+sources, so that only the refresh job runs it.
+
+## What changed in 2.0
+
+- One bundle in `build/databricks/` for every environment, in place of a
+  bundle per environment in `build/<environment>/`. Change the standard to
+  `databricks@2`.
+- The bundle name, pipeline and job keys, target names and `root_path` are the
+  same as in 1.x, so the first `bundle deploy -t dev` or `-t prod` updates
+  that environment's resources in place. Run `bundle validate` and
+  `bundle summary` first.
+- The mode is `production` for every environment except `dev`, unless the
+  platform sets it. In 1.x, environments such as `test`, `uat` and `staging`
+  were in `development` mode. On their first 2.0 deploy, jobs and pipelines
+  lose the `[dev <identity>]` prefix in their names, pipelines are no longer in
+  development, managed schemas lose their `dev_<identity>_` prefix (the old
+  schema is deleted after approval, and the pipelines write to a new one), and
+  production mode wants `run_as` and `permissions` unless a service principal
+  deploys. To keep `development` mode for such an environment, write
+  `mode: ${values.mode}` in the platform, `mode: production` in project.yaml's
+  `values`, and `mode: development` in that environment's `values`.
+- Managed schemas are keyed by layer (`bronze`), or by schema name when a
+  layer has several (`bronze_shop`), not `<catalog>_<schema>`. Each
+  environment's deployment state still holds the old key, so with
+  `manage.schemas`, before the first 2.0 deploy, for each environment and each
+  managed schema:
+  1. in the 1.x folder `build/<environment>/`, run
+     `databricks bundle deployment unbind <catalog>_<schema> -t <environment>`;
+  2. in `build/databricks/`, run
+     `databricks bundle deployment bind <key> <catalog>.<schema> -t <environment>`,
+     with the key from `resources/schemas.yml`;
+  3. run `bundle validate` and `bundle summary`, and check that the deploy
+     deletes no schema.
+
+  Without this, the deploy plans to delete each schema under its old key.
+
+- The after-refresh script moved from `src/<pipeline>/apply_contract.py`,
+  where the pipeline also loaded it as its own code, to
+  `tasks/<pipeline>/apply_contract.py`. The first deploy removes the old file.
+- Pipeline code names tables in the pipeline's catalog with two parts.
+- `prerequisites.sql` is `prerequisites/<environment>.sql`.
+- Environments may differ only in values. A table built in some environments
+  only, or code that differs in more than a value, is `ENVIRONMENTS_DIFFER`.
+  A PySpark transform reads names that differ from the pipeline's
+  configuration with `spark.conf.get("ingestron.<name>")`, into a module-level
+  constant for its decorator; values in its own code cannot differ yet.
+- Needs `@ingestron/definitions` 1.9 or later, and Databricks CLI v0.264.0 or
+  later, which adds preset tags to pipelines.
+
+The rules are in [spec/behaviour.md](spec/behaviour.md).
+
 ## With the ADF pack
 
 Used with the [ADF pack](https://github.com/ingestron/pack-adf)
-(`packs: [databricks@1, adf@1]`), streaming tables read the files ADF lands,
+(`packs: [databricks@2, adf@2]`), streaming tables read the files ADF lands,
 from its `landing` folder.
 
 Columns ADF adds to the landed rows (`audit_columns` in ADF 1.4) are read
@@ -70,10 +183,14 @@ with the files; timestamps, which ADF writes as text, are cast to `TIMESTAMP`.
   `templates/` and the transforms' SQL.
 - [`yaml`](https://eemeli.org/yaml/): writes the bundle's YAML files.
 
-## Sources (accessed 2026-10-24)
+## Sources
+
+Bundle variables, targets, presets and pipeline configuration:
+[spec/references.md](spec/references.md) (accessed 2026-11-04). Bundle fields
+from Databricks CLI v1.23.0 `databricks bundle schema` (2026-11-04).
 
-Bundle fields from Databricks CLI v1.22.1 `databricks bundle schema` and
-`databricks/bundle-examples` (`lakeflow_pipelines_sql`). SQL from
+Accessed 2026-10-24: `databricks/bundle-examples` (`lakeflow_pipelines_sql`).
+SQL from
 [CREATE STREAMING TABLE](https://docs.databricks.com/aws/en/ldp/developer/ldp-sql-ref-create-streaming-table.html),
 [CREATE MATERIALIZED VIEW](https://docs.databricks.com/aws/en/ldp/developer/ldp-sql-ref-create-materialized-view),
 [AUTO CDC INTO](https://docs.databricks.com/aws/en/dlt-ref/dlt-sql-ref-apply-changes-into),

package/dist/build.d.ts

@@ -1,7 +1,31 @@
 import type { ResolvedModel } from "@ingestron/definitions";
+import { type Data } from "./bundle.js";
+/** A YAML file as data, with its header comments. */
+export interface YamlFile {
+    headers: string[];
+    data: Data;
+}
+/** The bundle's files before they are written: YAML as data, code as text. */
+export interface BundleData {
+    /** By path, in the order they are built. */
+    files: Record<string, YamlFile | string>;
+    /** The definition file each table's code file comes from, by path. */
+    origins: Record<string, string>;
+}
 /**
- * Every file of the bundle for one environment.
+ * Every file of the bundle for one environment: one target, every value as resolved.
  * @example
  * const files = build(model); // { "databricks.yml": "...", "src/retail/bronze/orders.sql": "..." }
  */
 export declare function build(model: ResolvedModel): Record<string, string>;
+/** The files as text. */
+export declare function written(files: BundleData["files"]): Record<string, string>;
+/** The header of the bundle's own files. */
+export declare function headerOf(model: ResolvedModel): string;
+/** Every file of the bundle for one environment, as data. */
+export declare function bundleData(model: ResolvedModel): BundleData;
+/**
+ * The after-refresh script's path: in `tasks/`, outside every pipeline's `src/<pipeline>/**`
+ * sources whatever the naming standard, so that no pipeline loads it as its own code.
+ */
+export declare function scriptPath(pipeline: string): string;

package/dist/build.js

@@ -1,32 +1,49 @@
-import { bundleFile, jobFile, pipelineExtras, pipelineFile, schemaKeys, schemasFile, yamlText, } from "./bundle.js";
+import { bundleFile, jobFile, pipelineExtras, pipelineFile, pipelineTags, schemaKeys, schemasFile, yamlText, } from "./bundle.js";
 import { connectGroups, gatewayPipeline, ingestionPipeline, previewNote, } from "./connect.js";
 import { applyContract } from "./apply-contract.js";
 import { prerequisites } from "./capabilities.js";
 import { renderApply, renderSql, sqlFile } from "./sql.js";
 const doNotEdit = "Do not edit: change the definitions and generate again.";
 /**
- * Every file of the bundle for one environment.
+ * Every file of the bundle for one environment: one target, every value as resolved.
  * @example
  * const files = build(model); // { "databricks.yml": "...", "src/retail/bronze/orders.sql": "..." }
  */
 export function build(model) {
-    const header = `Generated by Ingestron from the ${model.project.id} definitions (${model.project.environment}). ${doNotEdit}`;
+    return written(bundleData(model).files);
+}
+/** The files as text. */
+export function written(files) {
+    return Object.fromEntries(Object.entries(files).map(([path, file]) => [
+        path,
+        typeof file === "string" ? file : yamlText(file.headers, file.data),
+    ]));
+}
+/** The header of the bundle's own files. */
+export function headerOf(model) {
+    return `Generated by Ingestron from the ${model.project.id} definitions. ${doNotEdit}`;
+}
+/** Every file of the bundle for one environment, as data. */
+export function bundleData(model) {
+    const header = headerOf(model);
     const keys = schemaKeys(model);
     const groups = connectGroups(model);
-    const files = { "databricks.yml": yamlText(header, bundleFile(model)) };
-    const output = { header, keys, files };
+    const files = {
+        "databricks.yml": { headers: [header], data: bundleFile(model) },
+    };
+    const output = { header, keys, files, origins: {} };
     const refreshes = [...connectFiles(model, groups, output), ...sqlFiles(model, groups, output)];
     const tasks = [...refreshes, ...applyFiles(model, refreshes, output)];
     const job = jobFile(model, tasks);
     if (job)
-        files[`resources/${model.names.job}.job.yml`] = yamlText(header, job);
+        files[`resources/${model.names.job}.job.yml`] = { headers: [header], data: job };
     const schemas = schemasFile(model, keys);
     if (schemas)
-        files["resources/schemas.yml"] = yamlText(header, schemas);
+        files["resources/schemas.yml"] = { headers: [header], data: schemas };
     const needs = prerequisites(model, header);
     if (needs)
-        files["prerequisites.sql"] = needs;
-    return files;
+        files[`prerequisites/${model.project.environment}.sql`] = needs;
+    return { files, origins: output.origins };
 }
 /** Lakeflow Connect pipelines; returns the job tasks for those the job refreshes. */
 function connectFiles(model, groups, output) {
@@ -34,14 +51,13 @@ function connectFiles(model, groups, output) {
     return groups.flatMap((group) => {
         const headers = [header, previewNote(group)];
         if (group.gateway) {
-            const gateway = {
-                resources: { pipelines: { [group.gateway]: gatewayPipeline(group, keys) } },
-            };
-            files[`resources/${group.gateway}.pipeline.yml`] = yamlText(headers, gateway);
+            const gateway = { ...gatewayPipeline(group, keys), ...pipelineTags(model) };
+            const data = { resources: { pipelines: { [group.gateway]: gateway } } };
+            files[`resources/${group.gateway}.pipeline.yml`] = { headers, data };
         }
         const pipeline = { ...ingestionPipeline(group, keys), ...pipelineExtras(model) };
-        const ingestion = { resources: { pipelines: { [group.ingestion]: pipeline } } };
-        files[`resources/${group.ingestion}.pipeline.yml`] = yamlText(headers, ingestion);
+        const data = { resources: { pipelines: { [group.ingestion]: pipeline } } };
+        files[`resources/${group.ingestion}.pipeline.yml`] = { headers, data };
         return group.continuous
             ? []
             : [{ key: `ingest_${group.connection.id}`, pipeline: group.ingestion, after: [] }];
@@ -54,23 +70,34 @@ function sqlFiles(model, groups, output) {
     if (tables.length === 0)
         return [];
     const { pipeline } = model.names;
-    files[`resources/${pipeline}.pipeline.yml`] = yamlText(output.header, pipelineFile(model, tables, keys));
+    const data = pipelineFile(model, tables, keys);
+    files[`resources/${pipeline}.pipeline.yml`] = { headers: [output.header], data };
+    const catalog = tables[0].fullName.catalog;
     for (const table of tables) {
         const tableHeader = `Generated by Ingestron from ${table.origin.file} (table ${table.name}). ${doNotEdit}`;
-        const file = sqlFile(table, tableHeader);
-        files[`src/${pipeline}/${table.layer}/${table.name}.${file.extension}`] = renderSql(file);
+        const file = sqlFile(table, tableHeader, catalog);
+        const path = `src/${pipeline}/${table.layer}/${table.name}.${file.extension}`;
+        files[path] = renderSql(file);
+        output.origins[path] = table.origin.file;
     }
     const ingestion = groups
         .filter((group) => !group.continuous)
         .map((group) => `ingest_${group.connection.id}`);
     return [{ key: "refresh", pipeline, after: ingestion }];
 }
+/**
+ * The after-refresh script's path: in `tasks/`, outside every pipeline's `src/<pipeline>/**`
+ * sources whatever the naming standard, so that no pipeline loads it as its own code.
+ */
+export function scriptPath(pipeline) {
+    return `tasks/${pipeline}/apply_contract.py`;
+}
 /** The Python file that applies tags, comments and table checks after every refresh. */
 function applyFiles(model, refreshes, output) {
     const { statements, checks } = applyContract(model);
     if (statements.length === 0 && checks.length === 0)
         return [];
-    const path = `src/${model.names.pipeline}/apply_contract.py`;
+    const path = scriptPath(model.names.pipeline);
     output.files[path] = renderApply({ header: output.header, statements, checks });
     return [
         { key: "apply_contract", python: `../${path}`, after: refreshes.map((task) => task.key) },

package/dist/bundle.d.ts

@@ -1,7 +1,15 @@
 import type { ResolvedModel, ResolvedTable } from "@ingestron/definitions";
-type Data = Record<string, unknown>;
+/** A YAML document as data. */
+export type Data = Record<string, unknown>;
+/**
+ * Files in the bundle folder that are not deployed: the generation manifest, its lock
+ * (left behind if writing stops part way), and the grants an administrator runs by hand.
+ */
+export declare const notDeployed: string[];
 /** `databricks.yml`: one target, named after the environment. */
 export declare function bundleFile(model: ResolvedModel): Data;
+/** The settings Databricks takes per target: mode, workspace, run_as and permissions. */
+export declare function targetOf(model: ResolvedModel): Data;
 /** The pipeline that builds the tables with SQL files under `src/`. */
 export declare function pipelineFile(model: ResolvedModel, tables: ResolvedTable[], schemaKeys: Map<string, string>): Data;
 /** A task of the refresh job: a pipeline to refresh, or a Python file to run, after others. */
@@ -17,14 +25,20 @@ export interface Task {
  * With a Python task it is built without a schedule too, to run on demand.
  */
 export declare function jobFile(model: ResolvedModel, tasks: Task[]): Data | undefined;
-/** Bundle keys of the schemas this bundle manages, by `catalog.schema`. Empty unless `manage.schemas`. */
+/**
+ * Bundle keys of the schemas this bundle manages, by `catalog.schema`, the same in every
+ * environment where only the catalog differs: the layer when the layer has one schema,
+ * else the schema's name. When two catalogs hold a schema of that name, or the key is
+ * taken, it is `<catalog>_<schema>`. Empty unless `manage.schemas`.
+ */
 export declare function schemaKeys(model: ResolvedModel): Map<string, string>;
 /** A schema's name, or a reference to it when the bundle manages it, so the bundle creates it first. */
 export declare function schemaRef(keys: Map<string, string>, catalog: string, schema: string): string;
 /** Managed schemas, with the platform's grants for their layer. */
 export declare function schemasFile(model: ResolvedModel, keys: Map<string, string>): Data | undefined;
-/** Tags and failure notifications, the same on every pipeline. */
+/** The platform's tags, on every pipeline and job. */
+export declare function pipelineTags(model: ResolvedModel): Data;
+/** Tags and failure notifications, the same on every pipeline that builds tables. */
 export declare function pipelineExtras(model: ResolvedModel): Data;
 /** YAML text with header comments, one per line given. */
 export declare function yamlText(headers: string | string[], data: Data): string;
-export {};

package/dist/bundle.js

@@ -1,19 +1,30 @@
 import { stringify } from "yaml";
+/**
+ * Files in the bundle folder that are not deployed: the generation manifest, its lock
+ * (left behind if writing stops part way), and the grants an administrator runs by hand.
+ */
+export const notDeployed = [".generated.json", ".generated.lock", "prerequisites/**"];
 /** `databricks.yml`: one target, named after the environment. */
 export function bundleFile(model) {
+    return {
+        bundle: { name: model.project.id },
+        include: ["resources/*.yml"],
+        sync: { exclude: notDeployed },
+        targets: { [model.project.environment]: { ...targetOf(model), default: true } },
+    };
+}
+/** The settings Databricks takes per target: mode, workspace, run_as and permissions. */
+export function targetOf(model) {
     const { platform } = model;
-    const target = { mode: platform.mode, default: true };
-    if (platform.workspace.host || platform.workspace.root_path)
-        target.workspace = { ...platform.workspace };
+    const target = { mode: platform.mode };
+    const workspace = Object.fromEntries(Object.entries(platform.workspace).filter(([, value]) => value));
+    if (Object.keys(workspace).length > 0)
+        target.workspace = workspace;
     if (platform.run_as)
         target.run_as = platform.run_as;
     if (platform.permissions.length > 0)
         target.permissions = platform.permissions;
-    return {
-        bundle: { name: model.project.id },
-        include: ["resources/*.yml"],
-        targets: { [model.project.environment]: target },
-    };
+    return target;
 }
 /** The pipeline that builds the tables with SQL files under `src/`. */
 export function pipelineFile(model, tables, schemaKeys) {
@@ -77,12 +88,31 @@ function jobTask(task) {
         }),
     };
 }
-/** Bundle keys of the schemas this bundle manages, by `catalog.schema`. Empty unless `manage.schemas`. */
+/**
+ * Bundle keys of the schemas this bundle manages, by `catalog.schema`, the same in every
+ * environment where only the catalog differs: the layer when the layer has one schema,
+ * else the schema's name. When two catalogs hold a schema of that name, or the key is
+ * taken, it is `<catalog>_<schema>`. Empty unless `manage.schemas`.
+ */
 export function schemaKeys(model) {
     if (!model.platform.manage.schemas)
         return new Map();
-    const keys = model.tables.map(({ fullName }) => [`${fullName.catalog}.${fullName.schema}`, `${fullName.catalog}_${fullName.schema}`]);
-    return new Map(keys);
+    const schemas = new Map();
+    for (const { fullName, layer } of model.tables)
+        schemas.set(`${fullName.catalog}.${fullName.schema}`, { ...fullName, layer });
+    const list = [...schemas.values()];
+    const keys = new Map();
+    const used = new Set();
+    for (const { catalog, schema, layer } of list) {
+        const alone = list.filter((other) => other.layer === layer).length === 1;
+        const shared = list.filter((other) => other.schema === schema).length > 1;
+        const both = `${catalog}_${schema}`;
+        const key = alone ? layer : shared ? both : schema;
+        const unique = used.has(key) ? both : key;
+        used.add(unique);
+        keys.set(`${catalog}.${schema}`, unique);
+    }
+    return keys;
 }
 /** A schema's name, or a reference to it when the bundle manages it, so the bundle creates it first. */
 export function schemaRef(keys, catalog, schema) {
@@ -116,12 +146,17 @@ function uniqueSchemas(tables) {
         return true;
     });
 }
-/** Tags and failure notifications, the same on every pipeline. */
+/** The platform's tags, on every pipeline and job. */
+export function pipelineTags(model) {
+    const { tags } = model.platform;
+    return Object.keys(tags).length > 0 ? { tags } : {};
+}
+/** Tags and failure notifications, the same on every pipeline that builds tables. */
 export function pipelineExtras(model) {
     const { platform } = model;
     const alerts = ["on-update-failure", "on-update-fatal-failure", "on-flow-failure"];
     return {
-        ...(Object.keys(platform.tags).length > 0 && { tags: platform.tags }),
+        ...pipelineTags(model),
         ...(platform.notifications.on_failure.length > 0 && {
             notifications: [{ email_recipients: platform.notifications.on_failure, alerts }],
         }),

package/dist/connect.d.ts

@@ -13,6 +13,8 @@ export interface ConnectGroup {
 type Keys = Map<string, string>;
 /** Tables ingested by lakeflow-connect, grouped by connection in path order. */
 export declare function connectGroups(model: ResolvedModel): ConnectGroup[];
+/** True when the bundle has a pipeline that runs continuously: a gateway, or Kafka read continuously. */
+export declare function hasContinuous(model: ResolvedModel): boolean;
 /**
  * Problems with settings that one connection's tables share: its gateway, or
  * whether its ingestion runs continuously.

package/dist/connect.js

@@ -20,6 +20,10 @@ export function connectGroups(model) {
     }
     return [...groups.values()];
 }
+/** True when the bundle has a pipeline that runs continuously: a gateway, or Kafka read continuously. */
+export function hasContinuous(model) {
+    return connectGroups(model).some((group) => group.gateway !== undefined || group.continuous);
+}
 /**
  * Problems with settings that one connection's tables share: its gateway, or
  * whether its ingestion runs continuously.

package/dist/fold-files.d.ts

@@ -0,0 +1,12 @@
+import type { Plan } from "./fold-plan.js";
+import { type Targets } from "./fold-target.js";
+import { type Variable } from "./fold-values.js";
+/** The folded files, and each target's host when it is supplied when deploying. */
+export interface Folded {
+    files: Record<string, string>;
+    hosts: Targets["hosts"];
+    /** The variables the files declare. */
+    declared: Variable[];
+}
+/** The files for every target, folded from the plan's template. */
+export declare function foldFiles(plan: Plan): Folded;