@ingestron/pack-databricks 1.0.1 → 2.0.0

package/dist/fold-python.js

@@ -0,0 +1,108 @@
+// Values that differ in a PySpark transform's file: Python reads the pipeline's
+// configuration with spark.conf.get (spec/behaviour.md, 3.3). Decorator arguments
+// are evaluated when the file is loaded, so each value they need is read once
+// into a module-level constant and put in an f-string, as Databricks' "Create
+// tables in a for loop" example builds `name=`. Reads for the transform's
+// inputs stay inside the function body.
+/** A double-quoted Python text on one line, as the pack writes them with JSON. */
+const quoted = /"(?:[^"\\\n]|\\.)*"/g;
+/** A marker, with the variable's name. */
+const marker = /⟦([^⟦⟧]+)⟧/;
+/** The function that defines the table: everything before it runs when the file loads. */
+const wrapper = /^def ingestron_/m;
+/** The comment above the module-level constants. */
+const header = "# Values that differ between environments, from the pipeline's configuration.";
+/** The configuration key a variable is read under. */
+export function configurationKey(name) {
+    return `ingestron.${name}`;
+}
+/** The module-level constant a variable is read into. */
+function constantOf(name) {
+    return `ingestron_${name}`;
+}
+/**
+ * The file with each marker inside a double-quoted text read from the configuration:
+ * before the table's function, `"⟦schema_silver⟧.totals"` becomes
+ * `f"{ingestron_schema_silver}.totals"` with
+ * `ingestron_schema_silver = spark.conf.get("ingestron.schema_silver")` above the
+ * decorators; inside it, `spark.conf.get("ingestron.schema_bronze") + ".orders"`.
+ * Undefined when a marker is anywhere else, which no Python expression can stand for.
+ */
+export function pythonValues(text) {
+    const at = Math.max(text.search(wrapper), 0);
+    const names = [];
+    const loaded = text
+        .slice(0, at)
+        .replace(quoted, (found) => (marker.test(found) ? formatted(found, names) : found));
+    const body = text
+        .slice(at)
+        .replace(quoted, (found) => (marker.test(found) ? joined(found) : found));
+    const folded = `${loaded}${body}`;
+    if (marker.test(folded))
+        return undefined;
+    if (names.length === 0)
+        return folded;
+    const reads = [...new Set(names)].map((name) => `${constantOf(name)} = spark.conf.get("${configurationKey(name)}")`);
+    return folded.replace(/^@dp\./m, `${[header, ...reads].join("\n")}\n\n\n@dp.`);
+}
+/** A text as an f-string, each marker as its constant, braces in the text doubled. */
+function formatted(found, names) {
+    const parts = found.slice(1, -1).split(new RegExp(marker.source));
+    const inner = parts.map((part, index) => {
+        if (index % 2 === 0)
+            return part.replaceAll("{", "{{").replaceAll("}", "}}");
+        names.push(part);
+        return `{${constantOf(part)}}`;
+    });
+    return `f"${inner.join("")}"`;
+}
+function joined(found) {
+    const parts = found.slice(1, -1).split(new RegExp(marker.source));
+    return parts
+        .map((part, index) => index % 2 === 1 ? `spark.conf.get("${configurationKey(part)}")` : part && `"${part}"`)
+        .filter(Boolean)
+        .join(" + ");
+}
+/**
+ * The file as the pipeline runs it, as the pack would have written it with the values:
+ * the module-level reads removed and each f-string that uses them a plain text, then
+ * each spark.conf.get of an `ingestron.*` key in the body joined into the texts around it.
+ */
+export function pythonWithValues(text, values) {
+    const constants = {};
+    const block = new RegExp(`^${header.replace(/[.()]/g, "\\$&")}\\n((?:\\w+ = spark\\.conf\\.get\\("[^"]+"\\)\\n)+)\\n\\n`, "m");
+    const unread = text.replace(block, (found, lines) => {
+        for (const [, constant, key] of lines.matchAll(/^(\w+) = spark\.conf\.get\("([^"]+)"\)$/gm))
+            if (values[key] !== undefined)
+                constants[constant] = values[key];
+        return lines.trim().split("\n").length === Object.keys(constants).length ? "" : found;
+    });
+    const plain = unread.replace(/f"((?:[^"\\\n]|\\.)*)"/g, (found, inner) => Object.keys(constants).some((name) => inner.includes(`{${name}}`))
+        ? `"${unformatted(inner, constants)}"`
+        : found);
+    return plain
+        .split("\n")
+        .map((line) => withReads(line, values))
+        .join("\n");
+}
+function unformatted(inner, constants) {
+    return inner.replace(/\{\{|\}\}|\{(\w+)\}/g, (found, name) => {
+        if (found === "{{")
+            return "{";
+        if (found === "}}")
+            return "}";
+        const value = name === undefined ? undefined : constants[name];
+        return value === undefined ? found : JSON.stringify(value).slice(1, -1);
+    });
+}
+/** A body line's spark.conf.get reads as their values, joined into the texts beside them. */
+function withReads(line, values) {
+    if (!line.includes('spark.conf.get("ingestron.'))
+        return line;
+    let merged = line.replace(/spark\.conf\.get\("([^"]+)"\)/g, (found, key) => values[key] === undefined ? found : JSON.stringify(values[key]));
+    for (let previous = ""; previous !== merged;) {
+        previous = merged;
+        merged = merged.replace(/"((?:[^"\\]|\\.)*)" \+ "((?:[^"\\]|\\.)*)"/, '"$1$2"');
+    }
+    return merged;
+}

package/dist/fold-target.d.ts

@@ -0,0 +1,29 @@
+import { type Data } from "./bundle.js";
+import type { Plan } from "./fold-plan.js";
+import { type Variable } from "./fold-values.js";
+/** Each target's settings, and the deploy-time value each target's host is, if any. */
+export interface Targets {
+    settings: Record<string, Data>;
+    /** NAME of the deploy-time value that is a target's whole host, by target. */
+    hosts: Record<string, string>;
+}
+/** The settings of every target, before the variables are set. */
+export declare function targetSettings(plan: Plan): Targets;
+/** How the refresh job's schedule is paused in each target. */
+export type Pause = {
+    kind: "same";
+} | {
+    kind: "presets";
+    presets: Record<string, "PAUSED" | "UNPAUSED">;
+} | {
+    kind: "variable";
+    variable: Variable;
+};
+/**
+ * When the schedule's pause differs: a preset only where the target's mode does not
+ * pause it by itself, or a variable when a preset would also change a continuous
+ * pipeline. Databricks sets `continuous: false` on every pipeline under a PAUSED
+ * preset, and development mode does the same unless an UNPAUSED preset replaces its
+ * own PAUSED one.
+ */
+export declare function pauseOf(plan: Plan): Pause;

package/dist/fold-target.js

@@ -0,0 +1,97 @@
+// Each target's own settings in the folded bundle: what Databricks takes per
+// target (mode, workspace, run_as, permissions) as each target's values, and
+// presets for the tags and schedule pause that differ (spec/behaviour.md, 2.3).
+import { splitDeployText } from "@ingestron/definitions";
+import { targetOf } from "./bundle.js";
+import { hasContinuous } from "./connect.js";
+import { mapStrings } from "./walk.js";
+import { deployProblem, deployVariable, placeVariable, supply, } from "./fold-values.js";
+/** The settings of every target, before the variables are set. */
+export function targetSettings(plan) {
+    const hosts = {};
+    const settings = Object.fromEntries(plan.targets.map((target, index) => {
+        const model = plan.models[target];
+        const own = targetOf(model);
+        const host = hostOf(plan, target, own);
+        if (host)
+            hosts[target] = host;
+        const tags = plan.tags[target];
+        const data = {
+            mode: own.mode,
+            ...(index === 0 && { default: true }),
+            ...withVariables(plan, target, own),
+            ...(Object.keys(tags).length > 0 && {
+                presets: { tags: withVariables(plan, target, tags) },
+            }),
+        };
+        return [target, data];
+    }));
+    return { settings, hosts };
+}
+/**
+ * A host supplied when deploying: Databricks takes no variable there, so the target
+ * leaves the host out and the pipeline sets DATABRICKS_HOST.
+ */
+function hostOf(plan, target, settings) {
+    const workspace = settings.workspace;
+    const model = plan.models[target];
+    const parts = splitDeployText(workspace?.host ?? "", model.deployValues ?? []);
+    const value = parts.find((part) => typeof part !== "string");
+    if (!value || typeof value === "string")
+        return undefined;
+    delete workspace.host;
+    if (Object.keys(workspace).length === 0)
+        delete settings.workspace;
+    if (parts.length === 1)
+        return value.name;
+    plan.problems.push(deployProblem(`The workspace host in ${target} holds \${envvar:${value.name}} within other text; Databricks takes a host supplied when deploying only as a whole, from DATABRICKS_HOST.`, model, value.name));
+    return undefined;
+}
+/** Each deploy-time value in a target's own settings becomes its variable, which the target leaves unset. */
+function withVariables(plan, target, node) {
+    return mapStrings(node, (text) => replaced(plan, target, plan.models[target], text));
+}
+function replaced(plan, target, model, text) {
+    return splitDeployText(text, model.deployValues ?? [])
+        .map((part) => {
+        if (typeof part === "string")
+            return part;
+        const variable = deployVariable(plan.variables, part.name);
+        plan.problems.push(...supply(variable, target, model));
+        return `\${var.${variable.name}}`;
+    })
+        .join("");
+}
+/**
+ * When the schedule's pause differs: a preset only where the target's mode does not
+ * pause it by itself, or a variable when a preset would also change a continuous
+ * pipeline. Databricks sets `continuous: false` on every pipeline under a PAUSED
+ * preset, and development mode does the same unless an UNPAUSED preset replaces its
+ * own PAUSED one.
+ */
+export function pauseOf(plan) {
+    const schedules = plan.targets.map((target) => plan.models[target].platform.schedule);
+    if (schedules.some((schedule) => !schedule))
+        return { kind: "same" };
+    if (new Set(schedules.map((schedule) => schedule.paused)).size === 1)
+        return { kind: "same" };
+    const presets = {};
+    for (const target of plan.targets) {
+        const { mode, schedule } = plan.models[target].platform;
+        if (mode === "production" && schedule.paused)
+            presets[target] = "PAUSED";
+        if (mode === "development" && !schedule.paused)
+            presets[target] = "UNPAUSED";
+    }
+    if (!Object.keys(presets).some((target) => hasContinuous(plan.models[target])))
+        return { kind: "presets", presets };
+    const values = Object.fromEntries(plan.targets.map((target) => [
+        target,
+        plan.models[target].platform.schedule.paused ? "PAUSED" : "UNPAUSED",
+    ]));
+    const place = {
+        name: "pause_status",
+        description: "Whether the refresh job's schedule is paused.",
+    };
+    return { kind: "variable", variable: placeVariable(plan.variables, place, values) };
+}

package/dist/fold-text.d.ts

@@ -0,0 +1,17 @@
+import { type EnvironmentsProblem, type ResolvedModel } from "@ingestron/definitions";
+import { type Variables } from "./fold-values.js";
+import type { Place } from "./fold-names.js";
+/** What folding a text needs, and where it reports problems. */
+export interface TextFold {
+    targets: string[];
+    models: Readonly<Record<string, ResolvedModel>>;
+    variables: Variables;
+    problems: EnvironmentsProblem[];
+}
+/** The folded text for one place, given each target's text; `place` names a new variable. */
+export declare function foldText(fold: TextFold, values: Record<string, string>, place: Place): string;
+/**
+ * Code folded token by token: each run of tokens that differs is one value. Undefined when
+ * the targets' code does not have the same number of tokens, so differs in more than values.
+ */
+export declare function foldCode(fold: TextFold, values: Record<string, string>, place: (count: number) => Place): string | undefined;

package/dist/fold-text.js

@@ -0,0 +1,87 @@
+// Folding one text that differs between targets: a variable for the whole
+// value, or, where a target supplies part of it when deploying, a variable for
+// that part with the rest literal. Code is folded token by token.
+import { splitDeployText, } from "@ingestron/definitions";
+import { deployProblem, deployVariable, marker, placeVariable, setIn, supply, } from "./fold-values.js";
+/** The folded text for one place, given each target's text; `place` names a new variable. */
+export function foldText(fold, values, place) {
+    const parts = (target) => splitDeployText(values[target], fold.models[target].deployValues ?? []);
+    const supplying = fold.targets.filter((target) => parts(target).some((part) => typeof part !== "string"));
+    if (supplying.length === 0)
+        return marker(placeVariable(fold.variables, place, values).name);
+    const [first, ...others] = supplying;
+    const model = fold.models[first];
+    const pattern = parts(first);
+    const names = pattern.flatMap((part) => (typeof part === "string" ? [] : [part.name]));
+    const other = others.find((target) => values[target] !== values[first]);
+    if (other) {
+        fold.problems.push(deployProblem(`This value is "${values[first]}" in ${first} and "${values[other]}" in ${other}; a value supplied when deploying must be written the same way in every environment that supplies it.`, model, names[0]));
+        return values[first];
+    }
+    for (const target of supplying)
+        for (const name of names)
+            fold.problems.push(...supply(deployVariable(fold.variables, name), target, fold.models[target]));
+    const literal = fold.targets.filter((target) => !supplying.includes(target));
+    for (const target of literal)
+        fold.problems.push(...capture(fold, values, target, pattern, first));
+    return pattern
+        .map((part) => typeof part === "string" ? part : marker(deployVariable(fold.variables, part.name).name))
+        .join("");
+}
+/** A literal target sets the one deploy-time value to its text between the literal parts. */
+function capture(fold, values, target, pattern, supplier) {
+    const text = values[target];
+    const at = pattern.findIndex((part) => typeof part !== "string");
+    const name = pattern[at].name;
+    const model = fold.models[supplier];
+    if (pattern.filter((part) => typeof part !== "string").length > 1)
+        return [
+            deployProblem(`This value is "${text}" in ${target} and "${values[supplier]}" in ${supplier}, which holds more than one value supplied when deploying; write it the same way in every environment.`, model, name),
+        ];
+    const before = pattern.slice(0, at).join("");
+    const after = pattern.slice(at + 1).join("");
+    const fits = text.length >= before.length + after.length && text.startsWith(before) && text.endsWith(after);
+    if (!fits)
+        return [
+            deployProblem(`This value is "${text}" in ${target}, which does not have the form "${values[supplier]}" of ${supplier}, so \${envvar:${name}} cannot stand for part of it.`, model, name),
+        ];
+    const middle = text.slice(before.length, text.length - after.length);
+    return setIn(deployVariable(fold.variables, name), target, middle, fold.models[target]);
+}
+/** Words, runs of space, and single other characters; a deploy-time value is one token. */
+function tokens(text, model) {
+    return splitDeployText(text, model.deployValues ?? []).flatMap((part) => typeof part === "string"
+        ? (part.match(/[A-Za-z0-9_]+|\s+|[^A-Za-z0-9_\s]/g) ?? [])
+        : [part.text]);
+}
+/**
+ * Code folded token by token: each run of tokens that differs is one value. Undefined when
+ * the targets' code does not have the same number of tokens, so differs in more than values.
+ */
+export function foldCode(fold, values, place) {
+    const lists = fold.targets.map((target) => tokens(values[target], fold.models[target]));
+    if (new Set(lists.map((list) => list.length)).size > 1)
+        return undefined;
+    const parts = [];
+    let run = [];
+    let count = 0;
+    const flush = () => {
+        if (run.length === 0)
+            return;
+        const texts = Object.fromEntries(fold.targets.map((target, index) => [target, run.map((tokens) => tokens[index]).join("")]));
+        count += 1;
+        parts.push(foldText(fold, texts, place(count)));
+        run = [];
+    };
+    lists[0].forEach((token, position) => {
+        const column = lists.map((list) => list[position]);
+        if (new Set(column).size === 1) {
+            flush();
+            parts.push(token);
+        }
+        else
+            run.push(column);
+    });
+    flush();
+    return parts.join("");
+}

package/dist/fold-values.d.ts

@@ -0,0 +1,54 @@
+import type { EnvironmentsProblem, ResolvedModel } from "@ingestron/definitions";
+/** The targets' models, by target name, in project.yaml's order. */
+export type Models = Readonly<Record<string, ResolvedModel>>;
+/** One bundle variable. */
+export interface Variable {
+    name: string;
+    description: string;
+    /** A list, declared `type: complex`. */
+    complex?: boolean;
+    /** Each target's value; a target not here leaves it to the deploying pipeline. */
+    values: Record<string, unknown>;
+    /** For a deploy-time value: its NAME, as in `${envvar:NAME}`. */
+    deploy?: string;
+    /** Targets in which the deploying pipeline must supply it. */
+    supplied: Set<string>;
+}
+/** Every variable of a fold, in the order made. */
+export interface Variables {
+    list: Variable[];
+    /** Variables for values that differ, by the name their place gives and their values in target order. */
+    byValues: Map<string, Variable>;
+}
+/** The marker that stands for a variable while folding. */
+export declare function marker(name: string): string;
+/** Each marker in a text, with the variable's name. */
+export declare const markers: RegExp;
+/** True when a text holds a marker. */
+export declare function hasMarker(text: string): boolean;
+/** A variable's name from a deploy-time value's NAME: lower case, `_` for other characters. */
+export declare function deployName(name: string): string;
+/**
+ * The variable for a value that differs: places of the same kind (the same name) with the
+ * same value in every target share one, such as the catalog of every table.
+ */
+export declare function placeVariable(variables: Variables, place: {
+    name: string;
+    description: string;
+    complex?: boolean;
+}, values: Record<string, unknown>): Variable;
+/** The variable of each deploy-time value in the models, in the order they first appear. */
+export declare function deployVariables(models: Readonly<Record<string, ResolvedModel>>): Variables;
+/** The variable of a deploy-time value. */
+export declare function deployVariable(variables: Variables, name: string): Variable;
+/** Where a target's deploy-time value is used, for a problem's location. */
+export declare function useOf(model: ResolvedModel | undefined, name: string): {
+    file: string;
+    line?: number;
+};
+/** A `DEPLOY_VALUE` error at a deploy-time value's first use in a target. */
+export declare function deployProblem(message: string, model: ResolvedModel | undefined, name: string): EnvironmentsProblem;
+/** The deploying pipeline supplies the value in this target; a problem if the target sets it. */
+export declare function supply(variable: Variable, target: string, model: ResolvedModel): EnvironmentsProblem[];
+/** The target sets the deploy-time value's variable to this text; a problem if it cannot. */
+export declare function setIn(variable: Variable, target: string, text: string, model: ResolvedModel): EnvironmentsProblem[];

package/dist/fold-values.js

@@ -0,0 +1,98 @@
+const open = "⟦";
+const close = "⟧";
+/** The marker that stands for a variable while folding. */
+export function marker(name) {
+    return `${open}${name}${close}`;
+}
+/** Each marker in a text, with the variable's name. */
+export const markers = /⟦([^⟦⟧]+)⟧/g;
+/** True when a text holds a marker. */
+export function hasMarker(text) {
+    return text.includes(open);
+}
+/** A variable's name from a deploy-time value's NAME: lower case, `_` for other characters. */
+export function deployName(name) {
+    return name.toLowerCase().replace(/[^a-z0-9_]+/g, "_");
+}
+function unique(variables, name) {
+    const taken = new Set(variables.list.map((variable) => variable.name));
+    let candidate = name;
+    for (let count = 2; taken.has(candidate); count += 1)
+        candidate = `${name}_${count}`;
+    return candidate;
+}
+/**
+ * The variable for a value that differs: places of the same kind (the same name) with the
+ * same value in every target share one, such as the catalog of every table.
+ */
+export function placeVariable(variables, place, values) {
+    const key = JSON.stringify([place.name, Object.values(values)]);
+    const found = variables.byValues.get(key);
+    if (found)
+        return found;
+    const variable = {
+        ...place,
+        name: unique(variables, place.name),
+        values,
+        supplied: new Set(),
+    };
+    variables.list.push(variable);
+    variables.byValues.set(key, variable);
+    return variable;
+}
+/** The variable of each deploy-time value in the models, in the order they first appear. */
+export function deployVariables(models) {
+    const variables = { list: [], byValues: new Map() };
+    for (const model of Object.values(models))
+        for (const value of model.deployValues ?? []) {
+            if (variables.list.some((variable) => variable.deploy === value.name))
+                continue;
+            const name = unique(variables, deployName(value.name));
+            variables.list.push({
+                name,
+                description: `Supplied when deploying, as BUNDLE_VAR_${name} (${value.name} in the definitions).`,
+                values: {},
+                deploy: value.name,
+                supplied: new Set(),
+            });
+        }
+    return variables;
+}
+/** The variable of a deploy-time value. */
+export function deployVariable(variables, name) {
+    return variables.list.find((variable) => variable.deploy === name);
+}
+/** Where a target's deploy-time value is used, for a problem's location. */
+export function useOf(model, name) {
+    const use = model?.deployValues?.find((value) => value.name === name)?.uses[0];
+    return use
+        ? { file: use.file, ...(use.line !== undefined && { line: use.line }) }
+        : { file: "project.yaml" };
+}
+/** A `DEPLOY_VALUE` error at a deploy-time value's first use in a target. */
+export function deployProblem(message, model, name) {
+    return { code: "DEPLOY_VALUE", severity: "error", message, ...useOf(model, name) };
+}
+/** The deploying pipeline supplies the value in this target; a problem if the target sets it. */
+export function supply(variable, target, model) {
+    variable.supplied.add(target);
+    if (variable.values[target] === undefined)
+        return [];
+    return [
+        conflict(variable, target, model, `writes "${String(variable.values[target])}" for it elsewhere`),
+    ];
+}
+/** The target sets the deploy-time value's variable to this text; a problem if it cannot. */
+export function setIn(variable, target, text, model) {
+    const current = variable.values[target];
+    if (variable.supplied.has(target))
+        return [conflict(variable, target, model, `writes "${text}" where other environments use it`)];
+    if (current !== undefined && current !== text)
+        return [conflict(variable, target, model, `writes "${String(current)}" and "${text}" for it`)];
+    variable.values[target] = text;
+    return [];
+}
+function conflict(variable, target, model, what) {
+    const text = `\${envvar:${variable.deploy}}`;
+    return deployProblem(`${text} stands for one value in each environment, but ${target} ${what}.`, model, variable.deploy);
+}

package/dist/fold.d.ts

@@ -0,0 +1,8 @@
+import { type EnvironmentsBuild } from "@ingestron/definitions";
+import type { Models } from "./fold-values.js";
+/**
+ * One bundle for every target. With an error among the problems, the files are empty.
+ * @example
+ * const { files, deployValues, problems } = buildEnvironments({ dev: devModel, prod: prodModel });
+ */
+export declare function buildEnvironments(models: Models): EnvironmentsBuild;

package/dist/fold.js

@@ -0,0 +1,69 @@
+// Every environment in one bundle, with a target per environment: the values
+// that differ become bundle variables, set in each target, and the result is
+// checked against each environment's own bundle (spec/behaviour.md).
+import { compareTargets, } from "@ingestron/definitions";
+import { build } from "./build.js";
+import { foldFiles } from "./fold-files.js";
+import { pipelineCatalog, planFold } from "./fold-plan.js";
+import { catalogProblems, continuousProblems, once, unlikeProblem } from "./fold-problems.js";
+import { bundleView, sameFile, suppliedFor } from "./view.js";
+/**
+ * One bundle for every target. With an error among the problems, the files are empty.
+ * @example
+ * const { files, deployValues, problems } = buildEnvironments({ dev: devModel, prod: prodModel });
+ */
+export function buildEnvironments(models) {
+    const { structural, differing } = compareTargets(models);
+    if (structural.length > 0)
+        return failed(structural);
+    const early = [...catalogProblems(models, pipelineCatalog), ...continuousProblems(models)];
+    if (early.length > 0)
+        return failed(early);
+    const plan = planFold(models, differing);
+    const folded = foldFiles(plan);
+    const problems = once(plan.problems);
+    if (problems.some((problem) => problem.severity === "error"))
+        return failed(problems);
+    const deployValues = supplied(models, folded);
+    const unlike = viewProblems(models, folded.files, deployValues, plan.unfolded);
+    if (unlike.length > 0)
+        return failed(unlike);
+    return { files: folded.files, deployValues, problems };
+}
+function failed(problems) {
+    return { files: {}, deployValues: [], problems: once(problems) };
+}
+/** Each deploy-time value the files take, in the order the models first hold them. */
+function supplied(models, folded) {
+    const names = [
+        ...new Set(Object.values(models).flatMap((model) => (model.deployValues ?? []).map((value) => value.name))),
+    ];
+    return names.flatMap((name) => {
+        const variable = folded.declared.find((item) => item.deploy === name);
+        const suppliedAs = [
+            ...(variable && variable.supplied.size > 0 ? [`BUNDLE_VAR_${variable.name}`] : []),
+            ...(Object.values(folded.hosts).includes(name) ? ["DATABRICKS_HOST"] : []),
+        ];
+        return suppliedAs.length > 0 ? [{ name, suppliedAs }] : [];
+    });
+}
+/**
+ * The fold property, checked when generating: each target's view of the folded files is
+ * what `build` gives for that target alone. What differs is ENVIRONMENTS_DIFFER.
+ */
+function viewProblems(models, files, values, unfolded) {
+    const unlike = new Map();
+    for (const [target, model] of Object.entries(models)) {
+        const viewed = bundleView(files, target, suppliedFor(model, values));
+        const rendered = build(model);
+        const paths = [...new Set([...Object.keys(viewed), ...Object.keys(rendered)])];
+        for (const path of paths) {
+            const same = viewed[path] !== undefined &&
+                rendered[path] !== undefined &&
+                sameFile(path, viewed[path], rendered[path]);
+            if (!same)
+                unlike.set(path, [...(unlike.get(path) ?? []), target]);
+        }
+    }
+    return [...unlike].map(([path, targets]) => unlikeProblem(models, path, targets, unfolded));
+}

package/dist/pack.d.ts

@@ -1,8 +1,9 @@
 import type { Pack } from "@ingestron/definitions";
 /**
- * The Databricks pack, version 1. Name it in a standard as `pack: databricks@1`.
+ * The Databricks pack, version 2. Name it in a standard as `packs: [databricks@2]`. It
+ * builds every environment into one bundle, with a target per environment.
  * @example
- * const result = loadDefinitions("definitions", { pack: databricks, environment: "dev" });
+ * const result = loadDefinitions("definitions", { packs: [databricks], environment: "dev" });
  * if (result.model) console.log(Object.keys(databricks.build(result.model)));
  */
 export declare const databricks: Pack;

package/dist/pack.js

@@ -2,6 +2,7 @@
 import { contract } from "./contract.js";
 import { connectors } from "./connectors.js";
 import { build } from "./build.js";
+import { buildEnvironments } from "./fold.js";
 import { request } from "./request.js";
 import { capabilities, tagProblems } from "./capabilities.js";
 import { lakeflowConnect } from "./connect-method.js";
@@ -9,14 +10,15 @@ import { connectGroups, sharedProblems } from "./connect.js";
 import { files, materialisedView, sql, streamingTable } from "./methods.js";
 import { pyspark } from "./pyspark-method.js";
 /**
- * The Databricks pack, version 1. Name it in a standard as `pack: databricks@1`.
+ * The Databricks pack, version 2. Name it in a standard as `packs: [databricks@2]`. It
+ * builds every environment into one bundle, with a target per environment.
  * @example
- * const result = loadDefinitions("definitions", { pack: databricks, environment: "dev" });
+ * const result = loadDefinitions("definitions", { packs: [databricks], environment: "dev" });
  * if (result.model) console.log(Object.keys(databricks.build(result.model)));
  */
 export const databricks = {
     id: "databricks",
-    version: 1,
+    version: 2,
     defaults: {
         naming: {
             catalog: "{{ environment }}",
@@ -58,6 +60,26 @@ export const databricks = {
     },
     connectors,
     capabilities,
+    platformFields: [
+        "mode",
+        "workspace.host",
+        "workspace.root_path",
+        "run_as",
+        "permissions",
+        "tags",
+        "compute.serverless",
+        "compute.budget_policy_id",
+        "compute.channel",
+        "schedule.cron",
+        "schedule.timezone",
+        "schedule.paused",
+        "notifications.on_failure",
+        "grants",
+        "manage.schemas",
+        "manage.tags",
+        "manage.comments",
+        "manage.checks",
+    ],
     contract,
     check(model) {
         const problems = [];
@@ -68,5 +90,6 @@ export const databricks = {
         return [...problems, ...tagProblems(model), ...sharedProblems(connectGroups(model))];
     },
     build,
+    buildEnvironments,
     request,
 };

package/dist/python-data.d.ts

@@ -1,6 +1,6 @@
-import { type ResolvedTable } from "@ingestron/definitions";
-/** Everything materialised-view.py.liquid needs. */
-export declare function pythonData(table: ResolvedTable, header: string): {
+import type { ResolvedTable } from "@ingestron/definitions";
+/** Everything materialised-view.py.liquid needs, in the pipeline whose catalog is `catalog`. */
+export declare function pythonData(table: ResolvedTable, header: string, catalog: string): {
     header: string;
     code: string;
     name: string;