@ingestron/pack-databricks 1.0.1 → 2.0.0

package/dist/fold-files.js

@@ -0,0 +1,190 @@
+import { bundleData, headerOf, scriptPath, written } from "./build.js";
+import { prerequisites } from "./capabilities.js";
+import { codeProblem, differ, listOf } from "./fold-problems.js";
+import { pauseOf, targetSettings } from "./fold-target.js";
+import { hasMarker, marker, markers } from "./fold-values.js";
+import { configurationKey, pythonValues } from "./fold-python.js";
+import { mapStrings } from "./walk.js";
+/** The files for every target, folded from the plan's template. */
+export function foldFiles(plan) {
+    const { files, origins } = bundleData(plan.template);
+    const { pipeline, job } = plan.template.names;
+    const used = { configuration: new Set(), parameters: new Set() };
+    for (const [path, file] of Object.entries(files)) {
+        if (hasMarker(path))
+            plan.problems.push(named(path));
+        const key = typeof file === "string" ? undefined : markedKey(file.data);
+        if (key !== undefined)
+            plan.problems.push(keyed(path, key));
+        if (path.startsWith("prerequisites/"))
+            delete files[path];
+        else if (typeof file === "string")
+            files[path] = foldCodeFile(plan, path, file, used, origins);
+    }
+    configure(files[`resources/${pipeline}.pipeline.yml`], pipeline, used.configuration);
+    const refresh = jobOf(files[`resources/${job}.job.yml`], job);
+    parameterise(refresh, used.parameters);
+    const targets = targetSettings(plan);
+    pause(plan, refresh, targets);
+    for (const file of Object.values(files))
+        if (typeof file !== "string")
+            file.data = variablesIn(file.data);
+    const declared = declare(plan, files, targets);
+    addPrerequisites(plan, files);
+    return { files: written(files), hosts: targets.hosts, declared };
+}
+function jobOf(file, job) {
+    if (!file || typeof file === "string")
+        return undefined;
+    return file.data.resources.jobs[job];
+}
+/** Each target's grants file, from its own model: an administrator runs it by hand. */
+function addPrerequisites(plan, files) {
+    for (const target of plan.targets) {
+        const model = plan.models[target];
+        const needs = prerequisites(model, headerOf(model));
+        if (needs)
+            files[`prerequisites/${target}.sql`] = placeholders(plan, target, needs);
+    }
+}
+function named(path) {
+    return differ(`The generated file ${shown(path)} would be named after a value that differs between environments; names that decide files and resource keys must be the same in every environment.`, "project.yaml");
+}
+function keyed(path, key) {
+    return differ(`The resource key ${shown(key)} in ${path} would be named after a value that differs between environments; names that decide files and resource keys must be the same in every environment.`, "project.yaml");
+}
+/** A text with each marker shown as `<variable>`. */
+function shown(text) {
+    return text.replaceAll(markers, (_, name) => `<${name}>`);
+}
+/** The first key in a YAML document that holds a marker. */
+function markedKey(node) {
+    if (node === null || typeof node !== "object")
+        return undefined;
+    for (const [key, value] of Object.entries(node)) {
+        const found = hasMarker(key) ? key : markedKey(value);
+        if (found !== undefined)
+            return found;
+    }
+    return undefined;
+}
+/** Pipeline SQL takes `${ingestron.<name>}` from the pipeline's configuration; the script, from parameters. */
+function foldCodeFile(plan, path, text, used, origins) {
+    if (!hasMarker(text))
+        return text;
+    const names = [...text.matchAll(markers)].map((match) => match[1]);
+    const kind = kindOf(plan, path);
+    const folded = kind && foldedCode(plan, path, text, kind);
+    if (!kind || folded === undefined) {
+        plan.problems.push(codeProblem(plan, path, names, origins[path] ?? "project.yaml"));
+        return text;
+    }
+    for (const name of names)
+        (kind === "script" ? used.parameters : used.configuration).add(name);
+    return folded;
+}
+/**
+ * SQL and the script take `${ingestron.<name>}`; a PySpark file reads spark.conf.get,
+ * except in the transform's own code. Undefined when the file cannot take the values.
+ */
+function foldedCode(plan, path, text, kind) {
+    if (kind !== "python")
+        return text.replaceAll(markers, (_, name) => `\${${configurationKey(name)}}`);
+    return userCode(plan, path) ? undefined : pythonValues(text);
+}
+/** The after-refresh script, or a SQL or Python file of the pipeline that builds the tables. */
+function kindOf(plan, path) {
+    const { pipeline } = plan.template.names;
+    if (path === scriptPath(pipeline))
+        return "script";
+    if (!path.startsWith(`src/${pipeline}/`))
+        return undefined;
+    if (path.endsWith(".sql"))
+        return "sql";
+    return path.endsWith(".py") ? "python" : undefined;
+}
+/** True when the values are in a PySpark transform's own code, which the pack copies as it is. */
+function userCode(plan, path) {
+    const name = path.replace(/^.*\//, "").replace(/\.py$/, "");
+    const table = plan.template.tables.find((candidate) => candidate.name === name);
+    return hasMarker(table?.transform?.code ?? "");
+}
+function configure(file, pipeline, names) {
+    if (!file || typeof file === "string" || names.size === 0)
+        return;
+    const pipelines = file.data.resources.pipelines;
+    const configuration = Object.fromEntries([...names].map((name) => [configurationKey(name), marker(name)]));
+    pipelines[pipeline] = { ...pipelines[pipeline], configuration };
+}
+function parameterise(job, names) {
+    if (!job || names.size === 0)
+        return;
+    for (const task of job.tasks) {
+        const python = task.spark_python_task;
+        if (python)
+            python.parameters = [...names].map((name) => `${configurationKey(name)}=${marker(name)}`);
+    }
+}
+/** The refresh job's pause, when it differs: unset in the job and set per target, or a variable. */
+function pause(plan, job, targets) {
+    const schedule = job?.schedule;
+    if (!schedule)
+        return;
+    const how = pauseOf(plan);
+    if (how.kind === "same")
+        return;
+    if (how.kind === "variable") {
+        schedule.pause_status = marker(how.variable.name);
+        return;
+    }
+    delete schedule.pause_status;
+    for (const [target, status] of Object.entries(how.presets)) {
+        const settings = targets.settings[target];
+        settings.presets = { ...settings.presets, trigger_pause_status: status };
+    }
+}
+/** Each marker in YAML becomes `${var.<name>}`. */
+function variablesIn(node) {
+    return mapStrings(node, (text) => text.replaceAll(markers, (_, name) => `\${var.${name}}`));
+}
+/** Declares the variables the files use, sets them in each target, and writes the targets. */
+function declare(plan, files, targets) {
+    const text = JSON.stringify([Object.values(files), targets.settings]);
+    const used = new Set([...text.matchAll(/\$\{var\.([a-z0-9_]+)\}/g)].map((match) => match[1]));
+    const declared = plan.variables.list.filter((variable) => used.has(variable.name));
+    const variables = Object.fromEntries(declared.map((variable) => [
+        variable.name,
+        { description: describe(plan, variable), ...(variable.complex && { type: "complex" }) },
+    ]));
+    for (const target of plan.targets) {
+        const set = declared.filter((variable) => setsVariable(target, variable));
+        const values = Object.fromEntries(set.map((variable) => [variable.name, variable.values[target]]));
+        if (set.length > 0)
+            targets.settings[target].variables = values;
+    }
+    const bundle = files["databricks.yml"];
+    const { bundle: name, include, sync } = bundle.data;
+    bundle.data = {
+        bundle: name,
+        include,
+        sync,
+        ...(declared.length > 0 && { variables }),
+        targets: targets.settings,
+    };
+    return declared;
+}
+/** A deploy-time value's variable says which targets leave it to the pipeline. */
+function describe(plan, variable) {
+    if (!variable.deploy || variable.supplied.size === plan.targets.length)
+        return variable.description;
+    const supplied = plan.targets.filter((target) => variable.supplied.has(target));
+    return `Supplied when deploying to ${listOf(supplied)}, as BUNDLE_VAR_${variable.name} (${variable.deploy} in the definitions); the other targets set it.`;
+}
+/** Whether a target sets the variable, rather than leaving it to the deploying CI pipeline. */
+function setsVariable(target, variable) {
+    return variable.values[target] !== undefined && !variable.supplied.has(target);
+}
+/** A file run by hand cannot take a value supplied when deploying: it names the variable to fill in. */
+function placeholders(plan, target, text) {
+    return (plan.models[target].deployValues ?? []).reduce((result, value) => result.replaceAll(value.text, `<value of ${value.name}>`), text);
+}

package/dist/fold-names.d.ts

@@ -0,0 +1,16 @@
+import type { ResolvedModel } from "@ingestron/definitions";
+/** A variable's name and description, before it is made unique. */
+export interface Place {
+    name: string;
+    description: string;
+}
+/** The segments of a JSON Pointer. */
+export declare function segments(pointer: string): string[];
+/** Lower case, with `_` for anything that is not a letter, digit or `_`; list positions left out. */
+export declare function variableName(...parts: string[]): string;
+/**
+ * The variable for the value at `pointer` in the first target's model. `pipeline` is
+ * true when the value is, in every target, the catalog of the pipeline that builds
+ * the tables.
+ */
+export declare function placeOf(pointer: string, model: ResolvedModel, pipeline: boolean): Place;

package/dist/fold-names.js

@@ -0,0 +1,73 @@
+/** The segments of a JSON Pointer. */
+export function segments(pointer) {
+    return pointer
+        .split("/")
+        .slice(1)
+        .map((part) => part.replaceAll("~1", "/").replaceAll("~0", "~"));
+}
+/** Lower case, with `_` for anything that is not a letter, digit or `_`; list positions left out. */
+export function variableName(...parts) {
+    return parts
+        .filter((part) => !/^\d+$/.test(part))
+        .join("_")
+        .toLowerCase()
+        .replace(/[^a-z0-9_]+/g, "_")
+        .replace(/_+/g, "_")
+        .replace(/^_|_$/g, "");
+}
+/**
+ * The variable for the value at `pointer` in the first target's model. `pipeline` is
+ * true when the value is, in every target, the catalog of the pipeline that builds
+ * the tables.
+ */
+export function placeOf(pointer, model, pipeline) {
+    const [area, index, ...rest] = segments(pointer);
+    if (area === "tables")
+        return tablePlace(model, Number(index), rest, pipeline);
+    const path = [index ?? "", ...rest].filter(Boolean);
+    const setting = path.filter((part) => !/^\d+$/.test(part)).join(".");
+    if (area === "platform")
+        return { name: variableName(...path), description: `Platform setting ${setting}.` };
+    return {
+        name: variableName(area ?? "", ...path),
+        description: `The value at ${pointer} of the resolved definitions.`,
+    };
+}
+function tablePlace(model, index, rest, pipeline) {
+    const table = model.tables[index];
+    const [part, key, ...more] = rest;
+    if (part === "fullName")
+        return namePlace(model, table.layer, table.name, key, pipeline);
+    if (part === "ingest" && key === "connection" && more[0] === "settings") {
+        const id = table.ingest.connection.id;
+        const setting = more.slice(1);
+        const description = setting.join(".") === "path"
+            ? `Folder the files of connection "${id}" are read from.`
+            : `Setting ${setting.join(".")} of connection "${id}".`;
+        return { name: variableName(id, ...setting), description };
+    }
+    if (key === "settings" && (more[0] === "landing" || more[0] === "published") && more.length === 1)
+        return { name: more[0], description: folderOf(more[0], part) };
+    const path = [part, key, ...more].filter((item) => Boolean(item));
+    return {
+        name: variableName(table.name, ...path.filter((item) => item !== "settings")),
+        description: `The value of ${path.join(".")} for table ${table.name}.`,
+    };
+}
+function folderOf(setting, step) {
+    return setting === "landing"
+        ? `Folder the landed files are read from (${step} setting landing).`
+        : `Folder the tables are published to (${step} setting published).`;
+}
+function namePlace(model, layer, table, key, pipeline) {
+    if (key === "catalog" && pipeline)
+        return { name: "catalog", description: `Catalog of the ${model.project.id} tables.` };
+    if (key === "catalog")
+        return {
+            name: `catalog_${variableName(layer)}`,
+            description: `Catalog of the ${layer} tables.`,
+        };
+    if (key === "schema")
+        return { name: `schema_${variableName(layer)}`, description: `Schema of the ${layer} tables.` };
+    return { name: variableName("table", table), description: `Name of the ${table} table.` };
+}

package/dist/fold-plan.d.ts

@@ -0,0 +1,15 @@
+import { type ResolvedModel, type TargetDifference } from "@ingestron/definitions";
+import { type TextFold } from "./fold-text.js";
+import { type Models } from "./fold-values.js";
+/** A fold's template, variables and the problems found while planning it. */
+export interface Plan extends TextFold {
+    template: ResolvedModel;
+    /** The tags that are not the same in every target, by target. */
+    tags: Record<string, Record<string, string>>;
+    /** Values that differ but are not text or lists, left as the first target has them. */
+    unfolded: TargetDifference[];
+}
+/** The template and variables for the targets' models, given where their values differ. */
+export declare function planFold(models: Models, differing: readonly TargetDifference[]): Plan;
+/** The catalog of the pipeline that builds tables: that of its first table. */
+export declare function pipelineCatalog(model: ResolvedModel): string | undefined;

package/dist/fold-plan.js

@@ -0,0 +1,156 @@
+// Planning a fold: a template model, the first target's model with a marker
+// for each value that differs between targets or is supplied when deploying,
+// and the variables those markers stand for (spec/behaviour.md, section 2).
+import { fullNameText, splitDeployText, } from "@ingestron/definitions";
+import { placeOf, segments, variableName } from "./fold-names.js";
+import { foldCode, foldText } from "./fold-text.js";
+import { deployVariable, deployVariables, marker, placeVariable, supply, } from "./fold-values.js";
+import { differ, listOf } from "./fold-problems.js";
+import { mapStrings } from "./walk.js";
+/** Values Databricks sets per target, written in each target as they are. */
+const perTarget = /^\/(project\/environment|platform\/(mode|workspace|run_as|permissions|tags|schedule\/paused))(\/|$)/;
+/** Transform inputs are full names, folded with the tables they name. */
+const inputs = /^\/tables\/\d+\/transform\/inputs\//;
+/** The template and variables for the targets' models, given where their values differ. */
+export function planFold(models, differing) {
+    const targets = Object.keys(models);
+    const first = models[targets[0]];
+    const catalogs = targets.map((target) => pipelineCatalog(models[target]));
+    const plan = {
+        targets,
+        models,
+        variables: deployVariables(models),
+        problems: [],
+        template: structuredClone(first),
+        tags: splitTags(models, targets),
+        unfolded: [],
+    };
+    for (const difference of differing)
+        foldPlace(plan, difference, catalogs);
+    for (const [index, table] of plan.template.tables.entries())
+        if (table.transform)
+            table.transform.inputs = retarget(first, plan.template, index);
+    plan.template.platform.tags = commonTags(first, plan.tags);
+    markDeployValues(plan);
+    return plan;
+}
+/** The catalog of the pipeline that builds tables: that of its first table. */
+export function pipelineCatalog(model) {
+    return (model.tables.find((table) => table.store) ?? model.tables[0])?.fullName.catalog;
+}
+function foldPlace(plan, difference, catalogs) {
+    const { pointer, values } = difference;
+    if (perTarget.test(pointer) || inputs.test(pointer))
+        return;
+    const path = segments(pointer);
+    if (path[0] === "names")
+        return plan.problems.push(renamed(plan, path[1], values));
+    if (!plan.targets.every((target) => values[target] !== undefined))
+        return plan.unfolded.push(difference);
+    if (path[2] === "transform" && path[3] === "code" && path.length === 4)
+        return foldTableCode(plan, Number(path[1]), values);
+    const pipeline = JSON.stringify(Object.values(values)) === JSON.stringify(catalogs);
+    if (!foldValue(plan, pointer, values, pipeline))
+        plan.unfolded.push(difference);
+}
+/**
+ * A text becomes a variable, or a variable for its deploy-time part; a list whose
+ * length differs becomes a complex variable. Other values stay as the first target
+ * has them, and the check against each target's own bundle finds what that changes.
+ */
+function foldValue(plan, pointer, values, pipeline) {
+    const all = Object.values(values);
+    const first = plan.models[plan.targets[0]];
+    const path = segments(pointer);
+    if (all.every((value) => typeof value === "string")) {
+        const place = placeOf(pointer, first, pipeline);
+        setAt(plan.template, path, foldText(plan, values, place));
+        return true;
+    }
+    if (all.every((value) => Array.isArray(value)) && !JSON.stringify(all).includes("${envvar:")) {
+        const place = { ...placeOf(pointer, first, false), complex: true };
+        setAt(plan.template, path, marker(placeVariable(plan.variables, place, values).name));
+        return true;
+    }
+    return false;
+}
+function renamed(plan, what, values) {
+    const [first, other] = plan.targets.filter((target) => values[target] !== undefined);
+    return differ(`The ${what} is named "${String(values[first])}" in ${first} and "${String(values[other])}" in ${other}; resource keys and folders are named after it, so it must be the same in every environment.`, "project.yaml");
+}
+function foldTableCode(plan, index, values) {
+    const table = plan.template.tables[index];
+    const code = foldCode(plan, values, (count) => ({
+        name: variableName(table.name, "value"),
+        description: `Part ${count} of the code of table ${table.name} that differs between environments.`,
+    }));
+    if (code === undefined) {
+        const message = `The code of table "${table.name}" differs between ${listOf(plan.targets)} in more than values.`;
+        plan.problems.push(differ(message, table.origin.file));
+    }
+    else
+        table.transform.code = code;
+}
+/** Sets the value at a path of segments. */
+function setAt(model, path, value) {
+    let node = model;
+    for (const key of path.slice(0, -1))
+        node = node[key];
+    node[path[path.length - 1]] = value;
+}
+/** A transform's inputs, named as the template names the tables they are. */
+function retarget(first, template, index) {
+    const entries = Object.entries(first.tables[index].transform.inputs).map(([name, full]) => {
+        const input = first.tables.findIndex((table) => fullNameText(table.fullName) === full);
+        return [name, input < 0 ? full : fullNameText(template.tables[input].fullName)];
+    });
+    return Object.fromEntries(entries);
+}
+/** The tags each target has that are not the same in every target. */
+function splitTags(models, targets) {
+    const keys = [
+        ...new Set(targets.flatMap((target) => Object.keys(models[target].platform.tags))),
+    ];
+    const same = (key) => new Set(targets.map((target) => models[target].platform.tags[key])).size === 1;
+    return Object.fromEntries(targets.map((target) => {
+        const tags = models[target].platform.tags;
+        return [
+            target,
+            Object.fromEntries(Object.entries(tags).filter(([key]) => !same(key) && keys.includes(key))),
+        ];
+    }));
+}
+function commonTags(first, differing) {
+    const other = new Set(Object.values(differing).flatMap((tags) => Object.keys(tags)));
+    return Object.fromEntries(Object.entries(first.platform.tags).filter(([key]) => !other.has(key)));
+}
+/** Values Databricks takes per target, and the model's list of deploy-time values. */
+const notWalked = new Set([
+    "/platform/workspace",
+    "/platform/run_as",
+    "/platform/permissions",
+    "/deployValues",
+]);
+/** Each deploy-time value left in the template: the same in every target, so each supplies it. */
+function markDeployValues(plan) {
+    const values = [
+        ...new Map(Object.values(plan.models).flatMap((model) => (model.deployValues ?? []).map((value) => [value.name, value]))).values(),
+    ];
+    const mark = (text) => marked(plan, text, values);
+    plan.template = mapStrings(plan.template, mark, (pointer) => notWalked.has(pointer));
+}
+function marked(plan, text, values) {
+    const parts = splitDeployText(text, values);
+    if (parts.every((part) => typeof part === "string"))
+        return text;
+    return parts
+        .map((part) => {
+        if (typeof part === "string")
+            return part;
+        const variable = deployVariable(plan.variables, part.name);
+        for (const target of plan.targets)
+            plan.problems.push(...supply(variable, target, plan.models[target]));
+        return marker(variable.name);
+    })
+        .join("");
+}

package/dist/fold-problems.d.ts

@@ -0,0 +1,29 @@
+import type { EnvironmentsProblem, ResolvedModel, TargetDifference } from "@ingestron/definitions";
+import { type Models, type Variables } from "./fold-values.js";
+/** An `ENVIRONMENTS_DIFFER` error at a definition. */
+export declare function differ(message: string, file: string, hint?: string): EnvironmentsProblem;
+/** Target names joined with `, `, the last two with ` and `. */
+export declare function listOf(targets: string[]): string;
+/** Each problem once. */
+export declare function once(problems: EnvironmentsProblem[]): EnvironmentsProblem[];
+/**
+ * A table the pipeline names `schema.table` in some targets, because it is in the
+ * pipeline's catalog there, and with three parts in others.
+ */
+export declare function catalogProblems(models: Models, catalog: (model: ResolvedModel) => string | undefined): EnvironmentsProblem[];
+/** A Kafka connection read continuously in some targets and by the refresh job in others. */
+export declare function continuousProblems(models: Models): EnvironmentsProblem[];
+/**
+ * A PySpark transform whose own code would need values that differ between targets, or
+ * that are supplied when deploying: the pack copies that code as it is.
+ */
+export declare function codeProblem(fold: {
+    variables: Variables;
+    models: Models;
+    targets: string[];
+}, path: string, names: string[], file: string): EnvironmentsProblem;
+/**
+ * A file whose view in some targets is not that target's own file. When values that are
+ * not text differ, the message names each, with its value in every target.
+ */
+export declare function unlikeProblem(models: Models, path: string, targets: string[], unfolded: TargetDifference[]): EnvironmentsProblem;

package/dist/fold-problems.js

@@ -0,0 +1,113 @@
+import { placeOf } from "./fold-names.js";
+import { connectGroups } from "./connect.js";
+import { deployProblem } from "./fold-values.js";
+/** An `ENVIRONMENTS_DIFFER` error at a definition. */
+export function differ(message, file, hint) {
+    return {
+        code: "ENVIRONMENTS_DIFFER",
+        severity: "error",
+        message,
+        file,
+        ...(hint && { hint }),
+    };
+}
+/** Target names joined with `, `, the last two with ` and `. */
+export function listOf(targets) {
+    if (targets.length < 2)
+        return targets.join("");
+    return `${targets.slice(0, -1).join(", ")} and ${targets[targets.length - 1]}`;
+}
+/** Each problem once. */
+export function once(problems) {
+    const seen = new Set();
+    return problems.filter((problem) => {
+        const key = JSON.stringify([problem.code, problem.message, problem.file, problem.line]);
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+}
+/** The targets in which `test` holds, and those in which it does not. */
+function split(targets, test) {
+    const yes = targets.filter(test);
+    return [yes, targets.filter((target) => !yes.includes(target))];
+}
+/**
+ * A table the pipeline names `schema.table` in some targets, because it is in the
+ * pipeline's catalog there, and with three parts in others.
+ */
+export function catalogProblems(models, catalog) {
+    const targets = Object.keys(models);
+    const first = models[targets[0]];
+    return first.tables
+        .filter((table) => table.store)
+        .flatMap((table) => {
+        const inPipeline = (target) => {
+            const model = models[target];
+            const same = model.tables.find((other) => other.name === table.name);
+            return same?.fullName.catalog === catalog(model);
+        };
+        const [yes, no] = split(targets, inPipeline);
+        if (yes.length === 0 || no.length === 0)
+            return [];
+        return [
+            differ(`Table "${table.name}" is in the pipeline's catalog in ${listOf(yes)} but not in ${listOf(no)}, so the pipeline's code cannot name it the same way in every environment.`, table.origin.file, "Put the table in the pipeline's catalog in every environment, or in another catalog in every environment."),
+        ];
+    });
+}
+/** A Kafka connection read continuously in some targets and by the refresh job in others. */
+export function continuousProblems(models) {
+    const targets = Object.keys(models);
+    const groups = targets.map((target) => connectGroups(models[target]));
+    return groups[0].flatMap((group) => {
+        const continuous = (target) => groups[targets.indexOf(target)].find((other) => other.connection.id === group.connection.id)
+            ?.continuous === true;
+        const [yes, no] = split(targets, continuous);
+        if (yes.length === 0 || no.length === 0)
+            return [];
+        return [
+            differ(`Connection "${group.connection.id}" is read continuously in ${listOf(yes)} and by the refresh job in ${listOf(no)}.`, group.tables[0].origin.file, "Set continuous in the flow's ingest settings, so that it is the same in every environment."),
+        ];
+    });
+}
+/**
+ * A PySpark transform whose own code would need values that differ between targets, or
+ * that are supplied when deploying: the pack copies that code as it is.
+ */
+export function codeProblem(fold, path, names, file) {
+    const table = path.replace(/^.*\//, "").replace(/\.py$/, "");
+    const variables = fold.variables.list.filter((variable) => names.includes(variable.name));
+    const deploy = variables.find((variable) => variable.deploy);
+    if (deploy)
+        return deployProblem(`The code of PySpark transform "${table}" would need \${envvar:${deploy.deploy}}, which is supplied when deploying; a PySpark transform's own code cannot take it yet.`, fold.models[[...deploy.supplied][0] ?? fold.targets[0]], deploy.deploy);
+    return differ(`The code of PySpark transform "${table}" would need values that differ between environments (${variables.map((variable) => variable.name).join(", ")}); a PySpark transform's own code cannot take them yet.`, file, "Make these values the same in every environment, or write the transform in SQL.");
+}
+/**
+ * A file whose view in some targets is not that target's own file. When values that are
+ * not text differ, the message names each, with its value in every target.
+ */
+export function unlikeProblem(models, path, targets, unfolded) {
+    const start = `${path} cannot be generated once for every environment: in ${listOf(targets)} it would not be what that environment gets on its own`;
+    if (unfolded.length === 0)
+        return differ(`${start}, because the environments differ there in more than values.`, "project.yaml");
+    const first = Object.values(models)[0];
+    const settings = unfolded.map((difference) => settingOf(difference.pointer, first));
+    const values = unfolded.map((difference, index) => `${settings[index]} is ${valuesOf(models, difference.values)}`);
+    return differ(`${start}, because a value that is not text differs between environments: ${values.join("; ")}.`, "project.yaml", `Make ${listOf(settings)} the same in every environment; only text values, such as names, paths and hosts, and lists can differ.`);
+}
+/** What a place in the model is, in words, such as "platform setting manage.schemas". */
+function settingOf(pointer, model) {
+    const { description } = placeOf(pointer, model, false);
+    return `${description.charAt(0).toLowerCase()}${description.slice(1)}`.replace(/\.$/, "");
+}
+/** Each value with the targets that have it, such as "false in dev, true in test and prod". */
+function valuesOf(models, values) {
+    const groups = new Map();
+    for (const target of Object.keys(models)) {
+        const value = values[target];
+        const text = value === undefined ? "not set" : JSON.stringify(value);
+        groups.set(text, [...(groups.get(text) ?? []), target]);
+    }
+    return [...groups].map(([value, targets]) => `${value} in ${listOf(targets)}`).join(", ");
+}

package/dist/fold-python.d.ts

@@ -0,0 +1,17 @@
+/** The configuration key a variable is read under. */
+export declare function configurationKey(name: string): string;
+/**
+ * The file with each marker inside a double-quoted text read from the configuration:
+ * before the table's function, `"⟦schema_silver⟧.totals"` becomes
+ * `f"{ingestron_schema_silver}.totals"` with
+ * `ingestron_schema_silver = spark.conf.get("ingestron.schema_silver")` above the
+ * decorators; inside it, `spark.conf.get("ingestron.schema_bronze") + ".orders"`.
+ * Undefined when a marker is anywhere else, which no Python expression can stand for.
+ */
+export declare function pythonValues(text: string): string | undefined;
+/**
+ * The file as the pipeline runs it, as the pack would have written it with the values:
+ * the module-level reads removed and each f-string that uses them a plain text, then
+ * each spark.conf.get of an `ingestron.*` key in the body joined into the texts around it.
+ */
+export declare function pythonWithValues(text: string, values: Record<string, string>): string;