@ingestron/pack-databricks 1.0.1 → 2.0.0

package/dist/python-data.js

@@ -1,20 +1,18 @@
-// Template data for a PySpark transform's Python file. Python string and
-// dict literals are written as JSON, which Python reads the same way.
-import { fullNameText } from "@ingestron/definitions";
+import { localName, localText } from "./sql-data.js";
 const decorators = {
     warn: "expect_all",
     drop: "expect_all_or_drop",
     fail: "expect_all_or_fail",
 };
-/** Everything materialised-view.py.liquid needs. */
-export function pythonData(table, header) {
+/** Everything materialised-view.py.liquid needs, in the pipeline whose catalog is `catalog`. */
+export function pythonData(table, header, catalog) {
     const store = table.store?.settings ?? {};
     const transform = table.transform;
     const properties = (store.properties ?? {});
     return {
         header,
         code: transform.code.trimEnd(),
-        name: JSON.stringify(fullNameText(table.fullName)),
+        name: JSON.stringify(localName(table.fullName, catalog)),
         comment: table.contract.description ? JSON.stringify(table.contract.description) : "",
         cluster: clusterOf(store.cluster_by),
         properties: Object.keys(properties).length > 0 ? pythonDict(Object.entries(properties)) : "",
@@ -32,7 +30,7 @@ export function pythonData(table, header) {
         function: String(transform.settings.function),
         inputs: Object.entries(transform.inputs).map(([name, full]) => ({
             name: JSON.stringify(name),
-            table: JSON.stringify(full),
+            table: JSON.stringify(localText(full, catalog)),
         })),
     };
 }

package/dist/sql-data.d.ts

@@ -1,4 +1,4 @@
-import { type ResolvedTable } from "@ingestron/definitions";
+import type { FullName, ResolvedTable } from "@ingestron/definitions";
 /** The table definition shared by every SQL file. */
 export interface TableData {
     kind: "STREAMING TABLE" | "MATERIALIZED VIEW";
@@ -28,12 +28,19 @@ export interface SourceData {
 }
 /** A SQL string literal. */
 export declare function sqlString(text: string): string;
-/** The definition of the table itself. */
-export declare function tableData(table: ResolvedTable, kind: TableData["kind"]): TableData;
+/**
+ * A table's name in a pipeline's code: `schema.table` when the pipeline's own
+ * catalog holds it, as the pipeline reads names in its catalog; else all three parts.
+ */
+export declare function localName(name: FullName, catalog: string): string;
+/** A full name written `catalog.schema.table`, as transform inputs are, in a pipeline's code. */
+export declare function localText(fullName: string, catalog: string): string;
+/** The definition of the table itself, in the pipeline whose catalog is `catalog`. */
+export declare function tableData(table: ResolvedTable, kind: TableData["kind"], catalog: string): TableData;
 /** The `read_files` call for a flow table's files. */
 export declare function sourceData(table: ResolvedTable, stream: boolean): SourceData;
-/** The AUTO CDC parts of a change-data table. */
-export declare function changeData(table: ResolvedTable): {
+/** The AUTO CDC parts of a change-data table, in the pipeline whose catalog is `catalog`. */
+export declare function changeData(table: ResolvedTable, catalog: string): {
     changes: string;
     flow: string;
     keys: string;

package/dist/sql-data.js

@@ -1,19 +1,28 @@
-// Template data for a table's SQL file. All quoting happens here, so the
-// templates only arrange text. Key names avoid Liquid keywords (such as
-// `required` or `with`) because python-liquid refuses them after a dot.
-import { fullNameText } from "@ingestron/definitions";
 import { quoteName } from "./query.js";
 const onViolation = { warn: "", drop: " ON VIOLATION DROP ROW", fail: " ON VIOLATION FAIL UPDATE" };
 /** A SQL string literal. */
 export function sqlString(text) {
     return `'${text.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}'`;
 }
-/** The definition of the table itself. */
-export function tableData(table, kind) {
+/**
+ * A table's name in a pipeline's code: `schema.table` when the pipeline's own
+ * catalog holds it, as the pipeline reads names in its catalog; else all three parts.
+ */
+export function localName(name, catalog) {
+    return name.catalog === catalog
+        ? `${name.schema}.${name.table}`
+        : `${name.catalog}.${name.schema}.${name.table}`;
+}
+/** A full name written `catalog.schema.table`, as transform inputs are, in a pipeline's code. */
+export function localText(fullName, catalog) {
+    return fullName.startsWith(`${catalog}.`) ? fullName.slice(catalog.length + 1) : fullName;
+}
+/** The definition of the table itself, in the pipeline whose catalog is `catalog`. */
+export function tableData(table, kind, catalog) {
     const settings = table.store?.settings ?? {};
     return {
         kind,
-        target: quoteName(fullNameText(table.fullName)),
+        target: quoteName(localName(table.fullName, catalog)),
         constraints: table.expectations.map((item) => ({
             name: quoteName(item.name),
             check: item.check,
@@ -46,13 +55,31 @@ function renamedColumns(table) {
         return [];
     return table.contract.columns.filter((column) => column.physicalName && column.physicalName !== column.name);
 }
-/** `*`, or `* EXCEPT (<source names>), <source> AS <name>, …` for renamed columns. */
+/**
+ * Timestamps the landed files carry beyond the contract (such as ADF's audit
+ * columns), which the step that adds them writes as ISO 8601 text.
+ */
+function textTimestamps(table) {
+    const extra = table.ingest?.output?.columns ?? [];
+    return extra.filter((column) => column.logicalType === "timestamp").map((column) => column.name);
+}
+/**
+ * `*`, or `* EXCEPT (<source names>), <source> AS <name>, …` for renamed
+ * columns, and `CAST(<name> AS TIMESTAMP) AS <name>` for timestamps written as text.
+ */
 function selectOf(table) {
     const renamed = renamedColumns(table);
-    if (renamed.length === 0)
+    const timestamps = textTimestamps(table);
+    if (renamed.length === 0 && timestamps.length === 0)
         return "*";
-    const sources = renamed.map((column) => quoteName(column.physicalName));
-    const aliases = renamed.map((column) => `${quoteName(column.physicalName)} AS ${quoteName(column.name)}`);
+    const sources = [
+        ...renamed.map((column) => quoteName(column.physicalName)),
+        ...timestamps.map(quoteName),
+    ];
+    const aliases = [
+        ...renamed.map((column) => `${quoteName(column.physicalName)} AS ${quoteName(column.name)}`),
+        ...timestamps.map((name) => `CAST(${quoteName(name)} AS TIMESTAMP) AS ${quoteName(name)}`),
+    ];
     return `* EXCEPT (${sources.join(", ")}), ${aliases.join(", ")}`;
 }
 /** The connection's `path` joined with the table's object. */
@@ -90,8 +117,8 @@ function optionsOf(table) {
     }
     return options;
 }
-/** The AUTO CDC parts of a change-data table. */
-export function changeData(table) {
+/** The AUTO CDC parts of a change-data table, in the pipeline whose catalog is `catalog`. */
+export function changeData(table, catalog) {
     const settings = table.store?.settings ?? {};
     const changes = {
         ...table.fullName,
@@ -99,7 +126,7 @@ export function changeData(table) {
     };
     const except = (settings.except_columns ?? []);
     return {
-        changes: quoteName(fullNameText(changes)),
+        changes: quoteName(localName(changes, catalog)),
         flow: quoteName(`${table.fullName.table}_cdc`),
         keys: table.contract.primaryKey.map(quoteName).join(", "),
         sequence: quoteName(String(settings.sequence_by)),

package/dist/sql.d.ts

@@ -7,8 +7,6 @@ export interface SqlFile {
     data: object;
     extension: "sql" | "py";
 }
-/** The SQL or Python file for one table; `header` is its first comment line. */
-export declare function tableSql(table: ResolvedTable, header: string): string;
 /** Renders a file chosen by `sqlFile`. */
 export declare function renderSql({ template, data }: SqlFile): string;
 /** The after-refresh Python file, with its statements and checks as JSON. */
@@ -17,5 +15,8 @@ export declare function renderApply(data: {
     statements: string[];
     checks: object[];
 }): string;
-/** Chooses the template by transform and store method, and builds its data. */
-export declare function sqlFile(table: ResolvedTable, header: string): SqlFile;
+/**
+ * Chooses the template by transform and store method, and builds its data. `catalog`
+ * is the pipeline's catalog, whose tables the code names with two parts.
+ */
+export declare function sqlFile(table: ResolvedTable, header: string, catalog: string): SqlFile;

package/dist/sql.js

@@ -4,7 +4,7 @@ import { fileURLToPath } from "node:url";
 import { Liquid } from "liquidjs";
 import { renderQuery } from "./query.js";
 import { pythonData } from "./python-data.js";
-import { changeData, sourceData, tableData } from "./sql-data.js";
+import { changeData, localText, sourceData, tableData } from "./sql-data.js";
 /** The templates folder, also used by the template parity check. */
 export const templatesFolder = join(dirname(fileURLToPath(import.meta.url)), "../templates/");
 const engine = new Liquid({
@@ -13,10 +13,6 @@ const engine = new Liquid({
     strictVariables: true,
     strictFilters: true,
 });
-/** The SQL or Python file for one table; `header` is its first comment line. */
-export function tableSql(table, header) {
-    return renderSql(sqlFile(table, header));
-}
 /** Renders a file chosen by `sqlFile`. */
 export function renderSql({ template, data }) {
     return `${engine.renderFileSync(template, data).trimEnd()}\n`;
@@ -33,39 +29,43 @@ export function renderApply(data) {
         extension: "py",
     });
 }
-/** Chooses the template by transform and store method, and builds its data. */
-export function sqlFile(table, header) {
+/**
+ * Chooses the template by transform and store method, and builds its data. `catalog`
+ * is the pipeline's catalog, whose tables the code names with two parts.
+ */
+export function sqlFile(table, header, catalog) {
     if (table.transform?.method === "pyspark") {
         return {
             template: "materialised-view.py.liquid",
-            data: pythonData(table, header),
+            data: pythonData(table, header, catalog),
             extension: "py",
         };
     }
     const store = table.store?.method;
     if (store === "streaming-table")
-        return streamingTable(table, header);
+        return streamingTable(table, header, catalog);
     if (store === "materialised-view")
-        return materialisedView(table, header);
+        return materialisedView(table, header, catalog);
     throw new Error(`No SQL for store method "${String(store)}" (table ${table.name}).`);
 }
-function streamingTable(table, header) {
+function streamingTable(table, header, catalog) {
     const data = {
         header,
-        table: tableData(table, "STREAMING TABLE"),
+        table: tableData(table, "STREAMING TABLE", catalog),
         source: sourceData(table, true),
     };
     if (table.store?.settings.mode === "append")
         return { template: "streaming-table.sql.liquid", data, extension: "sql" };
     return {
         template: "change-data.sql.liquid",
-        data: { ...data, cdc: changeData(table) },
+        data: { ...data, cdc: changeData(table, catalog) },
         extension: "sql",
     };
 }
-function materialisedView(table, header) {
+function materialisedView(table, header, catalog) {
+    const inputs = Object.entries(table.transform?.inputs ?? {}).map(([name, full]) => [name, localText(full, catalog)]);
     const rendered = table.transform
-        ? renderQuery(table.transform.code, table.transform.inputs)
+        ? renderQuery(table.transform.code, Object.fromEntries(inputs))
         : { sql: "" };
     // The sql method's check has already reported SQL that cannot be rendered.
     const query = "sql" in rendered ? rendered.sql : "";
@@ -74,7 +74,7 @@ function materialisedView(table, header) {
         : { stream: false, select: "*", path: "", options: [] };
     return {
         template: "materialised-view.sql.liquid",
-        data: { header, table: tableData(table, "MATERIALIZED VIEW"), source, query },
+        data: { header, table: tableData(table, "MATERIALIZED VIEW", catalog), source, query },
         extension: "sql",
     };
 }

package/dist/view.d.ts

@@ -0,0 +1,16 @@
+import { type ResolvedModel, type SuppliedValue } from "@ingestron/definitions";
+/** What the deploying pipeline supplies for one target, as the deploy-time values' text. */
+export interface Supplied {
+    /** Each variable the pipeline supplies as BUNDLE_VAR_<name>, by name. */
+    variables: Record<string, string>;
+    /** The host, when the pipeline supplies it as DATABRICKS_HOST. */
+    host?: string;
+    /** Each deploy-time value's text, by NAME. */
+    values: Record<string, string>;
+}
+/** What the pipeline supplies for a target, from the fold's deploy-time values. */
+export declare function suppliedFor(model: ResolvedModel, deployValues: readonly SuppliedValue[]): Supplied;
+/** Whether two texts at one path are the same: YAML as data, other files as text. */
+export declare function sameFile(path: string, viewed: string, rendered: string): boolean;
+/** The folded files as Databricks deploys them for `target`, laid out as `build` lays out one target. */
+export declare function bundleView(files: Readonly<Record<string, string>>, target: string, supplied: Supplied): Record<string, string>;

package/dist/view.js

@@ -0,0 +1,173 @@
+// What Databricks makes of the folded bundle for one target, in the layout of
+// that target's own bundle (spec/behaviour.md, section 4): the fold property
+// compares it with what `build` gives for the target alone.
+import { isDeepStrictEqual } from "node:util";
+import { splitDeployText } from "@ingestron/definitions";
+import { parse, stringify } from "yaml";
+import { pythonWithValues } from "./fold-python.js";
+import { mapStrings } from "./walk.js";
+/** What the pipeline supplies for a target, from the fold's deploy-time values. */
+export function suppliedFor(model, deployValues) {
+    const prefix = "BUNDLE_VAR_";
+    const variables = Object.fromEntries(deployValues.flatMap((value) => value.suppliedAs
+        .filter((name) => name.startsWith(prefix))
+        .map((name) => [name.slice(prefix.length), `\${envvar:${value.name}}`])));
+    const host = model.platform.workspace.host ?? "";
+    const parts = splitDeployText(host, model.deployValues ?? []);
+    const whole = parts.length === 1 && typeof parts[0] !== "string";
+    const values = Object.fromEntries((model.deployValues ?? []).map((value) => [value.name, value.text]));
+    return { variables, ...(whole && { host }), values };
+}
+/** Whether two texts at one path are the same: YAML as data, other files as text. */
+export function sameFile(path, viewed, rendered) {
+    return path.endsWith(".yml")
+        ? isDeepStrictEqual(parse(viewed), parse(rendered))
+        : viewed === rendered;
+}
+/** The folded files as Databricks deploys them for `target`, laid out as `build` lays out one target. */
+export function bundleView(files, target, supplied) {
+    const bundle = parse(files["databricks.yml"]);
+    const settings = bundle.targets[target];
+    const values = { ...supplied.variables, ...settings.variables };
+    const resolve = (node) => resolved(node, values);
+    const presets = resolve(settings.presets ?? {});
+    const development = settings.mode === "development";
+    const code = {};
+    const view = {};
+    let continuous = false;
+    for (const [path, text] of Object.entries(files)) {
+        if (path.startsWith("resources/")) {
+            const data = resolve(parse(text));
+            continuous ||= hasContinuous(data);
+            applyPresets(data, presets, development);
+            takeValues(data, code);
+            view[path] = stringify(data);
+        }
+        else if (path === `prerequisites/${target}.sql`)
+            view[path] = deployTexts(text, supplied);
+        else if (!path.startsWith("prerequisites/"))
+            view[path] = text;
+    }
+    const running = development && continuous && presets.trigger_pause_status === "UNPAUSED";
+    view["databricks.yml"] = stringify(resolve(bundleOf(bundle, target, supplied, running)));
+    for (const [path, text] of Object.entries(view))
+        view[path] = withValues(path, text, code);
+    return view;
+}
+/** True when a resource file has a pipeline that runs continuously. */
+function hasContinuous(data) {
+    const pipelines = (data.resources?.pipelines ?? {});
+    return Object.values(pipelines).some((pipeline) => pipeline.continuous === true);
+}
+/**
+ * The target alone. Development mode stops continuous pipelines unless an UNPAUSED preset
+ * replaces its own PAUSED one; no resource setting can say that, so such a preset stays.
+ */
+function bundleOf(bundle, target, supplied, running) {
+    const settings = { ...bundle.targets[target] };
+    delete settings.variables;
+    delete settings.presets;
+    if (running)
+        settings.presets = { trigger_pause_status: "UNPAUSED" };
+    const workspace = settings.workspace;
+    if (supplied.host && !workspace?.host)
+        settings.workspace = { ...workspace, host: supplied.host };
+    const { bundle: name, include, sync } = bundle;
+    return { bundle: name, include, sync, targets: { [target]: { ...settings, default: true } } };
+}
+/** Each `${var.<name>}` replaced: a whole value by the variable's value, a part by its text. */
+function resolved(node, values) {
+    return mapStrings(node, (text) => {
+        const whole = /^\$\{var\.([a-z0-9_]+)\}$/.exec(text);
+        if (whole && values[whole[1]] !== undefined)
+            return values[whole[1]];
+        return text.replace(/\$\{var\.([a-z0-9_]+)\}/g, (found, name) => values[name] === undefined ? found : String(values[name]));
+    });
+}
+/**
+ * Presets as Databricks applies them: tags that a resource does not set, the pause of
+ * schedules that do not set one (development mode pauses them), and no continuous
+ * pipelines under a PAUSED preset the target writes. Development mode's own default
+ * PAUSED also stops continuous pipelines, but it does so in the target's own bundle
+ * too, which keeps `continuous: true` in its files; so the view keeps it as well, and
+ * marks only an UNPAUSED preset that lifts that default (see `bundleOf`).
+ */
+function applyPresets(data, presets, development) {
+    const resources = { jobs: {}, pipelines: {}, ...data.resources };
+    const jobs = Object.values(resources.jobs);
+    const pipelines = Object.values(resources.pipelines);
+    const tags = { ...presets.tags };
+    if (Object.keys(tags).length > 0)
+        for (const resource of [...jobs, ...pipelines])
+            resource.tags = { ...tags, ...dropEmpty(resource.tags) };
+    const pause = presets.trigger_pause_status;
+    if (pause === "PAUSED")
+        for (const pipeline of pipelines)
+            delete pipeline.continuous;
+    const status = pause ?? (development ? "PAUSED" : "UNPAUSED");
+    for (const job of jobs)
+        pauseSchedule(job, status);
+}
+function pauseSchedule(job, status) {
+    const schedule = job.schedule;
+    if (schedule && !schedule.pause_status)
+        schedule.pause_status = status;
+}
+function dropEmpty(tags) {
+    return Object.fromEntries(Object.entries((tags ?? {})).filter(([, value]) => value !== ""));
+}
+/** Takes the pipelines' `ingestron.*` configuration and the job's `ingestron.*` parameters out, by folder or file. */
+function takeValues(data, code) {
+    const resources = (data.resources ?? {});
+    for (const pipeline of Object.values(resources.pipelines ?? {}))
+        takeConfiguration(pipeline, code);
+    for (const job of Object.values(resources.jobs ?? {}))
+        for (const task of (job.tasks ?? []))
+            takeParameters(task, code);
+}
+function takeConfiguration(pipeline, code) {
+    const configuration = pipeline.configuration;
+    if (!configuration)
+        return;
+    const folder = `${String(pipeline.root_path ?? "").replace(/^\.\.\//, "")}/`;
+    for (const [key, value] of Object.entries(configuration)) {
+        if (!key.startsWith("ingestron."))
+            continue;
+        code[folder] = { ...code[folder], [key]: value };
+        delete configuration[key];
+    }
+    if (Object.keys(configuration).length === 0)
+        delete pipeline.configuration;
+}
+function takeParameters(task, code) {
+    const python = task.spark_python_task;
+    const parameters = (python?.parameters ?? []);
+    if (!python || parameters.length === 0)
+        return;
+    const file = String(python.python_file).replace(/^\.\.\//, "");
+    for (const parameter of parameters) {
+        const [key, value] = parameter.split(/=(.*)/s);
+        code[file] = { ...code[file], [key]: value };
+    }
+    delete python.parameters;
+}
+/** Pipeline SQL with its folder's configuration values; the script with its parameters. */
+function withValues(path, text, code) {
+    return Object.entries(code).reduce((result, [at, values]) => {
+        if (path === at)
+            return replaced(result, values, (value) => JSON.stringify(value).slice(1, -1));
+        if (!at.endsWith("/") || !path.startsWith(at))
+            return result;
+        if (path.endsWith(".sql"))
+            return replaced(result, values, (value) => value);
+        return path.endsWith(".py") ? pythonWithValues(result, values) : result;
+    }, text);
+}
+/** Each `${<key>}` in a text as its value, written as `write` gives it. */
+function replaced(text, values, write) {
+    return Object.entries(values).reduce((result, [key, value]) => result.replaceAll(`\${${key}}`, () => write(value)), text);
+}
+/** The grants file's placeholders, as the deploy-time values they stand for. */
+function deployTexts(text, supplied) {
+    return Object.entries(supplied.values).reduce((result, [name, value]) => result.replaceAll(`<value of ${name}>`, () => value), text);
+}

package/dist/walk.d.ts

@@ -0,0 +1,7 @@
+/**
+ * A copy of `node` with each text replaced by what `map` gives for it. Subtrees whose
+ * pointer `skip` names are copied as they are.
+ * @example
+ * mapStrings({ a: ["x"] }, (text) => text.toUpperCase()); // { a: ["X"] }
+ */
+export declare function mapStrings(node: unknown, map: (text: string, pointer: string) => unknown, skip?: (pointer: string) => boolean, pointer?: string): unknown;

package/dist/walk.js

@@ -0,0 +1,22 @@
+// One tree walk for plain data: every text in a value, mapped, with its JSON
+// Pointer. Folding and the view both rewrite texts this way.
+/**
+ * A copy of `node` with each text replaced by what `map` gives for it. Subtrees whose
+ * pointer `skip` names are copied as they are.
+ * @example
+ * mapStrings({ a: ["x"] }, (text) => text.toUpperCase()); // { a: ["X"] }
+ */
+export function mapStrings(node, map, skip = () => false, pointer = "") {
+    if (skip(pointer))
+        return node;
+    if (typeof node === "string")
+        return map(node, pointer);
+    if (Array.isArray(node))
+        return node.map((item, index) => mapStrings(item, map, skip, `${pointer}/${index}`));
+    if (node === null || typeof node !== "object")
+        return node;
+    return Object.fromEntries(Object.entries(node).map(([key, value]) => [
+        key,
+        mapStrings(value, map, skip, `${pointer}/${key.replaceAll("~", "~0").replaceAll("/", "~1")}`),
+    ]));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ingestron/pack-databricks",
-  "version": "1.0.1",
+  "version": "2.0.0",
   "description": "Databricks methods for Ingestron: generates a bundle with a Lakeflow Spark Declarative Pipeline.",
   "license": "Apache-2.0",
   "author": "IntentLabs <hello@ingestron.io>",
@@ -34,11 +34,11 @@
     "yaml": "2.9.1"
   },
   "peerDependencies": {
-    "@ingestron/definitions": "^1.0.0"
+    "@ingestron/definitions": "^1.9.0"
   },
   "devDependencies": {
-    "@ingestron/config": "^1.0.0",
-    "@ingestron/definitions": "^1.0.0",
+    "@ingestron/config": "^1.5.0",
+    "@ingestron/definitions": "^1.9.0",
     "@microsoft/api-extractor": "7.59.4",
     "@types/node": "22.20.5",
     "ajv": "8.20.0",

package/templates/apply-contract.py.liquid

@@ -2,11 +2,22 @@
 # Runs after each refresh: Lakeflow pipelines cannot set tags or column
 # comments, nor compare counts over all rows, so this task does.
 import json
+import sys
 
 from pyspark.sql import SparkSession
 
 STATEMENTS = json.loads(r"""{{ statements }}""")
 CHECKS = json.loads(r"""{{ checks }}""")
+# Values that differ between environments: the job passes each as a parameter
+# ingestron.<name>=<value>, and ${ingestron.<name>} above stands for it.
+VALUES = dict(argument.split("=", 1) for argument in sys.argv[1:] if "=" in argument)
+
+
+def with_values(text):
+    """The text with each ${ingestron.<name>} replaced by its value."""
+    for name, value in VALUES.items():
+        text = text.replace("${" + name + "}", value)
+    return text
 
 
 def passes(operator, expected, measured):
@@ -30,7 +41,7 @@ def passes(operator, expected, measured):
 
 def measure(spark, check):
     """The rule's count, in rows or as a percentage of all rows."""
-    row = spark.sql(check["query"]).first()
+    row = spark.sql(with_values(check["query"])).first()
     counted, total = row["counted"] or 0, row["total"] or 0
     if check["unit"] == "percent":
         return 0 if total == 0 else counted * 100 / total
@@ -40,13 +51,13 @@ def measure(spark, check):
 def main():
     spark = SparkSession.builder.getOrCreate()
     for statement in STATEMENTS:
-        spark.sql(statement)
+        spark.sql(with_values(statement))
     failures = []
     for check in CHECKS:
         measured = measure(spark, check)
         if passes(check["operator"], check["value"], measured):
             continue
-        message = f'{check["table"]}: {check["name"]} measured {measured}, expected {check["operator"]} {check["value"]}'
+        message = f'{with_values(check["table"])}: {check["name"]} measured {measured}, expected {check["operator"]} {check["value"]}'
         if check["action"] == "fail":
             failures.append(message)
         else: