@ingestron/pack-databricks 0.0.0-stage → 1.0.0

package/dist/sql-data.d.ts

@@ -0,0 +1,45 @@
+import { type ResolvedTable } from "@ingestron/definitions";
+/** The table definition shared by every SQL file. */
+export interface TableData {
+    kind: "STREAMING TABLE" | "MATERIALIZED VIEW";
+    target: string;
+    constraints: {
+        name: string;
+        check: string;
+        on_violation: string;
+    }[];
+    comment: string;
+    cluster: string;
+    properties: {
+        key: string;
+        value: string;
+    }[];
+}
+/** A `read_files` call. */
+export interface SourceData {
+    stream: boolean;
+    /** The select list: `*`, or with source columns renamed to their contract names. */
+    select: string;
+    path: string;
+    options: {
+        key: string;
+        value: string;
+    }[];
+}
+/** A SQL string literal. */
+export declare function sqlString(text: string): string;
+/** The definition of the table itself. */
+export declare function tableData(table: ResolvedTable, kind: TableData["kind"]): TableData;
+/** The `read_files` call for a flow table's files. */
+export declare function sourceData(table: ResolvedTable, stream: boolean): SourceData;
+/** The AUTO CDC parts of a change-data table. */
+export declare function changeData(table: ResolvedTable): {
+    changes: string;
+    flow: string;
+    keys: string;
+    sequence: string;
+    delete_when: string;
+    except_columns: string;
+    scd: number;
+};
+//# sourceMappingURL=sql-data.d.ts.map

package/dist/sql-data.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql-data.d.ts","sourceRoot":"","sources":["../src/sql-data.ts"],"names":[],"mappings":"AAGA,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAG1E,qDAAqD;AACrD,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACrE,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC9C;AAED,2BAA2B;AAC3B,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,OAAO,CAAC;IAChB,oFAAoF;IACpF,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC3C;AAID,4BAA4B;AAC5B,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE9C;AAED,0CAA0C;AAC1C,wBAAgB,SAAS,CAAC,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,SAAS,CAgBlF;AASD,sDAAsD;AACtD,wBAAgB,UAAU,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,GAAG,UAAU,CAO5E;AA0DD,iDAAiD;AACjD,wBAAgB,UAAU,CAAC,KAAK,EAAE,aAAa;;;;;;;;EAgB9C"}

package/dist/sql-data.js

@@ -0,0 +1,111 @@
+// Template data for a table's SQL file. All quoting happens here, so the
+// templates only arrange text. Key names avoid Liquid keywords (such as
+// `required` or `with`) because python-liquid refuses them after a dot.
+import { fullNameText } from "@ingestron/definitions";
+import { quoteName } from "./query.js";
+const onViolation = { warn: "", drop: " ON VIOLATION DROP ROW", fail: " ON VIOLATION FAIL UPDATE" };
+/** A SQL string literal. */
+export function sqlString(text) {
+    return `'${text.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}'`;
+}
+/** The definition of the table itself. */
+export function tableData(table, kind) {
+    const settings = table.store?.settings ?? {};
+    return {
+        kind,
+        target: quoteName(fullNameText(table.fullName)),
+        constraints: table.expectations.map((item) => ({
+            name: quoteName(item.name),
+            check: item.check,
+            on_violation: onViolation[item.action],
+        })),
+        comment: table.contract.description ? sqlString(table.contract.description) : "",
+        cluster: clusterOf(settings.cluster_by),
+        properties: Object.entries((settings.properties ?? {})).map(([key, value]) => ({ key: sqlString(key), value: sqlString(value) })),
+    };
+}
+function clusterOf(value) {
+    if (value === "auto")
+        return "AUTO";
+    if (Array.isArray(value))
+        return `(${value.map((column) => quoteName(String(column))).join(", ")})`;
+    return "";
+}
+/** The `read_files` call for a flow table's files. */
+export function sourceData(table, stream) {
+    return {
+        stream,
+        select: selectOf(table),
+        path: sqlString(pathOf(table)),
+        options: optionsOf(table),
+    };
+}
+/** The columns read under another name at the source, unless the ingest step renamed them. */
+function renamedColumns(table) {
+    if (table.ingest?.renamed)
+        return [];
+    return table.contract.columns.filter((column) => column.physicalName && column.physicalName !== column.name);
+}
+/** `*`, or `* EXCEPT (<source names>), <source> AS <name>, …` for renamed columns. */
+function selectOf(table) {
+    const renamed = renamedColumns(table);
+    if (renamed.length === 0)
+        return "*";
+    const sources = renamed.map((column) => quoteName(column.physicalName));
+    const aliases = renamed.map((column) => `${quoteName(column.physicalName)} AS ${quoteName(column.name)}`);
+    return `* EXCEPT (${sources.join(", ")}), ${aliases.join(", ")}`;
+}
+/** The connection's `path` joined with the table's object. */
+function pathOf(table) {
+    // Files copied by another pack (such as adf-copy) are read where it lands them.
+    const ingest = table.ingest;
+    const folder = ingest?.settings.landing ?? ingest?.connection.settings.path ?? "";
+    const base = String(folder).replace(/\/+$/, "");
+    const object = (table.ingest?.object ?? "").replace(/^\/+/, "");
+    return `${base}/${object}`;
+}
+/** Format, schema hints from the contract, schema evolution, then the user's own options. */
+function optionsOf(table) {
+    const settings = table.ingest?.settings ?? {};
+    const options = [
+        { key: "format", value: sqlString(String(settings.format)) },
+    ];
+    // Hints name the columns as the files have them.
+    const source = new Set(renamedColumns(table));
+    const hints = table.contract.columns
+        .filter((column) => column.type)
+        .map((column) => {
+        const name = source.has(column) ? column.physicalName : column.name;
+        return `${quoteName(name)} ${column.type}`;
+    });
+    if (settings.schema_hints === true && hints.length > 0)
+        options.push({ key: "schemaHints", value: sqlString(hints.join(", ")) });
+    if (settings.schema_evolution)
+        options.push({
+            key: "schemaEvolutionMode",
+            value: sqlString(String(settings.schema_evolution)),
+        });
+    for (const [key, value] of Object.entries((settings.options ?? {}))) {
+        options.push({ key, value: typeof value === "string" ? sqlString(value) : String(value) });
+    }
+    return options;
+}
+/** The AUTO CDC parts of a change-data table. */
+export function changeData(table) {
+    const settings = table.store?.settings ?? {};
+    const changes = {
+        ...table.fullName,
+        table: `${table.fullName.table}${String(settings.changes_suffix)}`,
+    };
+    const except = (settings.except_columns ?? []);
+    return {
+        changes: quoteName(fullNameText(changes)),
+        flow: quoteName(`${table.fullName.table}_cdc`),
+        keys: table.contract.primaryKey.map(quoteName).join(", "),
+        sequence: quoteName(String(settings.sequence_by)),
+        delete_when: typeof settings.delete_when === "string" ? settings.delete_when : "",
+        except_columns: except.map(quoteName).join(", "),
+        scd: settings.mode === "history" ? 2 : 1,
+    };
+}
+//# sourceMappingURL=sql-data.js.map

package/dist/sql-data.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql-data.js","sourceRoot":"","sources":["../src/sql-data.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,wEAAwE;AACxE,wEAAwE;AACxE,OAAO,EAAE,YAAY,EAAsB,MAAM,wBAAwB,CAAC;AAC1E,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAqBvC,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC;AAEpG,4BAA4B;AAC5B,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,0CAA0C;AAC1C,MAAM,UAAU,SAAS,CAAC,KAAoB,EAAE,IAAuB;IACrE,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,EAAE,QAAQ,IAAI,EAAE,CAAC;IAC7C,OAAO;QACL,IAAI;QACJ,MAAM,EAAE,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/C,WAAW,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC7C,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,YAAY,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC;SACvC,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE;QAChF,OAAO,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;QACvC,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAA2B,CAAC,CAAC,GAAG,CACnF,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CACrE;KACF,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,KAAK,KAAK,MAAM;QAAE,OAAO,MAAM,CAAC;IACpC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACtB,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;IAC5E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,sDAAsD;AACtD,MAAM,UAAU,UAAU,CAAC,KAAoB,EAAE,MAAe;IAC9D,OAAO;QACL,MAAM;QACN,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC;QACvB,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC9B,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC;KAC1B,CAAC;AACJ,CAAC;AAED,8FAA8F;AAC9F,SAAS,cAAc,CAAC,KAAoB;IAC1C,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;QAAE,OAAO,EAAE,CAAC;IACrC,OAAO,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAClC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,YAAY,KAAK,MAAM,CAAC,IAAI,CACvE,CAAC;AACJ,CAAC;AAED,sFAAsF;AACtF,SAAS,QAAQ,CAAC,KAAoB;IACpC,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACtC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IACrC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,YAAa,CAAC,CAAC,CAAC;IACzE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CACzB,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,YAAa,CAAC,OAAO,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAC9E,CAAC;IACF,OAAO,aAAa,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AACnE,CAAC;AAED,8DAA8D;AAC9D,SAAS,MAAM,CAAC,KAAoB;IAClC,gFAAgF;IAChF,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,MAAM,MAAM,GAAG,MAAM,EAAE,QAAQ,CAAC,OAAO,IAAI,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;IAClF,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChE,OAAO,GAAG,IAAI,IAAI,MAAM,EAAE,CAAC;AAC7B,CAAC;AAED,6FAA6F;AAC7F,SAAS,SAAS,CAAC,KAAoB;IACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,QAAQ,IAAI,EAAE,CAAC;IAC9C,MAAM,OAAO,GAA0B;QACrC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE;KAC7D,CAAC;IACF,iDAAiD;IACjD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO;SACjC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC;SAC/B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACd,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,YAAa,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;QACrE,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;IAC7C,CAAC,CAAC,CAAC;IACL,IAAI,QAAQ,CAAC,YAAY,KAAK,IAAI,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,QAAQ,CAAC,gBAAgB;QAC3B,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,qBAAqB;YAC1B,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;SACpD,CAAC,CAAC;IACL,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAA4B,CAAC,EAAE,CAAC;QAC/F,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC7F,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,iDAAiD;AACjD,MAAM,UAAU,UAAU,CAAC,KAAoB;IAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,EAAE,QAAQ,IAAI,EAAE,CAAC;IAC7C,MAAM,OAAO,GAAG;QACd,GAAG,KAAK,CAAC,QAAQ;QACjB,KAAK,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE;KACnE,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,QAAQ,CAAC,cAAc,IAAI,EAAE,CAAa,CAAC;IAC3D,OAAO;QACL,OAAO,EAAE,SAAS,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,EAAE,SAAS,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC;QAC9C,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QACzD,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QACjD,WAAW,EAAE,OAAO,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE;QACjF,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAChD,GAAG,EAAE,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACzC,CAAC;AACJ,CAAC"}

package/dist/sql.d.ts

@@ -0,0 +1,22 @@
+import type { ResolvedTable } from "@ingestron/definitions";
+/** The templates folder, also used by the template parity check. */
+export declare const templatesFolder: string;
+/** Which template renders a table's file, its data, and the file's extension. */
+export interface SqlFile {
+    template: string;
+    data: object;
+    extension: "sql" | "py";
+}
+/** The SQL or Python file for one table; `header` is its first comment line. */
+export declare function tableSql(table: ResolvedTable, header: string): string;
+/** Renders a file chosen by `sqlFile`. */
+export declare function renderSql({ template, data }: SqlFile): string;
+/** The after-refresh Python file, with its statements and checks as JSON. */
+export declare function renderApply(data: {
+    header: string;
+    statements: string[];
+    checks: object[];
+}): string;
+/** Chooses the template by transform and store method, and builds its data. */
+export declare function sqlFile(table: ResolvedTable, header: string): SqlFile;
+//# sourceMappingURL=sql.d.ts.map

package/dist/sql.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql.d.ts","sourceRoot":"","sources":["../src/sql.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAM5D,oEAAoE;AACpE,eAAO,MAAM,eAAe,QAAiE,CAAC;AAS9F,iFAAiF;AACjF,MAAM,WAAW,OAAO;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,KAAK,GAAG,IAAI,CAAC;CACzB;AAED,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAErE;AAED,0CAA0C;AAC1C,wBAAgB,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,OAAO,GAAG,MAAM,CAE7D;AAED,6EAA6E;AAC7E,wBAAgB,WAAW,CAAC,IAAI,EAAE;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,GAAG,MAAM,CAUT;AAED,+EAA+E;AAC/E,wBAAgB,OAAO,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAYrE"}

package/dist/sql.js

@@ -0,0 +1,81 @@
+// Renders each table's SQL file from the templates in ../templates.
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { Liquid } from "liquidjs";
+import { renderQuery } from "./query.js";
+import { pythonData } from "./python-data.js";
+import { changeData, sourceData, tableData } from "./sql-data.js";
+/** The templates folder, also used by the template parity check. */
+export const templatesFolder = join(dirname(fileURLToPath(import.meta.url)), "../templates/");
+const engine = new Liquid({
+    root: templatesFolder,
+    extname: ".liquid",
+    strictVariables: true,
+    strictFilters: true,
+});
+/** The SQL or Python file for one table; `header` is its first comment line. */
+export function tableSql(table, header) {
+    return renderSql(sqlFile(table, header));
+}
+/** Renders a file chosen by `sqlFile`. */
+export function renderSql({ template, data }) {
+    return `${engine.renderFileSync(template, data).trimEnd()}\n`;
+}
+/** The after-refresh Python file, with its statements and checks as JSON. */
+export function renderApply(data) {
+    return renderSql({
+        template: "apply-contract.py.liquid",
+        data: {
+            header: data.header,
+            statements: JSON.stringify(data.statements, null, 2),
+            checks: JSON.stringify(data.checks, null, 2),
+        },
+        extension: "py",
+    });
+}
+/** Chooses the template by transform and store method, and builds its data. */
+export function sqlFile(table, header) {
+    if (table.transform?.method === "pyspark") {
+        return {
+            template: "materialised-view.py.liquid",
+            data: pythonData(table, header),
+            extension: "py",
+        };
+    }
+    const store = table.store?.method;
+    if (store === "streaming-table")
+        return streamingTable(table, header);
+    if (store === "materialised-view")
+        return materialisedView(table, header);
+    throw new Error(`No SQL for store method "${String(store)}" (table ${table.name}).`);
+}
+function streamingTable(table, header) {
+    const data = {
+        header,
+        table: tableData(table, "STREAMING TABLE"),
+        source: sourceData(table, true),
+    };
+    if (table.store?.settings.mode === "append")
+        return { template: "streaming-table.sql.liquid", data, extension: "sql" };
+    return {
+        template: "change-data.sql.liquid",
+        data: { ...data, cdc: changeData(table) },
+        extension: "sql",
+    };
+}
+function materialisedView(table, header) {
+    const rendered = table.transform
+        ? renderQuery(table.transform.code, table.transform.inputs)
+        : { sql: "" };
+    // The sql method's check has already reported SQL that cannot be rendered.
+    const query = "sql" in rendered ? rendered.sql : "";
+    const source = table.ingest
+        ? sourceData(table, false)
+        : { stream: false, select: "*", path: "", options: [] };
+    return {
+        template: "materialised-view.sql.liquid",
+        data: { header, table: tableData(table, "MATERIALIZED VIEW"), source, query },
+        extension: "sql",
+    };
+}
+//# sourceMappingURL=sql.js.map

package/dist/sql.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql.js","sourceRoot":"","sources":["../src/sql.ts"],"names":[],"mappings":"AAAA,oEAAoE;AACpE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAElE,oEAAoE;AACpE,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;AAE9F,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC;IACxB,IAAI,EAAE,eAAe;IACrB,OAAO,EAAE,SAAS;IAClB,eAAe,EAAE,IAAI;IACrB,aAAa,EAAE,IAAI;CACpB,CAAC,CAAC;AASH,gFAAgF;AAChF,MAAM,UAAU,QAAQ,CAAC,KAAoB,EAAE,MAAc;IAC3D,OAAO,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED,0CAA0C;AAC1C,MAAM,UAAU,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAW;IACnD,OAAO,GAAI,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAY,CAAC,OAAO,EAAE,IAAI,CAAC;AAC5E,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,WAAW,CAAC,IAI3B;IACC,OAAO,SAAS,CAAC;QACf,QAAQ,EAAE,0BAA0B;QACpC,IAAI,EAAE;YACJ,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;YACpD,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;SAC7C;QACD,SAAS,EAAE,IAAI;KAChB,CAAC,CAAC;AACL,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,OAAO,CAAC,KAAoB,EAAE,MAAc;IAC1D,IAAI,KAAK,CAAC,SAAS,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO;YACL,QAAQ,EAAE,6BAA6B;YACvC,IAAI,EAAE,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC;YAC/B,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC;IAClC,IAAI,KAAK,KAAK,iBAAiB;QAAE,OAAO,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACtE,IAAI,KAAK,KAAK,mBAAmB;QAAE,OAAO,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC1E,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,KAAK,CAAC,YAAY,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;AACvF,CAAC;AAED,SAAS,cAAc,CAAC,KAAoB,EAAE,MAAc;IAC1D,MAAM,IAAI,GAAG;QACX,MAAM;QACN,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,iBAAiB,CAAC;QAC1C,MAAM,EAAE,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC;KAChC,CAAC;IACF,IAAI,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,KAAK,QAAQ;QACzC,OAAO,EAAE,QAAQ,EAAE,4BAA4B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAC5E,OAAO;QACL,QAAQ,EAAE,wBAAwB;QAClC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC,EAAE;QACzC,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAoB,EAAE,MAAc;IAC5D,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS;QAC9B,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC;QAC3D,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;IAChB,2EAA2E;IAC3E,MAAM,KAAK,GAAG,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM;QACzB,CAAC,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,CAAC;QAC1B,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAC1D,OAAO;QACL,QAAQ,EAAE,8BAA8B;QACxC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,mBAAmB,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE;QAC7E,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,71 @@
 {
   "name": "@ingestron/pack-databricks",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "1.0.0",
+  "description": "Databricks methods for Ingestron: generates a bundle with a Lakeflow Spark Declarative Pipeline.",
+  "license": "Apache-2.0",
+  "author": "IntentLabs <hello@ingestron.io>",
+  "homepage": "https://docs.ingestron.io",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ingestron/pack-databricks.git"
+  },
+  "bugs": {
+    "url": "https://github.com/ingestron/pack-databricks/issues",
+    "email": "hello@ingestron.io"
+  },
+  "type": "module",
+  "engines": {
+    "node": ">=22.13.0"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "templates",
+    "README.md",
+    "NOTICE"
+  ],
+  "dependencies": {
+    "liquidjs": "10.30.0",
+    "yaml": "2.9.1"
+  },
+  "peerDependencies": {
+    "@ingestron/definitions": "^1.0.0"
+  },
+  "devDependencies": {
+    "@ingestron/config": "^1.0.0",
+    "@ingestron/definitions": "^1.0.0",
+    "@microsoft/api-extractor": "7.59.4",
+    "@types/node": "22.20.5",
+    "ajv": "8.20.0",
+    "eslint": "10.13.0",
+    "eslint-plugin-jsdoc": "65.4.2",
+    "prettier": "3.9.10",
+    "tsx": "4.23.15",
+    "typescript": "6.0.3",
+    "typescript-eslint": "8.72.0",
+    "vitest": "5.0.5"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "typecheck": "pnpm build && tsc -p tsconfig.json --noEmit",
+    "lint": "eslint .",
+    "test": "vitest run",
+    "check:docs": "tsx scripts/check-docs.ts",
+    "validate": "pnpm format:check && pnpm secrets:check && pnpm licences:check && node --test scripts/*.test.mjs && pnpm lint && pnpm typecheck && pnpm test && pnpm check:docs && pnpm check:api",
+    "api": "api-extractor run --local",
+    "check:api": "api-extractor run",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "secrets:check": "node scripts/scan-secrets.mjs",
+    "licences:check": "node scripts/check-licences.mjs",
+    "dco:check": "node scripts/check-dco.mjs"
+  }
 }

package/templates/apply-contract.py.liquid

@@ -0,0 +1,58 @@
+# {{ header }}
+# Runs after each refresh: Lakeflow pipelines cannot set tags or column
+# comments, nor compare counts over all rows, so this task does.
+import json
+
+from pyspark.sql import SparkSession
+
+STATEMENTS = json.loads(r"""{{ statements }}""")
+CHECKS = json.loads(r"""{{ checks }}""")
+
+
+def passes(operator, expected, measured):
+    """True when the measured count meets the rule's comparison."""
+    if operator == "mustBe":
+        return measured == expected
+    if operator == "mustNotBe":
+        return measured != expected
+    if operator == "mustBeGreaterThan":
+        return measured > expected
+    if operator == "mustBeGreaterOrEqualTo":
+        return measured >= expected
+    if operator == "mustBeLessThan":
+        return measured < expected
+    if operator == "mustBeLessOrEqualTo":
+        return measured <= expected
+    low, high = expected
+    inside = low <= measured <= high
+    return inside if operator == "mustBeBetween" else not inside
+
+
+def measure(spark, check):
+    """The rule's count, in rows or as a percentage of all rows."""
+    row = spark.sql(check["query"]).first()
+    counted, total = row["counted"] or 0, row["total"] or 0
+    if check["unit"] == "percent":
+        return 0 if total == 0 else counted * 100 / total
+    return counted
+
+
+def main():
+    spark = SparkSession.builder.getOrCreate()
+    for statement in STATEMENTS:
+        spark.sql(statement)
+    failures = []
+    for check in CHECKS:
+        measured = measure(spark, check)
+        if passes(check["operator"], check["value"], measured):
+            continue
+        message = f'{check["table"]}: {check["name"]} measured {measured}, expected {check["operator"]} {check["value"]}'
+        if check["action"] == "fail":
+            failures.append(message)
+        else:
+            print(f"Warning: {message}")
+    if failures:
+        raise RuntimeError("Table checks failed:\n" + "\n".join(failures))
+
+
+main()

package/templates/change-data.sql.liquid

@@ -0,0 +1,20 @@
+-- {{ header }}
+-- Changes as received, each read once.
+CREATE OR REFRESH STREAMING TABLE {{ cdc.changes }}
+AS {% render "read-files", source: source %};
+
+-- The current rows{% if cdc.scd == 2 %}, with history{% endif %}, kept by key.
+{% render "table", table: table %};
+
+CREATE FLOW {{ cdc.flow }}
+AS AUTO CDC INTO {{ table.target }}
+FROM STREAM({{ cdc.changes }})
+KEYS ({{ cdc.keys }})
+{%- if cdc.delete_when != "" %}
+APPLY AS DELETE WHEN {{ cdc.delete_when }}
+{%- endif %}
+SEQUENCE BY {{ cdc.sequence }}
+{%- if cdc.except_columns != "" %}
+COLUMNS * EXCEPT ({{ cdc.except_columns }})
+{%- endif %}
+STORED AS SCD TYPE {{ cdc.scd }};

package/templates/materialised-view.py.liquid

@@ -0,0 +1,28 @@
+# {{ header }}
+from pyspark import pipelines as dp
+
+# The transform's own code, unchanged:
+{{ code }}
+
+
+@dp.materialized_view(
+    name={{ name }},
+{%- if comment != "" %}
+    comment={{ comment }},
+{%- endif %}
+{%- if cluster != "" %}
+    {{ cluster }},
+{%- endif %}
+{%- if properties != "" %}
+    table_properties={{ properties }},
+{%- endif %}
+)
+{%- for group in expectations %}
+@dp.{{ group.decorator }}({{ group.checks }})
+{%- endfor %}
+def {{ wrapper }}():
+    return {{ function }}({
+{%- for input in inputs %}
+        {{ input.name }}: spark.read.table({{ input.table }}),
+{%- endfor %}
+    })

package/templates/materialised-view.sql.liquid

@@ -0,0 +1,4 @@
+-- {{ header }}
+{% render "table", table: table %}
+AS
+{% if query != "" %}{{ query }}{% else %}{% render "read-files", source: source %}{% endif %};

package/templates/read-files.liquid

@@ -0,0 +1,7 @@
+{%- comment %} Ends without a newline: the caller continues after it. {% endcomment -%}
+SELECT {{ source.select }} FROM {% if source.stream %}STREAM {% endif %}read_files(
+  {{ source.path }}
+{%- for option in source.options %},
+  {{ option.key }} => {{ option.value }}
+{%- endfor %}
+){%- comment %}{% endcomment -%}

package/templates/streaming-table.sql.liquid

@@ -0,0 +1,3 @@
+-- {{ header }}
+{% render "table", table: table %}
+AS {% render "read-files", source: source %};

package/templates/table.liquid

@@ -0,0 +1,17 @@
+{%- comment %} Ends without a newline: the caller continues after it. {% endcomment -%}
+CREATE OR REFRESH {{ table.kind }} {{ table.target }}
+{%- if table.constraints.size > 0 %} (
+{%- for constraint in table.constraints %}
+  CONSTRAINT {{ constraint.name }} EXPECT ({{ constraint.check }}){{ constraint.on_violation }}{% unless forloop.last %},{% endunless %}
+{%- endfor %}
+)
+{%- endif %}
+{%- if table.comment != "" %}
+COMMENT {{ table.comment }}
+{%- endif %}
+{%- if table.cluster != "" %}
+CLUSTER BY {{ table.cluster }}
+{%- endif %}
+{%- if table.properties.size > 0 %}
+TBLPROPERTIES ({% for property in table.properties %}{{ property.key }} = {{ property.value }}{% unless forloop.last %}, {% endunless %}{% endfor %})
+{%- endif -%}