@ingestron/pack-databricks 0.0.0-stage → 1.0.1

package/dist/methods.js

@@ -0,0 +1,166 @@
+import { renderQuery } from "./query.js";
+const clusterBy = {
+    description: "How the table is clustered: auto, none, or a list of columns.",
+    anyOf: [
+        {
+            enum: ["auto", "none"],
+            enumDescriptions: ["Databricks chooses the clustering columns.", "No clustering."],
+        },
+        { type: "array", minItems: 1, items: { type: "string" } },
+    ],
+    examples: [["customer_id", "ordered_at"]],
+};
+const properties = {
+    type: "object",
+    description: "Table properties (TBLPROPERTIES).",
+    additionalProperties: { type: "string" },
+};
+/** Reads files where they already are, with Auto Loader (`read_files`). */
+export const files = {
+    action: "ingest",
+    description: "Reads files in cloud storage or a volume with Auto Loader.",
+    connectors: ["files"],
+    connectivity: ["public", "workspace"],
+    produces: "files",
+    defaults: {
+        format: "parquet",
+        schema_evolution: "addNewColumns",
+        schema_hints: true,
+        options: {},
+    },
+    settings: {
+        type: "object",
+        additionalProperties: false,
+        properties: {
+            format: {
+                enum: ["parquet", "csv", "json", "avro", "orc", "xml", "text"],
+                enumDescriptions: [
+                    "Parquet files.",
+                    "CSV text; set options such as header: true.",
+                    "JSON, one object per line unless options say otherwise.",
+                    "Avro files.",
+                    "ORC files.",
+                    "XML; set options such as rowTag.",
+                    "Each line as one row.",
+                ],
+                description: "The files' format.",
+            },
+            schema_evolution: {
+                enum: [
+                    "addNewColumns",
+                    "addNewColumnsWithTypeWidening",
+                    "rescue",
+                    "failOnNewColumns",
+                    "none",
+                ],
+                enumDescriptions: [
+                    "Stop when files gain columns, then go on with the columns added.",
+                    "As addNewColumns, and also widen column types, such as int to bigint.",
+                    "Keep the columns as they are; values that do not fit go to _rescued_data.",
+                    "Stop when files gain columns, until the table is changed.",
+                    "Ignore new columns.",
+                ],
+                description: "What happens when files gain columns.",
+            },
+            schema_hints: {
+                type: "boolean",
+                description: "Pass the contract's column types as schemaHints.",
+            },
+            options: {
+                type: "object",
+                description: "Other read_files options, such as header: true.",
+                additionalProperties: { type: ["string", "number", "boolean"] },
+            },
+        },
+    },
+    check: (table) => typeof table.ingest?.connection.settings.path === "string"
+        ? []
+        : ['the connection needs "settings.path"'],
+};
+/** Table properties are the organisation's, such as change data feed: written in the standard. */
+const storePlacement = { standard: ["properties"] };
+/** A streaming table: each file is read once; optionally kept current with AUTO CDC. */
+export const streamingTable = {
+    action: "store",
+    description: "A streaming table. mode append adds rows; upsert (SCD type 1) and history (SCD type 2) apply changes by key.",
+    accepts: ["files"],
+    produces: "table",
+    defaults: { mode: "append", cluster_by: "auto", properties: {}, changes_suffix: "_changes" },
+    placement: storePlacement,
+    settings: {
+        type: "object",
+        additionalProperties: false,
+        properties: {
+            mode: {
+                enum: ["append", "upsert", "history"],
+                enumDescriptions: [
+                    "Add every row as it arrives.",
+                    "Keep the latest row per key (SCD type 1), ordered by sequence_by.",
+                    "Keep every version of a row with when it was valid (SCD type 2), ordered by sequence_by.",
+                ],
+                description: "How rows are kept.",
+            },
+            sequence_by: {
+                type: "string",
+                description: "Column that orders changes; needed for upsert and history.",
+                examples: ["updated_at"],
+            },
+            delete_when: {
+                type: "string",
+                description: "SQL condition for rows that are deletions, such as op = 'D'.",
+                examples: ["op = 'D'"],
+            },
+            except_columns: {
+                type: "array",
+                items: { type: "string" },
+                description: "Change columns not to keep, such as op.",
+            },
+            changes_suffix: {
+                type: "string",
+                pattern: "^_[a-z0-9_]+$",
+                description: "Suffix of the table that keeps changes as received.",
+                examples: ["_changes"],
+            },
+            cluster_by: clusterBy,
+            properties,
+        },
+    },
+    check: checkChangeData,
+};
+/** A materialised view, recomputed (incrementally where possible) on each update. */
+export const materialisedView = {
+    action: "store",
+    description: "A materialised view. Expectations make every refresh a full one.",
+    accepts: ["files", "table", "query"],
+    produces: "table",
+    defaults: { cluster_by: "auto", properties: {} },
+    placement: storePlacement,
+    settings: {
+        type: "object",
+        additionalProperties: false,
+        properties: { cluster_by: clusterBy, properties },
+    },
+};
+/** The transform's own SQL, with `{{ tables.<input> }}` replaced by full names. */
+export const sql = {
+    action: "transform",
+    description: "Runs the transform's SQL query.",
+    accepts: ["table"],
+    produces: "query",
+    settings: { type: "object", additionalProperties: false },
+    check: (table) => {
+        const result = renderQuery(table.transform?.code ?? "", table.transform?.inputs ?? {});
+        return "problem" in result ? [result.problem] : [];
+    },
+};
+function checkChangeData(table) {
+    const settings = table.store?.settings ?? {};
+    if (settings.mode === "append")
+        return [];
+    const problems = [];
+    if (table.contract.primaryKey.length === 0)
+        problems.push(`mode ${String(settings.mode)} needs primary key columns in the contract`);
+    if (typeof settings.sequence_by !== "string")
+        problems.push(`mode ${String(settings.mode)} needs "sequence_by"`);
+    return problems;
+}

package/dist/pack.d.ts

@@ -0,0 +1,8 @@
+import type { Pack } from "@ingestron/definitions";
+/**
+ * The Databricks pack, version 1. Name it in a standard as `pack: databricks@1`.
+ * @example
+ * const result = loadDefinitions("definitions", { pack: databricks, environment: "dev" });
+ * if (result.model) console.log(Object.keys(databricks.build(result.model)));
+ */
+export declare const databricks: Pack;

package/dist/pack.js

@@ -0,0 +1,72 @@
+// The pack: methods, smart defaults, rules and the build function.
+import { contract } from "./contract.js";
+import { connectors } from "./connectors.js";
+import { build } from "./build.js";
+import { request } from "./request.js";
+import { capabilities, tagProblems } from "./capabilities.js";
+import { lakeflowConnect } from "./connect-method.js";
+import { connectGroups, sharedProblems } from "./connect.js";
+import { files, materialisedView, sql, streamingTable } from "./methods.js";
+import { pyspark } from "./pyspark-method.js";
+/**
+ * The Databricks pack, version 1. Name it in a standard as `pack: databricks@1`.
+ * @example
+ * const result = loadDefinitions("definitions", { pack: databricks, environment: "dev" });
+ * if (result.model) console.log(Object.keys(databricks.build(result.model)));
+ */
+export const databricks = {
+    id: "databricks",
+    version: 1,
+    defaults: {
+        naming: {
+            catalog: "{{ environment }}",
+            schema: "{{ layer }}",
+            table: "{{ table }}",
+            pipeline: "{{ project }}",
+            job: "{{ project }}_refresh",
+        },
+        layers: ["bronze", "silver", "gold"],
+        types: {
+            string: "STRING",
+            integer: "BIGINT",
+            number: "DOUBLE",
+            boolean: "BOOLEAN",
+            date: "DATE",
+            timestamp: "TIMESTAMP",
+        },
+        quality: { required: "warn", default: "warn" },
+    },
+    rules: {
+        ingest: {
+            default: "files",
+            when: [
+                { connector: "sqlserver", use: "lakeflow-connect" },
+                { connector: "salesforce", use: "lakeflow-connect" },
+                { connector: "kafka", use: "lakeflow-connect", with: { mode: "append" } },
+            ],
+        },
+        transform: { default: "sql" },
+        store: { default: "streaming-table", when: [{ input: "query", use: "materialised-view" }] },
+    },
+    methods: {
+        files,
+        "lakeflow-connect": lakeflowConnect,
+        sql,
+        pyspark,
+        "streaming-table": streamingTable,
+        "materialised-view": materialisedView,
+    },
+    connectors,
+    capabilities,
+    contract,
+    check(model) {
+        const problems = [];
+        if (!model.platform.compute.serverless)
+            problems.push("classic compute is not supported yet; leave compute.serverless at true");
+        if (model.tables.length === 0)
+            problems.push("there is nothing to generate: add a flow or a transform");
+        return [...problems, ...tagProblems(model), ...sharedProblems(connectGroups(model))];
+    },
+    build,
+    request,
+};

package/dist/pyspark-method.d.ts

@@ -0,0 +1,3 @@
+import type { Method } from "@ingestron/definitions";
+/** A PySpark transform: the code defines a function that gets a DataFrame per input. */
+export declare const pyspark: Method;

package/dist/pyspark-method.js

@@ -0,0 +1,40 @@
+/** A PySpark transform: the code defines a function that gets a DataFrame per input. */
+export const pyspark = {
+    action: "transform",
+    description: "Calls a function in the transform's Python code with a dict of input DataFrames; it returns one DataFrame.",
+    accepts: ["table"],
+    produces: "query",
+    defaults: { function: "transform" },
+    settings: {
+        type: "object",
+        additionalProperties: false,
+        properties: {
+            function: {
+                type: "string",
+                pattern: "^[A-Za-z_][A-Za-z0-9_]*$",
+                description: "The function to call. Default: transform.",
+                examples: ["build_daily_sales"],
+            },
+        },
+    },
+    check: (table) => {
+        const code = table.transform?.code ?? "";
+        const name = String(table.transform?.settings.function);
+        const defined = new RegExp(`^def ${name}\\(`, "m").test(code);
+        return [
+            ...(defined ? [] : [`the code has no function "def ${name}(tables)"`]),
+            ...unknownInputs(code, Object.keys(table.transform?.inputs ?? {})),
+        ];
+    },
+};
+/**
+ * Each `tables["x"]` in the code whose `x` is not an input: the function gets
+ * a dict with one DataFrame per input, so reading another key fails.
+ */
+function unknownInputs(code, inputs) {
+    const lines = code.split("\n").map((line) => line.replace(/#.*/, ""));
+    const read = [...lines.join("\n").matchAll(/\btables\[\s*["']([^"']+)["']\s*\]/g)].map((match) => match[1]);
+    return [...new Set(read)]
+        .filter((key) => !inputs.includes(key))
+        .map((key) => `the code reads tables["${key}"], which is not an input; the inputs are ${inputs.map((input) => `"${input}"`).join(", ") || "none"}`);
+}

package/dist/python-data.d.ts

@@ -0,0 +1,21 @@
+import { type ResolvedTable } from "@ingestron/definitions";
+/** Everything materialised-view.py.liquid needs. */
+export declare function pythonData(table: ResolvedTable, header: string): {
+    header: string;
+    code: string;
+    name: string;
+    comment: string;
+    cluster: string;
+    properties: string;
+    expectations: {
+        decorator: "expect_all" | "expect_all_or_drop" | "expect_all_or_fail";
+        checks: string;
+        count: number;
+    }[];
+    wrapper: string;
+    function: string;
+    inputs: {
+        name: string;
+        table: string;
+    }[];
+};

package/dist/python-data.js

@@ -0,0 +1,49 @@
+// Template data for a PySpark transform's Python file. Python string and
+// dict literals are written as JSON, which Python reads the same way.
+import { fullNameText } from "@ingestron/definitions";
+const decorators = {
+    warn: "expect_all",
+    drop: "expect_all_or_drop",
+    fail: "expect_all_or_fail",
+};
+/** Everything materialised-view.py.liquid needs. */
+export function pythonData(table, header) {
+    const store = table.store?.settings ?? {};
+    const transform = table.transform;
+    const properties = (store.properties ?? {});
+    return {
+        header,
+        code: transform.code.trimEnd(),
+        name: JSON.stringify(fullNameText(table.fullName)),
+        comment: table.contract.description ? JSON.stringify(table.contract.description) : "",
+        cluster: clusterOf(store.cluster_by),
+        properties: Object.keys(properties).length > 0 ? pythonDict(Object.entries(properties)) : "",
+        expectations: ["warn", "drop", "fail"]
+            .map((action) => {
+            const checks = table.expectations.filter((item) => item.action === action);
+            return {
+                decorator: decorators[action],
+                checks: pythonDict(checks.map((item) => [item.name, item.check])),
+                count: checks.length,
+            };
+        })
+            .filter((group) => group.count > 0),
+        wrapper: `ingestron_${table.name}`,
+        function: String(transform.settings.function),
+        inputs: Object.entries(transform.inputs).map(([name, full]) => ({
+            name: JSON.stringify(name),
+            table: JSON.stringify(full),
+        })),
+    };
+}
+function clusterOf(value) {
+    if (value === "auto")
+        return "cluster_by_auto=True";
+    if (Array.isArray(value))
+        return `cluster_by=${JSON.stringify(value)}`;
+    return "";
+}
+/** `{"a": "b", …}`: a Python dict of strings, written the way Python code usually is. */
+function pythonDict(entries) {
+    return `{${entries.map(([key, value]) => `${JSON.stringify(key)}: ${JSON.stringify(value)}`).join(", ")}}`;
+}

package/dist/query.d.ts

@@ -0,0 +1,8 @@
+/** The SQL with inputs replaced, or a problem to report. A trailing `;` is removed. */
+export declare function renderQuery(code: string, inputs: Record<string, string>): {
+    sql: string;
+} | {
+    problem: string;
+};
+/** `a.b.c` → `` `a`.`b`.`c` ``, so any valid name is safe in SQL. */
+export declare function quoteName(fullName: string): string;

package/dist/query.js

@@ -0,0 +1,25 @@
+// Renders a transform's SQL: `{{ tables.<input> }}` becomes the input's full name.
+import { Liquid } from "liquidjs";
+// Strict, so an input that the transform did not declare is an error.
+const engine = new Liquid({ strictVariables: true, strictFilters: true });
+/** The SQL with inputs replaced, or a problem to report. A trailing `;` is removed. */
+export function renderQuery(code, inputs) {
+    const quoted = Object.fromEntries(Object.entries(inputs).map(([name, full]) => [name, quoteName(full)]));
+    try {
+        const sql = engine.parseAndRenderSync(code, { tables: quoted });
+        return { sql: sql.trim().replace(/;\s*$/, "") };
+    }
+    catch (cause) {
+        const reason = cause instanceof Error ? cause.message.split("\n")[0] : String(cause);
+        return {
+            problem: `the SQL cannot be rendered (${reason}); inputs are written {{ tables.<name> }} and must be listed in inputs`,
+        };
+    }
+}
+/** `a.b.c` → `` `a`.`b`.`c` ``, so any valid name is safe in SQL. */
+export function quoteName(fullName) {
+    return fullName
+        .split(".")
+        .map((part) => `\`${part.replaceAll("`", "``")}\``)
+        .join(".");
+}

package/dist/request.d.ts

@@ -0,0 +1,3 @@
+import type { RequestSection, ResolvedModel } from "@ingestron/definitions";
+/** The Databricks sections of the platform request. Nothing here is checked. */
+export declare function request(model: ResolvedModel): RequestSection[];

package/dist/request.js

@@ -0,0 +1,46 @@
+/** The Databricks sections of the platform request. Nothing here is checked. */
+export function request(model) {
+    return [
+        {
+            title: "Must exist before deploying",
+            intro: "Ingestron has not checked these; the generated files expect them.",
+            lines: prerequisites(model),
+        },
+        { title: "Schedules", lines: [schedule(model)] },
+    ];
+}
+function prerequisites(model) {
+    const { platform } = model;
+    const names = model.tables.map((table) => table.fullName);
+    const catalogs = unique(names.map((name) => name.catalog));
+    const schemas = platform.manage.schemas
+        ? []
+        : unique(names.map((name) => `${name.catalog}.${name.schema}`));
+    const folders = unique(model.tables.flatMap((table) => {
+        const folder = table.ingest?.settings.landing ?? table.ingest?.connection.settings.path;
+        return typeof folder === "string" ? [folder] : [];
+    }));
+    return [
+        ...catalogs.map((catalog) => `Catalog \`${catalog}\`, which the deploying identity can use.`),
+        ...schemas.map((schema) => `Schema \`${schema}\`, in which it can create tables and views.`),
+        ...folders.map((folder) => `Files at \`${folder}\`, which the pipeline can read.`),
+        platform.compute.serverless
+            ? "Serverless compute for pipelines and jobs, turned on in the workspace."
+            : "Classic compute that the pipelines may use.",
+        ...(platform.workspace.host
+            ? []
+            : [
+                "No workspace host is set, so `databricks bundle deploy` uses the workspace of the Databricks CLI profile that runs it.",
+            ]),
+    ];
+}
+function schedule(model) {
+    const { schedule } = model.platform;
+    if (!schedule)
+        return "No schedule: the pipelines run only when someone starts them.";
+    const state = schedule.paused ? "paused" : "runs";
+    return `Job \`${model.names.job}\`: \`${schedule.cron}\` (${schedule.timezone}), ${state}.`;
+}
+function unique(values) {
+    return [...new Set(values)];
+}

package/dist/sql-data.d.ts

@@ -0,0 +1,44 @@
+import { type ResolvedTable } from "@ingestron/definitions";
+/** The table definition shared by every SQL file. */
+export interface TableData {
+    kind: "STREAMING TABLE" | "MATERIALIZED VIEW";
+    target: string;
+    constraints: {
+        name: string;
+        check: string;
+        on_violation: string;
+    }[];
+    comment: string;
+    cluster: string;
+    properties: {
+        key: string;
+        value: string;
+    }[];
+}
+/** A `read_files` call. */
+export interface SourceData {
+    stream: boolean;
+    /** The select list: `*`, or with source columns renamed to their contract names. */
+    select: string;
+    path: string;
+    options: {
+        key: string;
+        value: string;
+    }[];
+}
+/** A SQL string literal. */
+export declare function sqlString(text: string): string;
+/** The definition of the table itself. */
+export declare function tableData(table: ResolvedTable, kind: TableData["kind"]): TableData;
+/** The `read_files` call for a flow table's files. */
+export declare function sourceData(table: ResolvedTable, stream: boolean): SourceData;
+/** The AUTO CDC parts of a change-data table. */
+export declare function changeData(table: ResolvedTable): {
+    changes: string;
+    flow: string;
+    keys: string;
+    sequence: string;
+    delete_when: string;
+    except_columns: string;
+    scd: number;
+};

package/dist/sql-data.js

@@ -0,0 +1,110 @@
+// Template data for a table's SQL file. All quoting happens here, so the
+// templates only arrange text. Key names avoid Liquid keywords (such as
+// `required` or `with`) because python-liquid refuses them after a dot.
+import { fullNameText } from "@ingestron/definitions";
+import { quoteName } from "./query.js";
+const onViolation = { warn: "", drop: " ON VIOLATION DROP ROW", fail: " ON VIOLATION FAIL UPDATE" };
+/** A SQL string literal. */
+export function sqlString(text) {
+    return `'${text.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}'`;
+}
+/** The definition of the table itself. */
+export function tableData(table, kind) {
+    const settings = table.store?.settings ?? {};
+    return {
+        kind,
+        target: quoteName(fullNameText(table.fullName)),
+        constraints: table.expectations.map((item) => ({
+            name: quoteName(item.name),
+            check: item.check,
+            on_violation: onViolation[item.action],
+        })),
+        comment: table.contract.description ? sqlString(table.contract.description) : "",
+        cluster: clusterOf(settings.cluster_by),
+        properties: Object.entries((settings.properties ?? {})).map(([key, value]) => ({ key: sqlString(key), value: sqlString(value) })),
+    };
+}
+function clusterOf(value) {
+    if (value === "auto")
+        return "AUTO";
+    if (Array.isArray(value))
+        return `(${value.map((column) => quoteName(String(column))).join(", ")})`;
+    return "";
+}
+/** The `read_files` call for a flow table's files. */
+export function sourceData(table, stream) {
+    return {
+        stream,
+        select: selectOf(table),
+        path: sqlString(pathOf(table)),
+        options: optionsOf(table),
+    };
+}
+/** The columns read under another name at the source, unless the ingest step renamed them. */
+function renamedColumns(table) {
+    if (table.ingest?.renamed)
+        return [];
+    return table.contract.columns.filter((column) => column.physicalName && column.physicalName !== column.name);
+}
+/** `*`, or `* EXCEPT (<source names>), <source> AS <name>, …` for renamed columns. */
+function selectOf(table) {
+    const renamed = renamedColumns(table);
+    if (renamed.length === 0)
+        return "*";
+    const sources = renamed.map((column) => quoteName(column.physicalName));
+    const aliases = renamed.map((column) => `${quoteName(column.physicalName)} AS ${quoteName(column.name)}`);
+    return `* EXCEPT (${sources.join(", ")}), ${aliases.join(", ")}`;
+}
+/** The connection's `path` joined with the table's object. */
+function pathOf(table) {
+    // Files copied by another pack (such as adf-copy) are read where it lands them.
+    const ingest = table.ingest;
+    const folder = ingest?.settings.landing ?? ingest?.connection.settings.path ?? "";
+    const base = String(folder).replace(/\/+$/, "");
+    const object = (table.ingest?.object ?? "").replace(/^\/+/, "");
+    return `${base}/${object}`;
+}
+/** Format, schema hints from the contract, schema evolution, then the user's own options. */
+function optionsOf(table) {
+    const settings = table.ingest?.settings ?? {};
+    const options = [
+        { key: "format", value: sqlString(String(settings.format)) },
+    ];
+    // Hints name the columns as the files have them.
+    const source = new Set(renamedColumns(table));
+    const hints = table.contract.columns
+        .filter((column) => column.type)
+        .map((column) => {
+        const name = source.has(column) ? column.physicalName : column.name;
+        return `${quoteName(name)} ${column.type}`;
+    });
+    if (settings.schema_hints === true && hints.length > 0)
+        options.push({ key: "schemaHints", value: sqlString(hints.join(", ")) });
+    if (settings.schema_evolution)
+        options.push({
+            key: "schemaEvolutionMode",
+            value: sqlString(String(settings.schema_evolution)),
+        });
+    for (const [key, value] of Object.entries((settings.options ?? {}))) {
+        options.push({ key, value: typeof value === "string" ? sqlString(value) : String(value) });
+    }
+    return options;
+}
+/** The AUTO CDC parts of a change-data table. */
+export function changeData(table) {
+    const settings = table.store?.settings ?? {};
+    const changes = {
+        ...table.fullName,
+        table: `${table.fullName.table}${String(settings.changes_suffix)}`,
+    };
+    const except = (settings.except_columns ?? []);
+    return {
+        changes: quoteName(fullNameText(changes)),
+        flow: quoteName(`${table.fullName.table}_cdc`),
+        keys: table.contract.primaryKey.map(quoteName).join(", "),
+        sequence: quoteName(String(settings.sequence_by)),
+        delete_when: typeof settings.delete_when === "string" ? settings.delete_when : "",
+        except_columns: except.map(quoteName).join(", "),
+        scd: settings.mode === "history" ? 2 : 1,
+    };
+}

package/dist/sql.d.ts

@@ -0,0 +1,21 @@
+import type { ResolvedTable } from "@ingestron/definitions";
+/** The templates folder, also used by the template parity check. */
+export declare const templatesFolder: string;
+/** Which template renders a table's file, its data, and the file's extension. */
+export interface SqlFile {
+    template: string;
+    data: object;
+    extension: "sql" | "py";
+}
+/** The SQL or Python file for one table; `header` is its first comment line. */
+export declare function tableSql(table: ResolvedTable, header: string): string;
+/** Renders a file chosen by `sqlFile`. */
+export declare function renderSql({ template, data }: SqlFile): string;
+/** The after-refresh Python file, with its statements and checks as JSON. */
+export declare function renderApply(data: {
+    header: string;
+    statements: string[];
+    checks: object[];
+}): string;
+/** Chooses the template by transform and store method, and builds its data. */
+export declare function sqlFile(table: ResolvedTable, header: string): SqlFile;