@ingestron/pack-databricks 0.0.0-stage → 1.0.1

package/dist/sql.js

@@ -0,0 +1,80 @@
+// Renders each table's SQL file from the templates in ../templates.
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { Liquid } from "liquidjs";
+import { renderQuery } from "./query.js";
+import { pythonData } from "./python-data.js";
+import { changeData, sourceData, tableData } from "./sql-data.js";
+/** The templates folder, also used by the template parity check. */
+export const templatesFolder = join(dirname(fileURLToPath(import.meta.url)), "../templates/");
+const engine = new Liquid({
+    root: templatesFolder,
+    extname: ".liquid",
+    strictVariables: true,
+    strictFilters: true,
+});
+/** The SQL or Python file for one table; `header` is its first comment line. */
+export function tableSql(table, header) {
+    return renderSql(sqlFile(table, header));
+}
+/** Renders a file chosen by `sqlFile`. */
+export function renderSql({ template, data }) {
+    return `${engine.renderFileSync(template, data).trimEnd()}\n`;
+}
+/** The after-refresh Python file, with its statements and checks as JSON. */
+export function renderApply(data) {
+    return renderSql({
+        template: "apply-contract.py.liquid",
+        data: {
+            header: data.header,
+            statements: JSON.stringify(data.statements, null, 2),
+            checks: JSON.stringify(data.checks, null, 2),
+        },
+        extension: "py",
+    });
+}
+/** Chooses the template by transform and store method, and builds its data. */
+export function sqlFile(table, header) {
+    if (table.transform?.method === "pyspark") {
+        return {
+            template: "materialised-view.py.liquid",
+            data: pythonData(table, header),
+            extension: "py",
+        };
+    }
+    const store = table.store?.method;
+    if (store === "streaming-table")
+        return streamingTable(table, header);
+    if (store === "materialised-view")
+        return materialisedView(table, header);
+    throw new Error(`No SQL for store method "${String(store)}" (table ${table.name}).`);
+}
+function streamingTable(table, header) {
+    const data = {
+        header,
+        table: tableData(table, "STREAMING TABLE"),
+        source: sourceData(table, true),
+    };
+    if (table.store?.settings.mode === "append")
+        return { template: "streaming-table.sql.liquid", data, extension: "sql" };
+    return {
+        template: "change-data.sql.liquid",
+        data: { ...data, cdc: changeData(table) },
+        extension: "sql",
+    };
+}
+function materialisedView(table, header) {
+    const rendered = table.transform
+        ? renderQuery(table.transform.code, table.transform.inputs)
+        : { sql: "" };
+    // The sql method's check has already reported SQL that cannot be rendered.
+    const query = "sql" in rendered ? rendered.sql : "";
+    const source = table.ingest
+        ? sourceData(table, false)
+        : { stream: false, select: "*", path: "", options: [] };
+    return {
+        template: "materialised-view.sql.liquid",
+        data: { header, table: tableData(table, "MATERIALIZED VIEW"), source, query },
+        extension: "sql",
+    };
+}

package/package.json

@@ -1,6 +1,71 @@
 {
   "name": "@ingestron/pack-databricks",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "1.0.1",
+  "description": "Databricks methods for Ingestron: generates a bundle with a Lakeflow Spark Declarative Pipeline.",
+  "license": "Apache-2.0",
+  "author": "IntentLabs <hello@ingestron.io>",
+  "homepage": "https://docs.ingestron.io",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ingestron/pack-databricks.git"
+  },
+  "bugs": {
+    "url": "https://github.com/ingestron/pack-databricks/issues",
+    "email": "hello@ingestron.io"
+  },
+  "type": "module",
+  "engines": {
+    "node": ">=22.13.0"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "templates",
+    "README.md",
+    "NOTICE"
+  ],
+  "dependencies": {
+    "liquidjs": "10.30.0",
+    "yaml": "2.9.1"
+  },
+  "peerDependencies": {
+    "@ingestron/definitions": "^1.0.0"
+  },
+  "devDependencies": {
+    "@ingestron/config": "^1.0.0",
+    "@ingestron/definitions": "^1.0.0",
+    "@microsoft/api-extractor": "7.59.4",
+    "@types/node": "22.20.5",
+    "ajv": "8.20.0",
+    "eslint": "10.13.0",
+    "eslint-plugin-jsdoc": "65.4.2",
+    "prettier": "3.9.10",
+    "tsx": "4.23.15",
+    "typescript": "6.0.3",
+    "typescript-eslint": "8.72.0",
+    "vitest": "5.0.5"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "typecheck": "pnpm build && tsc -p tsconfig.json --noEmit",
+    "lint": "eslint .",
+    "test": "vitest run",
+    "check:docs": "tsx scripts/check-docs.ts",
+    "validate": "pnpm format:check && pnpm secrets:check && pnpm licences:check && node --test scripts/*.test.mjs && pnpm lint && pnpm typecheck && pnpm test && pnpm check:docs && pnpm check:api",
+    "api": "api-extractor run --local",
+    "check:api": "api-extractor run",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "secrets:check": "node scripts/scan-secrets.mjs",
+    "licences:check": "node scripts/check-licences.mjs",
+    "dco:check": "node scripts/check-dco.mjs"
+  }
 }

package/templates/apply-contract.py.liquid

@@ -0,0 +1,58 @@
+# {{ header }}
+# Runs after each refresh: Lakeflow pipelines cannot set tags or column
+# comments, nor compare counts over all rows, so this task does.
+import json
+
+from pyspark.sql import SparkSession
+
+STATEMENTS = json.loads(r"""{{ statements }}""")
+CHECKS = json.loads(r"""{{ checks }}""")
+
+
+def passes(operator, expected, measured):
+    """True when the measured count meets the rule's comparison."""
+    if operator == "mustBe":
+        return measured == expected
+    if operator == "mustNotBe":
+        return measured != expected
+    if operator == "mustBeGreaterThan":
+        return measured > expected
+    if operator == "mustBeGreaterOrEqualTo":
+        return measured >= expected
+    if operator == "mustBeLessThan":
+        return measured < expected
+    if operator == "mustBeLessOrEqualTo":
+        return measured <= expected
+    low, high = expected
+    inside = low <= measured <= high
+    return inside if operator == "mustBeBetween" else not inside
+
+
+def measure(spark, check):
+    """The rule's count, in rows or as a percentage of all rows."""
+    row = spark.sql(check["query"]).first()
+    counted, total = row["counted"] or 0, row["total"] or 0
+    if check["unit"] == "percent":
+        return 0 if total == 0 else counted * 100 / total
+    return counted
+
+
+def main():
+    spark = SparkSession.builder.getOrCreate()
+    for statement in STATEMENTS:
+        spark.sql(statement)
+    failures = []
+    for check in CHECKS:
+        measured = measure(spark, check)
+        if passes(check["operator"], check["value"], measured):
+            continue
+        message = f'{check["table"]}: {check["name"]} measured {measured}, expected {check["operator"]} {check["value"]}'
+        if check["action"] == "fail":
+            failures.append(message)
+        else:
+            print(f"Warning: {message}")
+    if failures:
+        raise RuntimeError("Table checks failed:\n" + "\n".join(failures))
+
+
+main()

package/templates/change-data.sql.liquid

@@ -0,0 +1,20 @@
+-- {{ header }}
+-- Changes as received, each read once.
+CREATE OR REFRESH STREAMING TABLE {{ cdc.changes }}
+AS {% render "read-files", source: source %};
+
+-- The current rows{% if cdc.scd == 2 %}, with history{% endif %}, kept by key.
+{% render "table", table: table %};
+
+CREATE FLOW {{ cdc.flow }}
+AS AUTO CDC INTO {{ table.target }}
+FROM STREAM({{ cdc.changes }})
+KEYS ({{ cdc.keys }})
+{%- if cdc.delete_when != "" %}
+APPLY AS DELETE WHEN {{ cdc.delete_when }}
+{%- endif %}
+SEQUENCE BY {{ cdc.sequence }}
+{%- if cdc.except_columns != "" %}
+COLUMNS * EXCEPT ({{ cdc.except_columns }})
+{%- endif %}
+STORED AS SCD TYPE {{ cdc.scd }};

package/templates/materialised-view.py.liquid

@@ -0,0 +1,28 @@
+# {{ header }}
+from pyspark import pipelines as dp
+
+# The transform's own code, unchanged:
+{{ code }}
+
+
+@dp.materialized_view(
+    name={{ name }},
+{%- if comment != "" %}
+    comment={{ comment }},
+{%- endif %}
+{%- if cluster != "" %}
+    {{ cluster }},
+{%- endif %}
+{%- if properties != "" %}
+    table_properties={{ properties }},
+{%- endif %}
+)
+{%- for group in expectations %}
+@dp.{{ group.decorator }}({{ group.checks }})
+{%- endfor %}
+def {{ wrapper }}():
+    return {{ function }}({
+{%- for input in inputs %}
+        {{ input.name }}: spark.read.table({{ input.table }}),
+{%- endfor %}
+    })

package/templates/materialised-view.sql.liquid

@@ -0,0 +1,4 @@
+-- {{ header }}
+{% render "table", table: table %}
+AS
+{% if query != "" %}{{ query }}{% else %}{% render "read-files", source: source %}{% endif %};

package/templates/read-files.liquid

@@ -0,0 +1,7 @@
+{%- comment %} Ends without a newline: the caller continues after it. {% endcomment -%}
+SELECT {{ source.select }} FROM {% if source.stream %}STREAM {% endif %}read_files(
+  {{ source.path }}
+{%- for option in source.options %},
+  {{ option.key }} => {{ option.value }}
+{%- endfor %}
+){%- comment %}{% endcomment -%}

package/templates/streaming-table.sql.liquid

@@ -0,0 +1,3 @@
+-- {{ header }}
+{% render "table", table: table %}
+AS {% render "read-files", source: source %};

package/templates/table.liquid

@@ -0,0 +1,17 @@
+{%- comment %} Ends without a newline: the caller continues after it. {% endcomment -%}
+CREATE OR REFRESH {{ table.kind }} {{ table.target }}
+{%- if table.constraints.size > 0 %} (
+{%- for constraint in table.constraints %}
+  CONSTRAINT {{ constraint.name }} EXPECT ({{ constraint.check }}){{ constraint.on_violation }}{% unless forloop.last %},{% endunless %}
+{%- endfor %}
+)
+{%- endif %}
+{%- if table.comment != "" %}
+COMMENT {{ table.comment }}
+{%- endif %}
+{%- if table.cluster != "" %}
+CLUSTER BY {{ table.cluster }}
+{%- endif %}
+{%- if table.properties.size > 0 %}
+TBLPROPERTIES ({% for property in table.properties %}{{ property.key }} = {{ property.value }}{% unless forloop.last %}, {% endunless %}{% endfor %})
+{%- endif -%}