ingestron 0.17.8 → 1.1.0

package/schemas/platform.schema.yaml

@@ -0,0 +1,260 @@
+$schema: https://json-schema.org/draft/2020-12/schema
+title: Platform
+description: >-
+  Where this project's assets go. Usually differs per environment through `${values.…}`. Optional:
+  without it, the pack's defaults apply.
+type: object
+additionalProperties: false
+required: [apiVersion, kind, id]
+properties:
+  apiVersion: { const: ingestron.io/v1, description: Always ingestron.io/v1. }
+  kind: { const: Platform, description: The document kind. }
+  id:
+    {
+      type: string,
+      pattern: "^[a-z][a-z0-9_-]*$",
+      description: "The platform's id: lowercase letters, digits, underscores and hyphens.",
+      examples: [lakehouse],
+    }
+  description:
+    {
+      type: string,
+      description: "What this document is for, in a sentence.",
+      examples: [Where the sales project's assets go.],
+    }
+  workspace:
+    type: object
+    additionalProperties: false
+    properties:
+      host:
+        {
+          type: string,
+          description: "Workspace URL. Leave out to use the deploying user's profile.",
+          examples: [https://adb-1111111111111111.1.azuredatabricks.net],
+        }
+      root_path:
+        {
+          type: string,
+          description: Folder in the workspace for deployed files.,
+          examples: [/Workspace/Shared/.bundle/sales],
+        }
+    description: The Databricks workspace host and root path.
+  mode:
+    enum: [development, production]
+    description: "Default: production for environments named prod or production, otherwise development."
+    enumDescriptions:
+      - "A copy for development: names carry the deploying user and schedules are paused."
+      - The shared resources, as they run in production.
+  run_as:
+    type: object
+    additionalProperties: false
+    minProperties: 1
+    maxProperties: 1
+    properties:
+      service_principal_name:
+        {
+          type: string,
+          description: Application id of the service principal the pipelines and jobs run as.,
+          examples: [7f3c2a10-0000-4000-8000-000000000001],
+        }
+      user_name:
+        {
+          type: string,
+          description: The user the pipelines and jobs run as.,
+          examples: [ingest@example.com],
+        }
+    description: The identity the pipelines and jobs run as.
+  compute:
+    type: object
+    additionalProperties: false
+    properties:
+      serverless: { type: boolean, description: "Default: true." }
+      budget_policy_id:
+        {
+          type: string,
+          description: Serverless budget policy that pipelines are charged to.,
+          examples: [5b1e3f2a-0000-4000-8000-000000000002],
+        }
+      channel:
+        {
+          enum: [CURRENT, PREVIEW],
+          description: "Lakeflow pipeline channel: CURRENT or PREVIEW.",
+          enumDescriptions:
+            [
+              "The current Lakeflow runtime, for production.",
+              "The next runtime, to try upcoming changes; some connectors need it.",
+            ],
+        }
+    description: Serverless or classic compute, budget policy and channel.
+  settings:
+    type: object
+    additionalProperties: { type: object }
+    description: >-
+      Method settings for this platform, by method, over the standard's: where shared infrastructure
+      such as landing storage is, usually through `${values.…}`.
+    examples: [{ adf-files: { landing: "${values.landing}", key_vault: "${values.key_vault}" } }]
+  naming:
+    type: object
+    additionalProperties: false
+    description: Name patterns for this project, over the standard's.
+    properties:
+      catalog:
+        {
+          type: string,
+          description: "Name pattern for the catalog. Default: `the standard's`.",
+          examples: ["sales_{{ environment }}"],
+        }
+      schema:
+        {
+          type: string,
+          description: "Name pattern for the schema. Default: `the standard's`.",
+          examples: ["{{ layer }}"],
+        }
+      table:
+        {
+          type: string,
+          description: "Name pattern for the table. Default: `the standard's`.",
+          examples: ["{{ table }}"],
+        }
+      pipeline:
+        {
+          type: string,
+          description: "Name pattern for the pipeline. Default: `the standard's`.",
+          examples: ["{{ project }}"],
+        }
+      job:
+        {
+          type: string,
+          description: "Name pattern for the refresh job. Default: `the standard's`.",
+          examples: ["{{ project }}_refresh"],
+        }
+  tags:
+    {
+      type: object,
+      additionalProperties: { type: string },
+      description: Tags added to pipelines and jobs.,
+    }
+  permissions:
+    type: array
+    items:
+      type: object
+      additionalProperties: false
+      required: [level]
+      minProperties: 2
+      maxProperties: 2
+      properties:
+        level:
+          {
+            enum: [CAN_VIEW, CAN_RUN, CAN_MANAGE],
+            description: "CAN_VIEW, CAN_RUN or CAN_MANAGE.",
+            enumDescriptions:
+              [
+                See the pipelines and jobs and their runs.,
+                Also start and stop runs.,
+                "Also change and delete them, and their permissions.",
+              ],
+          }
+        group_name:
+          {
+            type: string,
+            description: "A group, or use user_name or service_principal_name instead.",
+            examples: [data-engineers],
+          }
+        user_name:
+          {
+            type: string,
+            description: "A user, or use group_name or service_principal_name instead.",
+            examples: [ana@example.com],
+          }
+        service_principal_name:
+          {
+            type: string,
+            description: "A service principal's application id, or use group_name or user_name instead.",
+            examples: [7f3c2a10-0000-4000-8000-000000000001],
+          }
+    description: Who can view, run or manage the bundle's resources.
+    examples:
+      - - group_name: data-engineers
+          level: CAN_MANAGE
+        - group_name: analysts
+          level: CAN_VIEW
+  schedule:
+    type: object
+    additionalProperties: false
+    required: [cron]
+    properties:
+      cron:
+        {
+          type: string,
+          description: "Quartz cron expression, such as `0 0 6 * * ?`.",
+          examples: [0 0 5 * * ?],
+        }
+      timezone: { type: string, description: "Default: UTC.", examples: [Pacific/Auckland] }
+      paused: { type: boolean, description: "Default: paused outside production." }
+    description: When the refresh job runs (Quartz cron), its time zone, and whether it is paused.
+    examples:
+      - cron: 0 0 5 * * ?
+        timezone: Pacific/Auckland
+  notifications:
+    type: object
+    additionalProperties: false
+    properties:
+      on_failure:
+        {
+          type: array,
+          items: { type: string },
+          description: Email addresses told when a pipeline or job fails.,
+          examples: [[data-team@example.com]],
+        }
+    description: Who is emailed when an update fails.
+  manage:
+    type: object
+    additionalProperties: false
+    properties:
+      schemas:
+        type: boolean
+        description:
+          "Create the schemas (with grants) in the bundle. Default: false, because schemas are
+          often shared infrastructure."
+      tags:
+        type: boolean
+        description:
+          "Apply the contracts' tags and classification to tables and columns as Unity Catalog tags,
+          after each refresh. Default: false."
+      comments:
+        type: boolean
+        description: "Apply the contracts' column descriptions as column comments, after each refresh. Default: false."
+      checks:
+        type: boolean
+        description:
+          "Run the contracts' rules that compare counts over all rows, such as row counts and
+          duplicates, after each refresh. Default: false."
+    description: Shared resources and metadata this bundle manages. Each needs privileges on the platform.
+  grants:
+    type: object
+    description: Grants per layer, used when `manage.schemas` is true.
+    additionalProperties:
+      type: array
+      items:
+        type: object
+        additionalProperties: false
+        required: [principal, privileges]
+        properties:
+          principal:
+            {
+              type: string,
+              description: "The user, group or service principal given the privileges.",
+              examples: [data-engineers],
+            }
+          privileges:
+            {
+              type: array,
+              items: { type: string },
+              description: "Unity Catalog privileges, such as USE_SCHEMA, SELECT and MODIFY.",
+            }
+    examples:
+      - bronze:
+          - principal: data-engineers
+            privileges:
+              - USE_SCHEMA
+              - SELECT

package/schemas/standard.schema.yaml

@@ -0,0 +1,211 @@
+$schema: https://json-schema.org/draft/2020-12/schema
+title: Standard
+description: >-
+  How things are done, shared by many projects: methods per action, naming, layers, quality and
+  types. Anything left out uses the pack's defaults.
+type: object
+additionalProperties: false
+required: [apiVersion, kind, id, packs]
+properties:
+  apiVersion: { const: ingestron.io/v1, description: Always ingestron.io/v1. }
+  kind: { const: Standard, description: The document kind. }
+  id:
+    {
+      type: string,
+      pattern: "^[a-z][a-z0-9_-]*$",
+      description: "The standard's id: lowercase letters, digits, underscores and hyphens.",
+      examples: [acme],
+    }
+  description:
+    {
+      type: string,
+      description: "What this document is for, in a sentence.",
+      examples: [How Acme builds its sales data products.],
+    }
+  packs:
+    type: array
+    minItems: 1
+    uniqueItems: true
+    items: { type: string, pattern: "^[a-z][a-z0-9-]*@[0-9]+$" }
+    description: >-
+      The packs to use, each with its major version, such as `[databricks@1, adf@1]`. The first
+      provides the defaults.
+  naming:
+    {
+      $ref: "#/$defs/naming",
+      description: "Name patterns for catalogs, schemas, tables, pipelines and jobs.",
+    }
+  layers:
+    type: array
+    minItems: 1
+    uniqueItems: true
+    items: { type: string, pattern: "^[a-z][a-z0-9_]*$" }
+    description: "Layers in order. Flow tables land in the first, transforms in the second. Default:
+      bronze, silver, gold."
+  types:
+    type: object
+    description:
+      "Platform type for each contract `logicalType`, such as `number: DECIMAL(38,6)`. Used
+      when a column has no `physicalType`."
+    additionalProperties: { type: string }
+  quality:
+    type: object
+    additionalProperties: false
+    properties:
+      required:
+        enum: [warn, drop, fail]
+        description: "Action for rows missing a contract `required` column. Default: warn."
+        enumDescriptions:
+          - Keep the row and record that it failed.
+          - Leave the row out of the table.
+          - Stop the update.
+      default:
+        enum: [warn, drop, fail]
+        description: "Action for expectations that name none. Default: warn."
+        enumDescriptions:
+          - Keep the row and record that it failed.
+          - Leave the row out of the table.
+          - Stop the update.
+    description: "What happens to rows that fail a check: required columns and your own expectations."
+  ingest:
+    {
+      $ref: "#/$defs/rules",
+      description: "How tables are ingested: method settings, rules, and allowed and fixed settings.",
+    }
+  transform:
+    {
+      $ref: "#/$defs/rules",
+      description: "How transforms run: method settings, rules, and allowed and fixed settings.",
+    }
+  store:
+    {
+      $ref: "#/$defs/rules",
+      description: "How tables are stored: method settings, rules, and allowed and fixed settings.",
+    }
+$defs:
+  method: { type: string, pattern: "^[a-z][a-z0-9-]*$" }
+  naming:
+    type: object
+    additionalProperties: false
+    description: Name patterns (Liquid) with the variables project, environment, layer, table, flow and source.
+    properties:
+      catalog:
+        {
+          type: string,
+          description: "Name pattern for the catalog. Default: `{{ environment }}`.",
+          examples: ["acme_{{ environment }}"],
+        }
+      schema:
+        {
+          type: string,
+          description: "Name pattern for the schema. Default: `{{ layer }}`.",
+          examples: ["{{ environment }}_{{ layer }}"],
+        }
+      table:
+        {
+          type: string,
+          description: "Name pattern for the table. Default: `{{ table }}`.",
+          examples: ["{% if source %}{{ source }}_{% endif %}{{ table }}"],
+        }
+      pipeline:
+        {
+          type: string,
+          description: "Name pattern for the pipeline. Default: `{{ project }}`.",
+          examples: ["{{ project }}_{{ environment }}"],
+        }
+      job:
+        {
+          type: string,
+          description: "Name pattern for the refresh job. Default: `{{ project }}_refresh`.",
+          examples: ["{{ project }}_daily"],
+        }
+  rules:
+    type: object
+    additionalProperties: false
+    properties:
+      default:
+        $ref: "#/$defs/method"
+        description: "Replaces the pack's default method for this action. For store, none ends flows at the files ingest lands."
+        examples:
+          - materialised-view
+      allowed:
+        type: array
+        items: { $ref: "#/$defs/method" }
+        description: "Methods a flow or transform may name. Default: all of the pack's methods for this action."
+        examples:
+          - - streaming-table
+            - materialised-view
+      when:
+        type: array
+        description:
+          Checked in order, before the pack's own rules; the first whose facts all match chooses
+          the method.
+        items:
+          type: object
+          additionalProperties: false
+          required: [use]
+          minProperties: 2
+          properties:
+            connectivity:
+              {
+                enum: [public, workspace, private],
+                description: "Matches the connection's connectivity: public, workspace or private.",
+                enumDescriptions: [
+                    The data platform reaches the source directly.,
+                    "The platform reaches it over private networking, such as ExpressRoute or a
+                    peered network.",
+                    The platform cannot reach it; something inside the source's network must bring
+                    the data.,
+                  ],
+              }
+            connector:
+              {
+                type: string,
+                description: "Matches the connection's connector, such as files or sqlserver.",
+                examples: [sqlserver],
+              }
+            input:
+              {
+                enum: [files, table, query],
+                description: "Matches what the previous action hands over: files, table or query.",
+                enumDescriptions:
+                  [
+                    "Files in storage, such as those an ingest method lands.",
+                    "A table, such as one a managed connector writes, or a transform's inputs.",
+                    "A query, such as a transform's SQL, for the store method to keep.",
+                  ],
+              }
+            layer:
+              {
+                type: string,
+                description: "Matches the table's layer, such as gold.",
+                examples: [gold],
+              }
+            use:
+              {
+                $ref: "#/$defs/method",
+                description: "The method to use when this rule matches, such as materialised-view.",
+                examples: [materialised-view],
+              }
+            with: { type: object, description: Settings for this case. }
+        examples:
+          - - connector: sqlserver
+              use: lakeflow-connect
+            - layer: gold
+              use: materialised-view
+      settings:
+        type: object
+        description: Settings per method, over the pack's defaults.
+        additionalProperties: { type: object }
+        examples:
+          - files:
+              format: csv
+              options:
+                header: true
+      fixed:
+        type: object
+        description: Settings per method that nothing may change.
+        additionalProperties: { type: object }
+        examples:
+          - streaming-table:
+              cluster_by: auto

package/schemas/transform.schema.yaml

@@ -0,0 +1,108 @@
+$schema: https://json-schema.org/draft/2020-12/schema
+title: Transform
+description: Builds one table from tables that flows or other transforms produce.
+type: object
+additionalProperties: false
+required: [apiVersion, kind, id, inputs, code, contract]
+properties:
+  apiVersion: { const: ingestron.io/v1, description: Always ingestron.io/v1. }
+  kind: { const: Transform, description: The document kind. }
+  id:
+    type: string
+    pattern: "^[a-z][a-z0-9_]*$"
+    description: Name of the table this transform produces.
+    examples:
+      - daily_sales
+  description:
+    {
+      type: string,
+      description: "What this document is for, in a sentence.",
+      examples: [Order totals per customer and day.],
+    }
+  layer:
+    {
+      type: string,
+      description: "Layer of the result. Default: the standard's second layer.",
+      examples: [silver],
+    }
+  inputs:
+    type: array
+    minItems: 1
+    uniqueItems: true
+    items: { type: string, pattern: "^[a-z][a-z0-9_]*$" }
+    description: Tables the code reads, by table name.
+  code:
+    type: string
+    description: >-
+      The code, usually `${file:name.sql}` or `${file:name.py}`. In SQL, write an input as `{{
+      tables.orders }}`; it becomes the input's full name. The method (`transform.uses`) says which
+      language it is.
+    examples:
+      - ${file:daily_sales.sql}
+  contract:
+    {
+      type: string,
+      description: Id of the DataContract (ODCS) for the result.,
+      examples: [shop.daily_sales],
+    }
+  expectations:
+    type: array
+    items:
+      type: object
+      additionalProperties: false
+      required: [name, check]
+      properties:
+        name:
+          {
+            type: string,
+            pattern: "^[a-z][a-z0-9_]*$",
+            description: "A name for the check: lowercase letters, digits and underscores.",
+            examples: [positive_total],
+          }
+        check:
+          {
+            type: string,
+            description: "A SQL condition every valid row meets, such as `total >= 0`.",
+            examples: [total >= 0],
+          }
+        action:
+          {
+            enum: [warn, drop, fail],
+            description: "What happens to rows that fail: warn, drop or fail. Default: the standard's.",
+            enumDescriptions:
+              [
+                Keep the row and record that it failed.,
+                Leave the row out of the table.,
+                Stop the update.,
+              ],
+          }
+    description: Your own checks on the table's rows.
+  transform: { $ref: "#/$defs/transform", description: How the code runs. }
+  store: { $ref: "#/$defs/store", description: How the result is kept. }
+$defs:
+  transform:
+    type: object
+    additionalProperties: false
+    properties:
+      uses:
+        type: string
+        pattern: ^[a-z][a-z0-9-]*$
+        description: The method for this step, instead of the standard's choice, such as pyspark.
+        examples:
+          - sql
+      with:
+        type: object
+        description: Settings for the method, over the standard's.
+  store:
+    type: object
+    additionalProperties: false
+    properties:
+      uses:
+        type: string
+        pattern: ^[a-z][a-z0-9-]*$
+        description: The method for this step, instead of the standard's choice, such as pyspark.
+        examples:
+          - materialised-view
+      with:
+        type: object
+        description: Settings for the method, over the standard's.

package/templates/apply-contract.py.liquid

@@ -0,0 +1,58 @@
+# {{ header }}
+# Runs after each refresh: Lakeflow pipelines cannot set tags or column
+# comments, nor compare counts over all rows, so this task does.
+import json
+
+from pyspark.sql import SparkSession
+
+STATEMENTS = json.loads(r"""{{ statements }}""")
+CHECKS = json.loads(r"""{{ checks }}""")
+
+
+def passes(operator, expected, measured):
+    """True when the measured count meets the rule's comparison."""
+    if operator == "mustBe":
+        return measured == expected
+    if operator == "mustNotBe":
+        return measured != expected
+    if operator == "mustBeGreaterThan":
+        return measured > expected
+    if operator == "mustBeGreaterOrEqualTo":
+        return measured >= expected
+    if operator == "mustBeLessThan":
+        return measured < expected
+    if operator == "mustBeLessOrEqualTo":
+        return measured <= expected
+    low, high = expected
+    inside = low <= measured <= high
+    return inside if operator == "mustBeBetween" else not inside
+
+
+def measure(spark, check):
+    """The rule's count, in rows or as a percentage of all rows."""
+    row = spark.sql(check["query"]).first()
+    counted, total = row["counted"] or 0, row["total"] or 0
+    if check["unit"] == "percent":
+        return 0 if total == 0 else counted * 100 / total
+    return counted
+
+
+def main():
+    spark = SparkSession.builder.getOrCreate()
+    for statement in STATEMENTS:
+        spark.sql(statement)
+    failures = []
+    for check in CHECKS:
+        measured = measure(spark, check)
+        if passes(check["operator"], check["value"], measured):
+            continue
+        message = f'{check["table"]}: {check["name"]} measured {measured}, expected {check["operator"]} {check["value"]}'
+        if check["action"] == "fail":
+            failures.append(message)
+        else:
+            print(f"Warning: {message}")
+    if failures:
+        raise RuntimeError("Table checks failed:\n" + "\n".join(failures))
+
+
+main()

package/templates/change-data.sql.liquid

@@ -0,0 +1,20 @@
+-- {{ header }}
+-- Changes as received, each read once.
+CREATE OR REFRESH STREAMING TABLE {{ cdc.changes }}
+AS {% render "read-files", source: source %};
+
+-- The current rows{% if cdc.scd == 2 %}, with history{% endif %}, kept by key.
+{% render "table", table: table %};
+
+CREATE FLOW {{ cdc.flow }}
+AS AUTO CDC INTO {{ table.target }}
+FROM STREAM({{ cdc.changes }})
+KEYS ({{ cdc.keys }})
+{%- if cdc.delete_when != "" %}
+APPLY AS DELETE WHEN {{ cdc.delete_when }}
+{%- endif %}
+SEQUENCE BY {{ cdc.sequence }}
+{%- if cdc.except_columns != "" %}
+COLUMNS * EXCEPT ({{ cdc.except_columns }})
+{%- endif %}
+STORED AS SCD TYPE {{ cdc.scd }};

package/templates/materialised-view.py.liquid

@@ -0,0 +1,28 @@
+# {{ header }}
+from pyspark import pipelines as dp
+
+# The transform's own code, unchanged:
+{{ code }}
+
+
+@dp.materialized_view(
+    name={{ name }},
+{%- if comment != "" %}
+    comment={{ comment }},
+{%- endif %}
+{%- if cluster != "" %}
+    {{ cluster }},
+{%- endif %}
+{%- if properties != "" %}
+    table_properties={{ properties }},
+{%- endif %}
+)
+{%- for group in expectations %}
+@dp.{{ group.decorator }}({{ group.checks }})
+{%- endfor %}
+def {{ wrapper }}():
+    return {{ function }}({
+{%- for input in inputs %}
+        {{ input.name }}: spark.read.table({{ input.table }}),
+{%- endfor %}
+    })

package/templates/materialised-view.sql.liquid

@@ -0,0 +1,4 @@
+-- {{ header }}
+{% render "table", table: table %}
+AS
+{% if query != "" %}{{ query }}{% else %}{% render "read-files", source: source %}{% endif %};

package/templates/read-files.liquid

@@ -0,0 +1,7 @@
+{%- comment %} Ends without a newline: the caller continues after it. {% endcomment -%}
+SELECT {{ source.select }} FROM {% if source.stream %}STREAM {% endif %}read_files(
+  {{ source.path }}
+{%- for option in source.options %},
+  {{ option.key }} => {{ option.value }}
+{%- endfor %}
+){%- comment %}{% endcomment -%}