ingestron 0.17.8 → 1.1.0

package/project-templates/pyspark/transforms/monthly_revenue.py

@@ -0,0 +1,10 @@
+from pyspark.sql import functions as F
+
+
+def transform(tables):
+    invoices = tables["invoices"]
+    return (
+        invoices.where(F.col("status") == "paid")
+        .groupBy(F.date_trunc("month", "issued_at").alias("month"), "plan")
+        .agg(F.sum("amount").alias("revenue"), F.countDistinct("customer_id").alias("customers"))
+    )

package/project-templates/pyspark/transforms/monthly_revenue.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Transform
+id: monthly_revenue
+inputs: [invoices]
+transform:
+  uses: pyspark
+code: ${file:monthly_revenue.py}
+contract: billing.monthly_revenue
+expectations:
+  - { name: revenue_not_negative, check: revenue >= 0, action: fail }

package/project-templates/retail/README.md

@@ -0,0 +1,22 @@
+# Retail example
+
+A complete project: web shop files brought into bronze, orders kept current
+from change records, and a silver table of daily sales.
+
+| File                          | What it shows                                                      |
+| ----------------------------- | ------------------------------------------------------------------ |
+| `project.yaml`                | Values that differ between `dev` and `prod`                        |
+| `standard.yaml`               | The organisation's catalog naming, quality and a fixed setting     |
+| `platform.yaml`               | Workspace, identity, tags, permissions, schedule and notifications |
+| `connections/shop.yaml`       | A files source whose path differs per environment                  |
+| `flows/shop.yaml`             | Change data (`mode: upsert`), CSV options and an expectation       |
+| `transforms/daily_sales.yaml` | SQL in its own file, reading two flow tables                       |
+| `contracts/*.yaml`            | ODCS contracts: types, keys and descriptions                       |
+
+```sh
+pnpm ingestron plan --definitions examples/08-retail --out examples/08-retail/build
+```
+
+The generated files for both environments are in [build/](build/).
+For editor help, run `ingestron schemas` in this folder and add the printed
+setting to VS Code.

package/project-templates/retail/connections/shop.yaml

@@ -0,0 +1,8 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: shop
+description: Files the web shop exports every hour.
+connector: files
+connectivity: public
+settings:
+  path: ${values.storage}/shop

package/project-templates/retail/contracts/customers.yaml

@@ -0,0 +1,12 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: shop.customers
+version: 1.0.0
+status: active
+schema:
+  - name: customers
+    logicalType: object
+    description: Web shop customers.
+    properties:
+      - { name: id, logicalType: integer, required: true, primaryKey: true }
+      - { name: email, logicalType: string, required: true }

package/project-templates/retail/contracts/daily_sales.yaml

@@ -0,0 +1,19 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: shop.daily_sales
+version: 1.0.0
+status: active
+schema:
+  - name: daily_sales
+    logicalType: object
+    description: Order totals per customer and day.
+    properties:
+      - {
+          name: email,
+          logicalType: string,
+          required: true,
+          primaryKey: true,
+          primaryKeyPosition: 1,
+        }
+      - { name: day, logicalType: date, required: true, primaryKey: true, primaryKeyPosition: 2 }
+      - { name: total, logicalType: number, physicalType: "DECIMAL(18,2)" }

package/project-templates/retail/contracts/orders.yaml

@@ -0,0 +1,16 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: shop.orders
+version: 1.0.0
+status: active
+schema:
+  - name: orders
+    logicalType: object
+    description: One row per order, kept current.
+    properties:
+      - { name: id, logicalType: integer, required: true, primaryKey: true }
+      - { name: customer_id, logicalType: integer, required: true }
+      - { name: ordered_at, logicalType: timestamp, required: true }
+      - { name: total, logicalType: number, physicalType: "DECIMAL(18,2)" }
+      - { name: updated_at, logicalType: timestamp, required: true }
+      - { name: op, logicalType: string }

package/project-templates/retail/flows/shop.yaml

@@ -0,0 +1,18 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: shop
+description: Orders and customers from the web shop.
+source: shop
+tables:
+  orders:
+    object: orders
+    contract: shop.orders
+    store:
+      with: { mode: upsert, sequence_by: updated_at, delete_when: "op = 'D'", except_columns: [op] }
+    expectations:
+      - { name: positive_total, check: total >= 0 }
+  customers:
+    object: customers
+    contract: shop.customers
+    ingest:
+      with: { format: csv, options: { header: true } }

package/project-templates/retail/platform.yaml

@@ -0,0 +1,21 @@
+# Where this project's assets go. Values differ per environment.
+apiVersion: ingestron.io/v1
+kind: Platform
+id: lakehouse
+workspace:
+  host: ${values.host}
+run_as:
+  service_principal_name: 7f3c2a10-0000-4000-8000-000000000001
+tags:
+  team: sales
+  environment: ${environment.name}
+permissions:
+  - group_name: sales-engineers
+    level: CAN_MANAGE
+  - group_name: sales-analysts
+    level: CAN_VIEW
+schedule:
+  cron: 0 0 5 * * ?
+  timezone: Pacific/Auckland
+notifications:
+  on_failure: [sales-data@example.com]

package/project-templates/retail/project.yaml

@@ -0,0 +1,18 @@
+id: retail
+name: Retail sales
+values:
+  host: https://adb-1111111111111111.1.azuredatabricks.net
+  storage: abfss://landing@retaildev.dfs.core.windows.net
+environments:
+  dev: {}
+  prod:
+    values:
+      host: https://adb-2222222222222222.2.azuredatabricks.net
+      storage: abfss://landing@retailprod.dfs.core.windows.net
+imports:
+  - standard.yaml
+  - platform.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - transforms/*.yaml
+  - contracts/*.yaml

package/project-templates/retail/standard.yaml

@@ -0,0 +1,15 @@
+# How the organisation builds data products. Shared by every project.
+apiVersion: ingestron.io/v1
+kind: Standard
+id: acme
+packs: [databricks@1]
+naming:
+  catalog: "{{ environment }}_sales"
+quality:
+  required: drop
+store:
+  settings:
+    streaming-table:
+      properties: { delta.enableChangeDataFeed: "true" }
+  fixed:
+    streaming-table: { cluster_by: auto }

package/project-templates/retail/template.yaml

@@ -0,0 +1,10 @@
+# Used by `ingestron init --template retail`.
+title: Retail
+summary: "A complete project: files into bronze, orders kept current from change records, and daily sales in silver."
+questions:
+  - value: host
+    ask: "Databricks workspace URL"
+    default: https://adb-1111111111111111.1.azuredatabricks.net
+  - value: storage
+    ask: "Storage account URL for the landing files"
+    default: abfss://landing@retaildev.dfs.core.windows.net

package/project-templates/retail/transforms/daily_sales.sql

@@ -0,0 +1,7 @@
+SELECT
+  c.email,
+  DATE(o.ordered_at) AS day,
+  SUM(o.total) AS total
+FROM {{ tables.orders }} AS o
+JOIN {{ tables.customers }} AS c ON c.id = o.customer_id
+GROUP BY c.email, DATE(o.ordered_at)

package/project-templates/retail/transforms/daily_sales.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Transform
+id: daily_sales
+description: Order totals per customer and day.
+inputs: [orders, customers]
+code: ${file:daily_sales.sql}
+contract: shop.daily_sales

package/project-templates/salesforce/README.md

@@ -0,0 +1,6 @@
+# Salesforce
+
+Two Salesforce objects through a Unity Catalog connection. SaaS sources need
+no gateway: [one ingestion pipeline](build/dev/resources/crm_salesforce_ingestion.pipeline.yml)
+writes `accounts` and `opportunities`, keyed by `Id`, with only the
+contract's fields.

package/project-templates/salesforce/connections/salesforce.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: salesforce
+connector: salesforce
+connectivity: public
+settings:
+  uc_connection: salesforce_prod

package/project-templates/salesforce/contracts/accounts.yaml

@@ -0,0 +1,13 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: crm.accounts
+version: 1.0.0
+status: active
+schema:
+  - name: accounts
+    logicalType: object
+    description: Salesforce accounts.
+    properties:
+      - { name: Id, logicalType: string, required: true, primaryKey: true }
+      - { name: Name, logicalType: string }
+      - { name: Industry, logicalType: string }

package/project-templates/salesforce/contracts/opportunities.yaml

@@ -0,0 +1,14 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: crm.opportunities
+version: 1.0.0
+status: active
+schema:
+  - name: opportunities
+    logicalType: object
+    description: Salesforce opportunities.
+    properties:
+      - { name: Id, logicalType: string, required: true, primaryKey: true }
+      - { name: AccountId, logicalType: string }
+      - { name: Amount, logicalType: number }
+      - { name: StageName, logicalType: string }

package/project-templates/salesforce/flows/salesforce.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: salesforce
+source: salesforce
+tables:
+  accounts: { object: Account, contract: crm.accounts }
+  opportunities: { object: Opportunity, contract: crm.opportunities }

package/project-templates/salesforce/project.yaml

@@ -0,0 +1,9 @@
+# Salesforce objects ingested with Lakeflow Connect; no gateway is needed.
+id: crm
+environments:
+  dev: {}
+imports:
+  - standard.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - contracts/*.yaml

package/project-templates/salesforce/standard.yaml

@@ -0,0 +1,4 @@
+apiVersion: ingestron.io/v1
+kind: Standard
+id: crm
+packs: [databricks@1]

package/project-templates/salesforce/template.yaml

@@ -0,0 +1,3 @@
+# Used by `ingestron init --template salesforce`.
+title: Salesforce through Lakeflow Connect
+summary: Two Salesforce objects through a Unity Catalog connection, keyed and limited to the contract's fields.

package/project-templates/sql-server/README.md

@@ -0,0 +1,17 @@
+# SQL Server
+
+The ERP database is reachable from the workspace over private networking
+(`connectivity: workspace`), so the pack ingests it with Lakeflow Connect:
+
+- [a gateway](build/prod/resources/erp_erp_gateway.pipeline.yml) that captures
+  changes continuously, with the cluster policy from the standard;
+- [an ingestion pipeline](build/prod/resources/erp_erp_ingestion.pipeline.yml)
+  that writes `customers` (SCD type 1) and `invoices` (SCD type 2, from
+  `mode: history`), each with only its contract's columns;
+- [our pipeline](build/prod/src/erp/silver/customer_balances.sql), whose
+  transform reads both, using the current invoice versions;
+- [a job](build/prod/resources/erp_refresh.job.yml) that refreshes ingestion
+  first and our pipeline after it.
+
+The Unity Catalog connection `erp_sqlserver_<environment>` must exist; the
+definitions name it but hold no credentials.

package/project-templates/sql-server/connections/erp.yaml

@@ -0,0 +1,8 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: erp
+description: The ERP's SQL Server, reachable from the workspace through ExpressRoute.
+connector: sqlserver
+connectivity: workspace
+settings:
+  uc_connection: erp_sqlserver_${environment.name} # created by an administrator beforehand

package/project-templates/sql-server/contracts/customer_balances.yaml

@@ -0,0 +1,13 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: erp.customer_balances
+version: 1.0.0
+status: active
+schema:
+  - name: customer_balances
+    logicalType: object
+    description: Open balance per customer.
+    properties:
+      - { name: customer_id, logicalType: integer, required: true, primaryKey: true }
+      - { name: name, logicalType: string }
+      - { name: balance, logicalType: number, physicalType: "DECIMAL(18,2)" }

package/project-templates/sql-server/contracts/customers.yaml

@@ -0,0 +1,13 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: erp.customers
+version: 1.0.0
+status: active
+schema:
+  - name: customers
+    logicalType: object
+    description: ERP customers.
+    properties:
+      - { name: customer_id, logicalType: integer, required: true, primaryKey: true }
+      - { name: name, logicalType: string }
+      - { name: country, logicalType: string }

package/project-templates/sql-server/contracts/invoices.yaml

@@ -0,0 +1,14 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: erp.invoices
+version: 1.0.0
+status: active
+schema:
+  - name: invoices
+    logicalType: object
+    description: ERP invoices with every past version.
+    properties:
+      - { name: invoice_id, logicalType: integer, required: true, primaryKey: true }
+      - { name: customer_id, logicalType: integer, required: true }
+      - { name: amount, logicalType: number, physicalType: "DECIMAL(18,2)" }
+      - { name: status, logicalType: string }

package/project-templates/sql-server/flows/erp.yaml

@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: erp
+source: erp
+tables:
+  customers:
+    object: erp.dbo.Customers
+    contract: erp.customers
+  invoices:
+    object: erp.dbo.Invoices
+    contract: erp.invoices
+    ingest:
+      with: { mode: history } # keep every version of an invoice

package/project-templates/sql-server/platform.yaml

@@ -0,0 +1,6 @@
+apiVersion: ingestron.io/v1
+kind: Platform
+id: lakehouse
+schedule:
+  cron: 0 0 * * * ?
+  timezone: Pacific/Auckland

package/project-templates/sql-server/project.yaml

@@ -0,0 +1,13 @@
+# A SQL Server database reached over private networking from the workspace,
+# ingested with Lakeflow Connect, then summarised by our own pipeline.
+id: erp
+environments:
+  dev: {}
+  prod: {}
+imports:
+  - standard.yaml
+  - platform.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - transforms/*.yaml
+  - contracts/*.yaml

package/project-templates/sql-server/standard.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Standard
+id: erp
+packs: [databricks@1]
+naming:
+  catalog: erp_{{ environment }}
+ingest:
+  settings:
+    lakeflow-connect: { gateway_policy_id: 0011AABBCCDDEEFF }

package/project-templates/sql-server/template.yaml

@@ -0,0 +1,3 @@
+# Used by `ingestron init --template sql-server`.
+title: SQL Server through Lakeflow Connect
+summary: A gateway and an ingestion pipeline for SQL Server, then the pipeline that reads the ingested tables.

package/project-templates/sql-server/transforms/customer_balances.sql

@@ -0,0 +1,5 @@
+SELECT c.customer_id, c.name, SUM(i.amount) AS balance
+FROM {{ tables.customers }} AS c
+JOIN {{ tables.invoices }} AS i ON i.customer_id = c.customer_id
+WHERE i.__END_AT IS NULL AND i.status = 'open'
+GROUP BY c.customer_id, c.name

package/project-templates/sql-server/transforms/customer_balances.yaml

@@ -0,0 +1,6 @@
+apiVersion: ingestron.io/v1
+kind: Transform
+id: customer_balances
+inputs: [customers, invoices]
+code: ${file:customer_balances.sql}
+contract: erp.customer_balances

package/recipes/connections/adls-gen2.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: adls_gen2
+connector: adls-gen2
+connectivity: private
+settings:
+  url: "https://source.example.test"

package/recipes/connections/amazon-rds-oracle.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: amazon_rds_oracle
+connector: amazon-rds-oracle
+connectivity: private
+settings:
+  server: "source.example.test"
+  user: "reader"
+  password: "${secret:source-password}"

package/recipes/connections/amazon-rds-sqlserver.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: amazon_rds_sqlserver
+connector: amazon-rds-sqlserver
+connectivity: private
+settings:
+  server: "source.example.test"
+  user: "reader"
+  password: "${secret:source-password}"

package/recipes/connections/amazon-redshift.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: amazon_redshift
+connector: amazon-redshift
+connectivity: private
+settings:
+  server: "source.example.test"
+  database: "sales"
+  user: "reader"
+  password: "${secret:source-password}"

package/recipes/connections/amazon-s3.yaml

@@ -0,0 +1,8 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: amazon_s3
+connector: amazon-s3
+connectivity: private
+settings:
+  access_key: "synthetic-access_key"
+  secret_key: "${secret:source-secret-key}"

package/recipes/connections/azure-blob.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: azure_blob
+connector: azure-blob
+connectivity: private
+settings:
+  url: "https://source.example.test"

package/recipes/connections/azure-cosmos-db-mongodb.yaml

@@ -0,0 +1,8 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: azure_cosmos_db_mongodb
+connector: azure-cosmos-db-mongodb
+connectivity: private
+settings:
+  connection_string: "${secret:source-connection-string}"
+  database: "sales"

package/recipes/connections/azure-cosmos-db-nosql.yaml

@@ -0,0 +1,8 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: azure_cosmos_db_nosql
+connector: azure-cosmos-db-nosql
+connectivity: private
+settings:
+  url: "https://source.example.test"
+  database: "sales"

package/recipes/connections/azure-cosmos-postgresql.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: azure_cosmos_postgresql
+connector: azure-cosmos-postgresql
+connectivity: private
+settings:
+  server: "source.example.test"
+  database: "sales"
+  user: "reader"
+  password: "${secret:source-password}"

package/recipes/connections/azure-data-explorer.yaml

@@ -0,0 +1,8 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: azure_data_explorer
+connector: azure-data-explorer
+connectivity: private
+settings:
+  url: "https://source.example.test"
+  database: "sales"

package/recipes/connections/azure-databricks-delta.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: azure_databricks_delta
+connector: azure-databricks-delta
+connectivity: private
+settings:
+  url: "https://source.example.test"
+  cluster_id: "synthetic-cluster_id"
+  workspace_resource_id: "synthetic-workspace_resource_id"

package/recipes/connections/azure-files.yaml

@@ -0,0 +1,8 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: azure_files
+connector: azure-files
+connectivity: private
+settings:
+  connection_string: "${secret:source-connection-string}"
+  share: "synthetic-share"

package/recipes/connections/azure-mysql.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: azure_mysql
+connector: azure-mysql
+connectivity: private
+settings:
+  connection_string: "${secret:source-connection-string}"

package/recipes/connections/azure-postgresql.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: azure_postgresql
+connector: azure-postgresql
+connectivity: private
+settings:
+  server: "source.example.test"
+  database: "sales"
+  user: "reader"
+  password: "${secret:source-password}"

package/recipes/connections/azure-sql-database.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: azure_sql_database
+connector: azure-sql-database
+connectivity: private
+settings:
+  server: "source.example.test"
+  user: "reader"
+  password: "${secret:source-password}"

package/recipes/connections/azure-sql-managed-instance.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: azure_sql_managed_instance
+connector: azure-sql-managed-instance
+connectivity: private
+settings:
+  server: "source.example.test"
+  user: "reader"
+  password: "${secret:source-password}"

package/recipes/connections/azure-synapse.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: azure_synapse
+connector: azure-synapse
+connectivity: private
+settings:
+  server: "source.example.test"
+  user: "reader"
+  password: "${secret:source-password}"

package/recipes/connections/azure-table-storage.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: azure_table_storage
+connector: azure-table-storage
+connectivity: private
+settings:
+  url: "https://source.example.test"

package/recipes/connections/cassandra.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: cassandra
+connector: cassandra
+connectivity: private
+settings:
+  host: "source.example.test"
+  user: "reader"
+  password: "${secret:source-password}"

package/recipes/connections/dataverse.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: dataverse
+connector: dataverse
+connectivity: private
+settings:
+  url: "https://source.example.test"
+  client_id: "synthetic-client"
+  client_secret: "${secret:source-client-secret}"

package/recipes/connections/db2.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: db2
+connector: db2
+connectivity: private
+settings:
+  connection_string: "${secret:source-connection-string}"