@ingestron/cli 0.0.0-stage → 1.0.0

package/project-templates/private-files/template.yaml

@@ -0,0 +1,10 @@
+# Used by `ingestron init --template private-files`.
+title: Private file share through Azure Data Factory
+summary: "Two packs: ADF copies new files from a private share into landing storage and Databricks reads them."
+questions:
+  - value: landing
+    ask: "Landing folder for the copied files (abfss://…)"
+    default: abfss://landing@storeslakedev.dfs.core.windows.net/stores
+  - value: key_vault
+    ask: "Key Vault holding the share account's password"
+    default: https://kv-stores-dev.vault.azure.net

package/project-templates/private-sql-server/README.md

@@ -0,0 +1,25 @@
+# Private SQL Server
+
+The ERP of the sql-server starter, but now **unreachable** from the workspace
+(`connectivity: private`). The standard uses two packs,
+`[databricks@1, adf@1]`. Databricks' Lakeflow Connect rule is skipped
+because it cannot reach a private source, and the ADF pack's rule chooses
+`adf-copy`:
+
+- **ADF** ([build/prod/adf](build/prod/adf/)), in the folder layout ADF
+  keeps in Git:
+  - a [SQL Server linked service](build/prod/adf/linkedService/ls_erp_erp_erp.json)
+    through the self-hosted integration runtime `shir-erp`, with the password
+    from Key Vault;
+  - a [pipeline](build/prod/adf/pipeline/pl_erp_erp.json) with one copy per
+    table, selecting only the contract's columns changed in the window, into
+    `<landing>/erp.dbo.Customers/` as Parquet;
+  - an hourly [tumbling window trigger](build/prod/adf/trigger/tr_erp_erp.json),
+    started in production and stopped elsewhere.
+- **Databricks** ([build/prod/databricks](build/prod/databricks/)): streaming
+  tables over the same landing folders that keep the current row per key
+  (`mode: upsert`), and the refresh job.
+
+Each side runs on its own schedule; Auto Loader picks up new files whenever
+the pipeline runs. The integration runtime and Key Vault are shared
+infrastructure: they must exist and are only referenced.

package/project-templates/private-sql-server/connections/erp.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: erp
+description: The ERP's SQL Server, inside the corporate network only.
+connector: sqlserver
+connectivity: private
+settings:
+  server: erp-sql01.corp.local
+  user: ingest_reader
+  password: ${secret:erp-sql-password} # a Key Vault secret name

package/project-templates/private-sql-server/contracts/customers.yaml

@@ -0,0 +1,14 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: erp.customers
+version: 1.0.0
+status: active
+schema:
+  - name: customers
+    logicalType: object
+    description: ERP customers.
+    properties:
+      - { name: customer_id, logicalType: integer, required: true, primaryKey: true }
+      - { name: name, logicalType: string }
+      - { name: country, logicalType: string }
+      - { name: modified_at, logicalType: timestamp, required: true }

package/project-templates/private-sql-server/contracts/invoices.yaml

@@ -0,0 +1,15 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: erp.invoices
+version: 1.0.0
+status: active
+schema:
+  - name: invoices
+    logicalType: object
+    description: ERP invoices.
+    properties:
+      - { name: invoice_id, logicalType: integer, required: true, primaryKey: true }
+      - { name: customer_id, logicalType: integer, required: true }
+      - { name: amount, logicalType: number, physicalType: "DECIMAL(18,2)" }
+      - { name: status, logicalType: string }
+      - { name: updated_at, logicalType: timestamp, required: true }

package/project-templates/private-sql-server/flows/erp.yaml

@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: erp
+source: erp
+tables:
+  customers: { object: erp.dbo.Customers, contract: erp.customers }
+  invoices:
+    object: erp.dbo.Invoices
+    contract: erp.invoices
+    ingest:
+      with: { watermark_column: updated_at }
+    store:
+      with: { sequence_by: updated_at }

package/project-templates/private-sql-server/platform.yaml

@@ -0,0 +1,11 @@
+apiVersion: ingestron.io/v1
+kind: Platform
+id: lakehouse
+schedule:
+  cron: 0 15 * * * ?
+  timezone: Pacific/Auckland
+settings:
+  adf-copy: # shared infrastructure, referenced by name
+    landing: ${values.landing}
+    key_vault: ${values.key_vault}
+    integration_runtime: shir-erp

package/project-templates/private-sql-server/project.yaml

@@ -0,0 +1,19 @@
+# The same ERP as the sql-server starter, but the workspace cannot reach it. ADF copies
+# the changed rows through a self-hosted integration runtime into a landing
+# folder; Databricks keeps the tables current from those files.
+id: erp
+environments:
+  dev: {}
+  prod:
+    values:
+      landing: abfss://landing@erplakeprod.dfs.core.windows.net/erp
+      key_vault: https://kv-erp-prod.vault.azure.net
+values:
+  landing: abfss://landing@erplakedev.dfs.core.windows.net/erp
+  key_vault: https://kv-erp-dev.vault.azure.net
+imports:
+  - standard.yaml
+  - platform.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - contracts/*.yaml

package/project-templates/private-sql-server/standard.yaml

@@ -0,0 +1,17 @@
+apiVersion: ingestron.io/v1
+kind: Standard
+id: erp
+packs: [databricks@1, adf@1] # Databricks first: it provides the defaults
+naming:
+  catalog: erp_{{ environment }}
+ingest:
+  settings:
+    adf-copy: # the schedule; where it lands is the platform's
+      mode: incremental
+      watermark_column: modified_at
+      every: { frequency: Hour, interval: 1 }
+      start: 2026-10-01T00:00:00Z
+store:
+  settings:
+    # Each window's file holds the rows changed in it; keep the current version.
+    streaming-table: { mode: upsert, sequence_by: modified_at }

package/project-templates/private-sql-server/template.yaml

@@ -0,0 +1,10 @@
+# Used by `ingestron init --template private-sql-server`.
+title: Private SQL Server through Azure Data Factory
+summary: "Two packs: ADF copies a private SQL Server into landing storage and Databricks keeps the tables current."
+questions:
+  - value: landing
+    ask: "Landing folder for the copied files (abfss://…)"
+    default: abfss://landing@erplakedev.dfs.core.windows.net/erp
+  - value: key_vault
+    ask: "Key Vault holding the SQL login's password"
+    default: https://kv-erp-dev.vault.azure.net

package/project-templates/pyspark/README.md

@@ -0,0 +1,12 @@
+# PySpark
+
+`monthly_revenue` is written in PySpark (`transform: { uses: pyspark }`).
+[Its code](transforms/monthly_revenue.py) only defines
+`transform(tables)`, which receives a DataFrame per input and returns the
+result. The pack generates [a Python file](build/dev/src/subscriptions/silver/monthly_revenue.py)
+that copies the code unchanged and wraps it as a materialised view with the
+pipelines Python API (`from pyspark import pipelines as dp`): the full name,
+comment and clustering from the definitions, and the checks as
+`expect_all…` decorators.
+
+The bronze table stays SQL; one pipeline runs both files.

package/project-templates/pyspark/connections/billing.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: billing
+connector: files
+connectivity: public
+settings:
+  path: /Volumes/dev/landing/billing

package/project-templates/pyspark/contracts/invoices.yaml

@@ -0,0 +1,16 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: billing.invoices
+version: 1.0.0
+status: active
+schema:
+  - name: invoices
+    logicalType: object
+    description: Invoices from the billing system.
+    properties:
+      - { name: invoice_id, logicalType: string, required: true, primaryKey: true }
+      - { name: customer_id, logicalType: string, required: true }
+      - { name: plan, logicalType: string }
+      - { name: amount, logicalType: number, physicalType: "DECIMAL(18,2)" }
+      - { name: status, logicalType: string }
+      - { name: issued_at, logicalType: timestamp }

package/project-templates/pyspark/contracts/monthly_revenue.yaml

@@ -0,0 +1,14 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: billing.monthly_revenue
+version: 1.0.0
+status: active
+schema:
+  - name: monthly_revenue
+    logicalType: object
+    description: Paid revenue and customers per month and plan.
+    properties:
+      - { name: month, logicalType: timestamp, required: true }
+      - { name: plan, logicalType: string }
+      - { name: revenue, logicalType: number, physicalType: "DECIMAL(18,2)" }
+      - { name: customers, logicalType: integer }

package/project-templates/pyspark/flows/billing.yaml

@@ -0,0 +1,6 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: billing
+source: billing
+tables:
+  invoices: { object: invoices, contract: billing.invoices }

package/project-templates/pyspark/project.yaml

@@ -0,0 +1,11 @@
+# A transform written in PySpark: the code is a function that receives a
+# DataFrame per input and returns the result.
+id: subscriptions
+environments:
+  dev: {}
+imports:
+  - standard.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - transforms/*.yaml
+  - contracts/*.yaml

package/project-templates/pyspark/standard.yaml

@@ -0,0 +1,4 @@
+apiVersion: ingestron.io/v1
+kind: Standard
+id: subscriptions
+packs: [databricks@1]

package/project-templates/pyspark/template.yaml

@@ -0,0 +1,3 @@
+# Used by `ingestron init --template pyspark`.
+title: PySpark transform
+summary: A PySpark transform wrapped as a materialised view.

package/project-templates/pyspark/transforms/monthly_revenue.py

@@ -0,0 +1,10 @@
+from pyspark.sql import functions as F
+
+
+def transform(tables):
+    invoices = tables["invoices"]
+    return (
+        invoices.where(F.col("status") == "paid")
+        .groupBy(F.date_trunc("month", "issued_at").alias("month"), "plan")
+        .agg(F.sum("amount").alias("revenue"), F.countDistinct("customer_id").alias("customers"))
+    )

package/project-templates/pyspark/transforms/monthly_revenue.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Transform
+id: monthly_revenue
+inputs: [invoices]
+transform:
+  uses: pyspark
+code: ${file:monthly_revenue.py}
+contract: billing.monthly_revenue
+expectations:
+  - { name: revenue_not_negative, check: revenue >= 0, action: fail }

package/project-templates/retail/README.md

@@ -0,0 +1,22 @@
+# Retail example
+
+A complete project: web shop files brought into bronze, orders kept current
+from change records, and a silver table of daily sales.
+
+| File                          | What it shows                                                      |
+| ----------------------------- | ------------------------------------------------------------------ |
+| `project.yaml`                | Values that differ between `dev` and `prod`                        |
+| `standard.yaml`               | The organisation's catalog naming, quality and a fixed setting     |
+| `platform.yaml`               | Workspace, identity, tags, permissions, schedule and notifications |
+| `connections/shop.yaml`       | A files source whose path differs per environment                  |
+| `flows/shop.yaml`             | Change data (`mode: upsert`), CSV options and an expectation       |
+| `transforms/daily_sales.yaml` | SQL in its own file, reading two flow tables                       |
+| `contracts/*.yaml`            | ODCS contracts: types, keys and descriptions                       |
+
+```sh
+pnpm ingestron plan --definitions examples/08-retail --out examples/08-retail/build
+```
+
+The generated files for both environments are in [build/](build/).
+For editor help, run `ingestron schemas` in this folder and add the printed
+setting to VS Code.

package/project-templates/retail/connections/shop.yaml

@@ -0,0 +1,8 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: shop
+description: Files the web shop exports every hour.
+connector: files
+connectivity: public
+settings:
+  path: ${values.storage}/shop

package/project-templates/retail/contracts/customers.yaml

@@ -0,0 +1,12 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: shop.customers
+version: 1.0.0
+status: active
+schema:
+  - name: customers
+    logicalType: object
+    description: Web shop customers.
+    properties:
+      - { name: id, logicalType: integer, required: true, primaryKey: true }
+      - { name: email, logicalType: string, required: true }

package/project-templates/retail/contracts/daily_sales.yaml

@@ -0,0 +1,19 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: shop.daily_sales
+version: 1.0.0
+status: active
+schema:
+  - name: daily_sales
+    logicalType: object
+    description: Order totals per customer and day.
+    properties:
+      - {
+          name: email,
+          logicalType: string,
+          required: true,
+          primaryKey: true,
+          primaryKeyPosition: 1,
+        }
+      - { name: day, logicalType: date, required: true, primaryKey: true, primaryKeyPosition: 2 }
+      - { name: total, logicalType: number, physicalType: "DECIMAL(18,2)" }

package/project-templates/retail/contracts/orders.yaml

@@ -0,0 +1,16 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: shop.orders
+version: 1.0.0
+status: active
+schema:
+  - name: orders
+    logicalType: object
+    description: One row per order, kept current.
+    properties:
+      - { name: id, logicalType: integer, required: true, primaryKey: true }
+      - { name: customer_id, logicalType: integer, required: true }
+      - { name: ordered_at, logicalType: timestamp, required: true }
+      - { name: total, logicalType: number, physicalType: "DECIMAL(18,2)" }
+      - { name: updated_at, logicalType: timestamp, required: true }
+      - { name: op, logicalType: string }

package/project-templates/retail/flows/shop.yaml

@@ -0,0 +1,18 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: shop
+description: Orders and customers from the web shop.
+source: shop
+tables:
+  orders:
+    object: orders
+    contract: shop.orders
+    store:
+      with: { mode: upsert, sequence_by: updated_at, delete_when: "op = 'D'", except_columns: [op] }
+    expectations:
+      - { name: positive_total, check: total >= 0 }
+  customers:
+    object: customers
+    contract: shop.customers
+    ingest:
+      with: { format: csv, options: { header: true } }

package/project-templates/retail/platform.yaml

@@ -0,0 +1,21 @@
+# Where this project's assets go. Values differ per environment.
+apiVersion: ingestron.io/v1
+kind: Platform
+id: lakehouse
+workspace:
+  host: ${values.host}
+run_as:
+  service_principal_name: 7f3c2a10-0000-4000-8000-000000000001
+tags:
+  team: sales
+  environment: ${environment.name}
+permissions:
+  - group_name: sales-engineers
+    level: CAN_MANAGE
+  - group_name: sales-analysts
+    level: CAN_VIEW
+schedule:
+  cron: 0 0 5 * * ?
+  timezone: Pacific/Auckland
+notifications:
+  on_failure: [sales-data@example.com]

package/project-templates/retail/project.yaml

@@ -0,0 +1,18 @@
+id: retail
+name: Retail sales
+values:
+  host: https://adb-1111111111111111.1.azuredatabricks.net
+  storage: abfss://landing@retaildev.dfs.core.windows.net
+environments:
+  dev: {}
+  prod:
+    values:
+      host: https://adb-2222222222222222.2.azuredatabricks.net
+      storage: abfss://landing@retailprod.dfs.core.windows.net
+imports:
+  - standard.yaml
+  - platform.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - transforms/*.yaml
+  - contracts/*.yaml

package/project-templates/retail/standard.yaml

@@ -0,0 +1,15 @@
+# How the organisation builds data products. Shared by every project.
+apiVersion: ingestron.io/v1
+kind: Standard
+id: acme
+packs: [databricks@1]
+naming:
+  catalog: "{{ environment }}_sales"
+quality:
+  required: drop
+store:
+  settings:
+    streaming-table:
+      properties: { delta.enableChangeDataFeed: "true" }
+  fixed:
+    streaming-table: { cluster_by: auto }

package/project-templates/retail/template.yaml

@@ -0,0 +1,10 @@
+# Used by `ingestron init --template retail`.
+title: Retail
+summary: "A complete project: files into bronze, orders kept current from change records, and daily sales in silver."
+questions:
+  - value: host
+    ask: "Databricks workspace URL"
+    default: https://adb-1111111111111111.1.azuredatabricks.net
+  - value: storage
+    ask: "Storage account URL for the landing files"
+    default: abfss://landing@retaildev.dfs.core.windows.net

package/project-templates/retail/transforms/daily_sales.sql

@@ -0,0 +1,7 @@
+SELECT
+  c.email,
+  DATE(o.ordered_at) AS day,
+  SUM(o.total) AS total
+FROM {{ tables.orders }} AS o
+JOIN {{ tables.customers }} AS c ON c.id = o.customer_id
+GROUP BY c.email, DATE(o.ordered_at)

package/project-templates/retail/transforms/daily_sales.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Transform
+id: daily_sales
+description: Order totals per customer and day.
+inputs: [orders, customers]
+code: ${file:daily_sales.sql}
+contract: shop.daily_sales

package/project-templates/salesforce/README.md

@@ -0,0 +1,6 @@
+# Salesforce
+
+Two Salesforce objects through a Unity Catalog connection. SaaS sources need
+no gateway: [one ingestion pipeline](build/dev/resources/crm_salesforce_ingestion.pipeline.yml)
+writes `accounts` and `opportunities`, keyed by `Id`, with only the
+contract's fields.

package/project-templates/salesforce/connections/salesforce.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: salesforce
+connector: salesforce
+connectivity: public
+settings:
+  uc_connection: salesforce_prod

package/project-templates/salesforce/contracts/accounts.yaml

@@ -0,0 +1,13 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: crm.accounts
+version: 1.0.0
+status: active
+schema:
+  - name: accounts
+    logicalType: object
+    description: Salesforce accounts.
+    properties:
+      - { name: Id, logicalType: string, required: true, primaryKey: true }
+      - { name: Name, logicalType: string }
+      - { name: Industry, logicalType: string }

package/project-templates/salesforce/contracts/opportunities.yaml

@@ -0,0 +1,14 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: crm.opportunities
+version: 1.0.0
+status: active
+schema:
+  - name: opportunities
+    logicalType: object
+    description: Salesforce opportunities.
+    properties:
+      - { name: Id, logicalType: string, required: true, primaryKey: true }
+      - { name: AccountId, logicalType: string }
+      - { name: Amount, logicalType: number }
+      - { name: StageName, logicalType: string }

package/project-templates/salesforce/flows/salesforce.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: salesforce
+source: salesforce
+tables:
+  accounts: { object: Account, contract: crm.accounts }
+  opportunities: { object: Opportunity, contract: crm.opportunities }

package/project-templates/salesforce/project.yaml

@@ -0,0 +1,9 @@
+# Salesforce objects ingested with Lakeflow Connect; no gateway is needed.
+id: crm
+environments:
+  dev: {}
+imports:
+  - standard.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - contracts/*.yaml

package/project-templates/salesforce/standard.yaml

@@ -0,0 +1,4 @@
+apiVersion: ingestron.io/v1
+kind: Standard
+id: crm
+packs: [databricks@1]

package/project-templates/salesforce/template.yaml

@@ -0,0 +1,3 @@
+# Used by `ingestron init --template salesforce`.
+title: Salesforce through Lakeflow Connect
+summary: Two Salesforce objects through a Unity Catalog connection, keyed and limited to the contract's fields.

package/project-templates/sql-server/README.md

@@ -0,0 +1,17 @@
+# SQL Server
+
+The ERP database is reachable from the workspace over private networking
+(`connectivity: workspace`), so the pack ingests it with Lakeflow Connect:
+
+- [a gateway](build/prod/resources/erp_erp_gateway.pipeline.yml) that captures
+  changes continuously, with the cluster policy from the standard;
+- [an ingestion pipeline](build/prod/resources/erp_erp_ingestion.pipeline.yml)
+  that writes `customers` (SCD type 1) and `invoices` (SCD type 2, from
+  `mode: history`), each with only its contract's columns;
+- [our pipeline](build/prod/src/erp/silver/customer_balances.sql), whose
+  transform reads both, using the current invoice versions;
+- [a job](build/prod/resources/erp_refresh.job.yml) that refreshes ingestion
+  first and our pipeline after it.
+
+The Unity Catalog connection `erp_sqlserver_<environment>` must exist; the
+definitions name it but hold no credentials.

package/project-templates/sql-server/connections/erp.yaml

@@ -0,0 +1,8 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: erp
+description: The ERP's SQL Server, reachable from the workspace through ExpressRoute.
+connector: sqlserver
+connectivity: workspace
+settings:
+  uc_connection: erp_sqlserver_${environment.name} # created by an administrator beforehand

package/project-templates/sql-server/contracts/customer_balances.yaml

@@ -0,0 +1,13 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: erp.customer_balances
+version: 1.0.0
+status: active
+schema:
+  - name: customer_balances
+    logicalType: object
+    description: Open balance per customer.
+    properties:
+      - { name: customer_id, logicalType: integer, required: true, primaryKey: true }
+      - { name: name, logicalType: string }
+      - { name: balance, logicalType: number, physicalType: "DECIMAL(18,2)" }

package/project-templates/sql-server/contracts/customers.yaml

@@ -0,0 +1,13 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: erp.customers
+version: 1.0.0
+status: active
+schema:
+  - name: customers
+    logicalType: object
+    description: ERP customers.
+    properties:
+      - { name: customer_id, logicalType: integer, required: true, primaryKey: true }
+      - { name: name, logicalType: string }
+      - { name: country, logicalType: string }

package/project-templates/sql-server/contracts/invoices.yaml

@@ -0,0 +1,14 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: erp.invoices
+version: 1.0.0
+status: active
+schema:
+  - name: invoices
+    logicalType: object
+    description: ERP invoices with every past version.
+    properties:
+      - { name: invoice_id, logicalType: integer, required: true, primaryKey: true }
+      - { name: customer_id, logicalType: integer, required: true }
+      - { name: amount, logicalType: number, physicalType: "DECIMAL(18,2)" }
+      - { name: status, logicalType: string }

package/project-templates/sql-server/flows/erp.yaml

@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: erp
+source: erp
+tables:
+  customers:
+    object: erp.dbo.Customers
+    contract: erp.customers
+  invoices:
+    object: erp.dbo.Invoices
+    contract: erp.invoices
+    ingest:
+      with: { mode: history } # keep every version of an invoice

package/project-templates/sql-server/platform.yaml

@@ -0,0 +1,6 @@
+apiVersion: ingestron.io/v1
+kind: Platform
+id: lakehouse
+schedule:
+  cron: 0 0 * * * ?
+  timezone: Pacific/Auckland

package/project-templates/sql-server/project.yaml

@@ -0,0 +1,13 @@
+# A SQL Server database reached over private networking from the workspace,
+# ingested with Lakeflow Connect, then summarised by our own pipeline.
+id: erp
+environments:
+  dev: {}
+  prod: {}
+imports:
+  - standard.yaml
+  - platform.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - transforms/*.yaml
+  - contracts/*.yaml