ingestron 0.17.8 → 1.1.0

package/package.json

@@ -1,76 +1,43 @@
 {
   "name": "ingestron",
-  "version": "0.17.8",
-  "type": "module",
-  "description": "Build repeatable data pipelines from reviewed contracts and versioned standards.",
-  "license": "Apache-2.0",
-  "bin": {
-    "ingestron": "build/cli/cli/index.js"
-  },
-  "files": [
-    "build/cli/cli",
-    "build/cli/mcp",
-    "README.md",
-    "SECURITY.md",
-    "CHANGELOG.md",
-    "build/cli/version.js",
-    "build/cli/version.d.ts",
-    "LICENSE",
-    "NOTICE",
-    "THIRD_PARTY_NOTICES.md"
+  "version": "1.1.0",
+  "description": "Declare data ingestion in reviewed YAML definitions and generate Databricks and Azure Data Factory files under your standards.",
+  "keywords": [
+    "ingestion",
+    "databricks",
+    "azure-data-factory",
+    "data-engineering",
+    "yaml",
+    "cli"
   ],
-  "engines": {
-    "node": ">=22.12.0"
-  },
-  "packageManager": "pnpm@10.15.0",
-  "scripts": {
-    "build": "node scripts/clean-build.mjs && tsc",
-    "test": "tsx --test --test-concurrency=1 test/unit/*.test.ts test/providers/*.test.ts test/integration/*.test.ts",
-    "format": "prettier --write .",
-    "format:check": "prettier --check .",
-    "secrets:check": "node scripts/secrets.mjs",
-    "validate": "pnpm format:check && pnpm build && pnpm test && pnpm examples:check && pnpm secrets:check && pnpm docs:check && pnpm release:check && pnpm package:check",
-    "prepack": "pnpm build",
-    "package:check": "tsx scripts/package-check.ts",
-    "release:check": "node scripts/release-check.mjs",
-    "prepublishOnly": "node scripts/release-check.mjs --publish",
-    "docs:check": "node scripts/docs-check.mjs",
-    "examples:check": "node examples/quickstart.mjs",
-    "licences:inventory": "node scripts/licence-inventory.mjs"
-  },
-  "dependencies": {
-    "@clack/prompts": "1.7.0",
-    "@ingestron/core": "0.12.14",
-    "@modelcontextprotocol/sdk": "1.30.0",
-    "commander": "15.0.0",
-    "yaml": "2.9.0"
-  },
-  "devDependencies": {
-    "@types/node": "22.20.1",
-    "prettier": "3.9.6",
-    "tsx": "4.23.13",
-    "typescript": "7.0.2"
-  },
+  "homepage": "https://docs.ingestron.io",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/ingestron/cli.git"
+    "url": "git+https://github.com/ingestron/ingestron.git",
+    "directory": "packages/cli"
   },
-  "homepage": "https://ingestron.io",
   "bugs": {
-    "url": "https://github.com/ingestron/cli/issues"
+    "url": "https://github.com/ingestron/ingestron/issues",
+    "email": "hello@ingestron.io"
   },
-  "publishConfig": {
-    "access": "public",
-    "registry": "https://registry.npmjs.org/"
+  "author": "IntentLabs <hello@ingestron.io>",
+  "license": "Apache-2.0",
+  "type": "module",
+  "bin": {
+    "ingestron": "dist/main.js"
   },
-  "keywords": [
-    "ingestron",
-    "data-engineering",
-    "data-contracts",
-    "etl",
-    "elt",
-    "cli",
-    "mcp",
-    "code-generation"
+  "engines": {
+    "node": ">=22.13.0"
+  },
+  "files": [
+    "dist",
+    "schemas",
+    "templates",
+    "project-templates",
+    "recipes",
+    "LICENSE",
+    "NOTICE",
+    "THIRD_PARTY_NOTICES",
+    "README.md"
   ]
 }

package/project-templates/kafka/README.md

@@ -0,0 +1,12 @@
+# Kafka
+
+Two topics of an Event Hubs namespace, read through its Kafka endpoint.
+[The ingestion pipeline](build/dev/resources/clickstream_events_ingestion.pipeline.yml)
+runs on the `PREVIEW` channel, appends every message, and parses JSON values
+with the contract's column types. In production it runs continuously; in
+`dev` it runs when started (`databricks bundle run`), which costs less. Set
+`continuous: true` to change that. The pack chose `mode: append`
+for Kafka by its own rule.
+
+Lakeflow Connect for Kafka is in Beta on Databricks (checked 2026-10-25); the
+generated file says so.

package/project-templates/kafka/connections/events.yaml

@@ -0,0 +1,8 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: events
+description: Event Hubs namespace, used through its Kafka endpoint.
+connector: kafka
+connectivity: public
+settings:
+  uc_connection: web_events_kafka

package/project-templates/kafka/contracts/page_views.yaml

@@ -0,0 +1,13 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: web.page_views
+version: 1.0.0
+status: active
+schema:
+  - name: page_views
+    logicalType: object
+    description: One event per page view.
+    properties:
+      - { name: session_id, logicalType: string, required: true }
+      - { name: url, logicalType: string }
+      - { name: viewed_at, logicalType: timestamp }

package/project-templates/kafka/contracts/purchases.yaml

@@ -0,0 +1,13 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: web.purchases
+version: 1.0.0
+status: active
+schema:
+  - name: purchases
+    logicalType: object
+    description: One event per purchase.
+    properties:
+      - { name: order_id, logicalType: string, required: true }
+      - { name: amount, logicalType: number, physicalType: "DECIMAL(18,2)" }
+      - { name: purchased_at, logicalType: timestamp }

package/project-templates/kafka/flows/events.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: events
+source: events
+ingest:
+  # Keep append explicit when this flow is composed with a chosen ingestion method.
+  with: { starting_offset: earliest, mode: append }
+tables:
+  page_views: { object: web.page-views, contract: web.page_views }
+  purchases: { object: web.purchases, contract: web.purchases }

package/project-templates/kafka/project.yaml

@@ -0,0 +1,10 @@
+# Kafka topics streamed continuously with Lakeflow Connect (Beta), JSON
+# values parsed with the contract's types.
+id: clickstream
+environments:
+  dev: {}
+imports:
+  - standard.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - contracts/*.yaml

package/project-templates/kafka/standard.yaml

@@ -0,0 +1,4 @@
+apiVersion: ingestron.io/v1
+kind: Standard
+id: web
+packs: [databricks@1]

package/project-templates/kafka/template.yaml

@@ -0,0 +1,3 @@
+# Used by `ingestron init --template kafka`.
+title: Kafka through Lakeflow Connect
+summary: Two Kafka topics appended and parsed with the contract's column types.

package/project-templates/minimal/README.md

@@ -0,0 +1,10 @@
+# Minimal
+
+The standard only names the pack, so every choice is a pack default.
+
+| Definition              | Becomes                                                                                                                                            |
+| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `flows/orders.yaml`     | [a streaming table](build/dev/src/shop/bronze/orders.sql) reading new files once each                                                              |
+| `contracts/orders.yaml` | `schemaHints` from the column types, `COMMENT`, and a not-null check on the required `id`                                                          |
+| pack defaults           | catalog `dev` (the environment), schema `bronze` (the first layer), `CLUSTER BY AUTO`, parquet                                                     |
+| `project.yaml`          | [a bundle](build/dev/databricks.yml) with one `dev` target in development mode, and [a serverless pipeline](build/dev/resources/shop.pipeline.yml) |

package/project-templates/minimal/connections/shop.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: shop
+connector: files
+connectivity: public
+settings:
+  path: ${values.landing}

package/project-templates/minimal/contracts/orders.yaml

@@ -0,0 +1,14 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: shop.orders
+version: 1.0.0
+status: active
+schema:
+  - name: orders
+    logicalType: object
+    description: One row per order.
+    properties:
+      - { name: id, logicalType: integer, required: true, primaryKey: true }
+      - { name: customer_id, logicalType: integer }
+      - { name: total, logicalType: number, physicalType: "DECIMAL(18,2)" }
+      - { name: ordered_at, logicalType: timestamp }

package/project-templates/minimal/flows/orders.yaml

@@ -0,0 +1,6 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: orders
+source: shop
+tables:
+  orders: { object: orders, contract: shop.orders }

package/project-templates/minimal/project.yaml

@@ -0,0 +1,11 @@
+# The smallest project: everything not written here comes from the pack's defaults.
+id: shop
+values:
+  landing: /Volumes/dev/landing/shop
+environments:
+  dev: {}
+imports:
+  - standard.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - contracts/*.yaml

package/project-templates/minimal/standard.yaml

@@ -0,0 +1,4 @@
+apiVersion: ingestron.io/v1
+kind: Standard
+id: defaults
+packs: [databricks@1]

package/project-templates/minimal/template.yaml

@@ -0,0 +1,7 @@
+# Used by `ingestron init --template minimal`.
+title: Minimal project
+summary: "Pack defaults only: an Auto Loader streaming table with schema hints and clustering."
+questions:
+  - value: landing
+    ask: "Where do the source files land?"
+    default: /Volumes/dev/landing/shop

package/project-templates/private-files/README.md

@@ -0,0 +1,29 @@
+# Private file share
+
+Each store exports its sales and stock as CSV files to a share inside the
+corporate network, `\\fs01\exports`, which the workspace cannot reach
+(`connectivity: private`). The standard uses two packs,
+`[databricks@1, adf@1]`. Databricks' `files` method cannot reach a private
+source, so the ADF pack's rule chooses `adf-files`:
+
+- **ADF** ([build/prod/adf](build/prod/adf/)), in the folder layout ADF
+  keeps in Git:
+  - a [file server linked service](build/prod/adf/linkedService/ls_stores_share.json)
+    through the self-hosted integration runtime `shir-stores`, with the
+    password from Key Vault;
+  - a [pipeline](build/prod/adf/pipeline/pl_stores_share.json) with one
+    binary copy per table: the files changed in the window, from
+    `\\fs01\exports\sales` to `<landing>/sales/`, keeping their names and
+    folders;
+  - an hourly [tumbling window trigger](build/prod/adf/trigger/tr_stores_share.json),
+    started in production and stopped elsewhere.
+- **Databricks** ([build/prod/databricks](build/prod/databricks/)): streaming
+  tables that read the landing folders with Auto Loader, as CSV with a
+  header, typed by the contracts' columns.
+
+The files are copied as they are, so the flow says their format
+(`format: csv`). To copy every file each window instead of the changed ones,
+set `mode: all`; to copy only some, set `files`, such as `"*.csv"`.
+
+The integration runtime and Key Vault are shared infrastructure: they must
+exist and are only referenced.

package/project-templates/private-files/connections/share.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: share
+description: The stores' export share, inside the corporate network only.
+connector: files
+connectivity: private
+settings:
+  path: '\\fs01\exports'
+  user: corp\ingest
+  password: ${secret:fs01-password} # a Key Vault secret name

package/project-templates/private-files/contracts/sales.yaml

@@ -0,0 +1,14 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: stores.sales
+version: 1.0.0
+status: active
+schema:
+  - name: sales
+    logicalType: object
+    description: One row per sale, exported by each store every hour.
+    properties:
+      - { name: sale_id, logicalType: string, required: true, primaryKey: true }
+      - { name: store_id, logicalType: integer, required: true }
+      - { name: amount, logicalType: number, physicalType: "DECIMAL(18,2)" }
+      - { name: sold_at, logicalType: timestamp, required: true }

package/project-templates/private-files/contracts/stock.yaml

@@ -0,0 +1,14 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: stores.stock
+version: 1.0.0
+status: active
+schema:
+  - name: stock
+    logicalType: object
+    description: Stock on hand per store and product, exported every hour.
+    properties:
+      - { name: store_id, logicalType: integer, required: true }
+      - { name: product_id, logicalType: integer, required: true }
+      - { name: on_hand, logicalType: integer }
+      - { name: counted_at, logicalType: timestamp, required: true }

package/project-templates/private-files/flows/stores.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: stores
+source: share
+ingest:
+  with: { format: csv, options: { header: true } } # the files' own format
+tables:
+  sales: { object: sales, contract: stores.sales }
+  stock: { object: stock, contract: stores.stock }

package/project-templates/private-files/platform.yaml

@@ -0,0 +1,11 @@
+apiVersion: ingestron.io/v1
+kind: Platform
+id: lakehouse
+schedule:
+  cron: 0 15 * * * ?
+  timezone: Pacific/Auckland
+settings:
+  adf-files: # shared infrastructure, referenced by name
+    landing: ${values.landing}
+    key_vault: ${values.key_vault}
+    integration_runtime: shir-stores

package/project-templates/private-files/project.yaml

@@ -0,0 +1,19 @@
+# Store exports on a file share inside the corporate network. ADF copies new
+# files through a self-hosted integration runtime into a landing folder as
+# they are; Databricks reads them there with Auto Loader.
+id: stores
+environments:
+  dev: {}
+  prod:
+    values:
+      landing: abfss://landing@storeslakeprod.dfs.core.windows.net/stores
+      key_vault: https://kv-stores-prod.vault.azure.net
+values:
+  landing: abfss://landing@storeslakedev.dfs.core.windows.net/stores
+  key_vault: https://kv-stores-dev.vault.azure.net
+imports:
+  - standard.yaml
+  - platform.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - contracts/*.yaml

package/project-templates/private-files/standard.yaml

@@ -0,0 +1,11 @@
+apiVersion: ingestron.io/v1
+kind: Standard
+id: stores
+packs: [databricks@1, adf@1] # Databricks first: it provides the defaults
+naming:
+  catalog: stores_{{ environment }}
+ingest:
+  settings:
+    adf-files: # the schedule; where it lands is the platform's
+      every: { frequency: Hour, interval: 1 }
+      start: 2026-10-01T00:00:00Z

package/project-templates/private-files/template.yaml

@@ -0,0 +1,10 @@
+# Used by `ingestron init --template private-files`.
+title: Private file share through Azure Data Factory
+summary: "Two packs: ADF copies new files from a private share into landing storage and Databricks reads them."
+questions:
+  - value: landing
+    ask: "Landing folder for the copied files (abfss://…)"
+    default: abfss://landing@storeslakedev.dfs.core.windows.net/stores
+  - value: key_vault
+    ask: "Key Vault holding the share account's password"
+    default: https://kv-stores-dev.vault.azure.net

package/project-templates/private-sql-server/README.md

@@ -0,0 +1,25 @@
+# Private SQL Server
+
+The ERP of the sql-server starter, but now **unreachable** from the workspace
+(`connectivity: private`). The standard uses two packs,
+`[databricks@1, adf@1]`. Databricks' Lakeflow Connect rule is skipped
+because it cannot reach a private source, and the ADF pack's rule chooses
+`adf-copy`:
+
+- **ADF** ([build/prod/adf](build/prod/adf/)), in the folder layout ADF
+  keeps in Git:
+  - a [SQL Server linked service](build/prod/adf/linkedService/ls_erp_erp_erp.json)
+    through the self-hosted integration runtime `shir-erp`, with the password
+    from Key Vault;
+  - a [pipeline](build/prod/adf/pipeline/pl_erp_erp.json) with one copy per
+    table, selecting only the contract's columns changed in the window, into
+    `<landing>/erp.dbo.Customers/` as Parquet;
+  - an hourly [tumbling window trigger](build/prod/adf/trigger/tr_erp_erp.json),
+    started in production and stopped elsewhere.
+- **Databricks** ([build/prod/databricks](build/prod/databricks/)): streaming
+  tables over the same landing folders that keep the current row per key
+  (`mode: upsert`), and the refresh job.
+
+Each side runs on its own schedule; Auto Loader picks up new files whenever
+the pipeline runs. The integration runtime and Key Vault are shared
+infrastructure: they must exist and are only referenced.

package/project-templates/private-sql-server/connections/erp.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: erp
+description: The ERP's SQL Server, inside the corporate network only.
+connector: sqlserver
+connectivity: private
+settings:
+  server: erp-sql01.corp.local
+  user: ingest_reader
+  password: ${secret:erp-sql-password} # a Key Vault secret name

package/project-templates/private-sql-server/contracts/customers.yaml

@@ -0,0 +1,14 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: erp.customers
+version: 1.0.0
+status: active
+schema:
+  - name: customers
+    logicalType: object
+    description: ERP customers.
+    properties:
+      - { name: customer_id, logicalType: integer, required: true, primaryKey: true }
+      - { name: name, logicalType: string }
+      - { name: country, logicalType: string }
+      - { name: modified_at, logicalType: timestamp, required: true }

package/project-templates/private-sql-server/contracts/invoices.yaml

@@ -0,0 +1,15 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: erp.invoices
+version: 1.0.0
+status: active
+schema:
+  - name: invoices
+    logicalType: object
+    description: ERP invoices.
+    properties:
+      - { name: invoice_id, logicalType: integer, required: true, primaryKey: true }
+      - { name: customer_id, logicalType: integer, required: true }
+      - { name: amount, logicalType: number, physicalType: "DECIMAL(18,2)" }
+      - { name: status, logicalType: string }
+      - { name: updated_at, logicalType: timestamp, required: true }

package/project-templates/private-sql-server/flows/erp.yaml

@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: erp
+source: erp
+tables:
+  customers: { object: erp.dbo.Customers, contract: erp.customers }
+  invoices:
+    object: erp.dbo.Invoices
+    contract: erp.invoices
+    ingest:
+      with: { watermark_column: updated_at }
+    store:
+      with: { sequence_by: updated_at }

package/project-templates/private-sql-server/platform.yaml

@@ -0,0 +1,11 @@
+apiVersion: ingestron.io/v1
+kind: Platform
+id: lakehouse
+schedule:
+  cron: 0 15 * * * ?
+  timezone: Pacific/Auckland
+settings:
+  adf-copy: # shared infrastructure, referenced by name
+    landing: ${values.landing}
+    key_vault: ${values.key_vault}
+    integration_runtime: shir-erp

package/project-templates/private-sql-server/project.yaml

@@ -0,0 +1,19 @@
+# The same ERP as the sql-server starter, but the workspace cannot reach it. ADF copies
+# the changed rows through a self-hosted integration runtime into a landing
+# folder; Databricks keeps the tables current from those files.
+id: erp
+environments:
+  dev: {}
+  prod:
+    values:
+      landing: abfss://landing@erplakeprod.dfs.core.windows.net/erp
+      key_vault: https://kv-erp-prod.vault.azure.net
+values:
+  landing: abfss://landing@erplakedev.dfs.core.windows.net/erp
+  key_vault: https://kv-erp-dev.vault.azure.net
+imports:
+  - standard.yaml
+  - platform.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - contracts/*.yaml

package/project-templates/private-sql-server/standard.yaml

@@ -0,0 +1,17 @@
+apiVersion: ingestron.io/v1
+kind: Standard
+id: erp
+packs: [databricks@1, adf@1] # Databricks first: it provides the defaults
+naming:
+  catalog: erp_{{ environment }}
+ingest:
+  settings:
+    adf-copy: # the schedule; where it lands is the platform's
+      mode: incremental
+      watermark_column: modified_at
+      every: { frequency: Hour, interval: 1 }
+      start: 2026-10-01T00:00:00Z
+store:
+  settings:
+    # Each window's file holds the rows changed in it; keep the current version.
+    streaming-table: { mode: upsert, sequence_by: modified_at }

package/project-templates/private-sql-server/template.yaml

@@ -0,0 +1,10 @@
+# Used by `ingestron init --template private-sql-server`.
+title: Private SQL Server through Azure Data Factory
+summary: "Two packs: ADF copies a private SQL Server into landing storage and Databricks keeps the tables current."
+questions:
+  - value: landing
+    ask: "Landing folder for the copied files (abfss://…)"
+    default: abfss://landing@erplakedev.dfs.core.windows.net/erp
+  - value: key_vault
+    ask: "Key Vault holding the SQL login's password"
+    default: https://kv-erp-dev.vault.azure.net

package/project-templates/pyspark/README.md

@@ -0,0 +1,12 @@
+# PySpark
+
+`monthly_revenue` is written in PySpark (`transform: { uses: pyspark }`).
+[Its code](transforms/monthly_revenue.py) only defines
+`transform(tables)`, which receives a DataFrame per input and returns the
+result. The pack generates [a Python file](build/dev/src/subscriptions/silver/monthly_revenue.py)
+that copies the code unchanged and wraps it as a materialised view with the
+pipelines Python API (`from pyspark import pipelines as dp`): the full name,
+comment and clustering from the definitions, and the checks as
+`expect_all…` decorators.
+
+The bronze table stays SQL; one pipeline runs both files.

package/project-templates/pyspark/connections/billing.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: billing
+connector: files
+connectivity: public
+settings:
+  path: /Volumes/dev/landing/billing

package/project-templates/pyspark/contracts/invoices.yaml

@@ -0,0 +1,16 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: billing.invoices
+version: 1.0.0
+status: active
+schema:
+  - name: invoices
+    logicalType: object
+    description: Invoices from the billing system.
+    properties:
+      - { name: invoice_id, logicalType: string, required: true, primaryKey: true }
+      - { name: customer_id, logicalType: string, required: true }
+      - { name: plan, logicalType: string }
+      - { name: amount, logicalType: number, physicalType: "DECIMAL(18,2)" }
+      - { name: status, logicalType: string }
+      - { name: issued_at, logicalType: timestamp }

package/project-templates/pyspark/contracts/monthly_revenue.yaml

@@ -0,0 +1,14 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: billing.monthly_revenue
+version: 1.0.0
+status: active
+schema:
+  - name: monthly_revenue
+    logicalType: object
+    description: Paid revenue and customers per month and plan.
+    properties:
+      - { name: month, logicalType: timestamp, required: true }
+      - { name: plan, logicalType: string }
+      - { name: revenue, logicalType: number, physicalType: "DECIMAL(18,2)" }
+      - { name: customers, logicalType: integer }

package/project-templates/pyspark/flows/billing.yaml

@@ -0,0 +1,6 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: billing
+source: billing
+tables:
+  invoices: { object: invoices, contract: billing.invoices }

package/project-templates/pyspark/project.yaml

@@ -0,0 +1,11 @@
+# A transform written in PySpark: the code is a function that receives a
+# DataFrame per input and returns the result.
+id: subscriptions
+environments:
+  dev: {}
+imports:
+  - standard.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - transforms/*.yaml
+  - contracts/*.yaml

package/project-templates/pyspark/standard.yaml

@@ -0,0 +1,4 @@
+apiVersion: ingestron.io/v1
+kind: Standard
+id: subscriptions
+packs: [databricks@1]

package/project-templates/pyspark/template.yaml

@@ -0,0 +1,3 @@
+# Used by `ingestron init --template pyspark`.
+title: PySpark transform
+summary: A PySpark transform wrapped as a materialised view.