ingestron 0.17.8 → 1.1.0

package/recipes/flows/netezza.yaml

@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: netezza
+source: netezza
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/netezza"
+    runtime_kind: "self-hosted"
+tables:
+  netezza_records:
+    object: "sales.orders"
+    contract: records

package/recipes/flows/odata.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: odata
+source: odata
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/odata"
+tables:
+  odata_records:
+    object: "orders"
+    contract: records

package/recipes/flows/odbc.yaml

@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: odbc
+source: odbc
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/odbc"
+    runtime_kind: "self-hosted"
+tables:
+  odbc_records:
+    object: "orders"
+    contract: records

package/recipes/flows/oracle-cloud-storage.yaml

@@ -0,0 +1,14 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: oracle_cloud_storage
+source: oracle_cloud_storage
+ingest:
+  uses: adf-files
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/oracle-cloud-storage"
+    format: "json"
+    mode: "all"
+tables:
+  oracle_cloud_storage_records:
+    object: "exports/orders"
+    contract: records

package/recipes/flows/oracle.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: oracle
+source: oracle
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/oracle"
+tables:
+  oracle_records:
+    object: "sales.orders"
+    contract: records

package/recipes/flows/postgresql.yaml

@@ -0,0 +1,18 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: postgresql
+source: postgresql
+ingest:
+  uses: adf-table
+  with:
+    resource_names:
+      source_linked_service: "ls_${project.id}_postgresql_${environment.name}"
+      source_dataset: "ds_${project.id}_orders_${environment.name}"
+      pipeline: "pl_${project.id}_orders_${environment.name}"
+      trigger: "tr_${project.id}_orders_${environment.name}"
+      copy_activity: "copy_orders"
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/postgresql"
+tables:
+  postgresql_records:
+    object: "sales.orders"
+    contract: records

package/recipes/flows/presto.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: presto
+source: presto
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/presto"
+tables:
+  presto_records:
+    object: "sales.orders"
+    contract: records

package/recipes/flows/quickbooks.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: quickbooks
+source: quickbooks
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/quickbooks"
+tables:
+  quickbooks_records:
+    object: "orders"
+    contract: records

package/recipes/flows/rest.yaml

@@ -0,0 +1,21 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: rest
+source: rest
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/rest"
+    source_options:
+      collection_reference: "$['items']"
+      column_paths:
+        invoice_id: "['invoice']['id']"
+        amount: "['invoice']['total']"
+        modified_at: "['updatedAt']"
+      pagination:
+        AbsoluteUrl: "$.nextLink"
+        MaxRequestNumber: 100
+tables:
+  rest_records:
+    object: "orders"
+    contract: records

package/recipes/flows/s3-compatible.yaml

@@ -0,0 +1,14 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: s3_compatible
+source: s3_compatible
+ingest:
+  uses: adf-files
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/s3-compatible"
+    format: "json"
+    mode: "all"
+tables:
+  s3_compatible_records:
+    object: "exports/orders"
+    contract: records

package/recipes/flows/salesforce-service-cloud.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: salesforce_service_cloud
+source: salesforce_service_cloud
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/salesforce-service-cloud"
+tables:
+  salesforce_service_cloud_records:
+    object: "orders"
+    contract: records

package/recipes/flows/salesforce.yaml

@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: salesforce
+source: salesforce
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/salesforce"
+    query: "SELECT Id, Name, SystemModstamp FROM Account WHERE Name != null"
+tables:
+  salesforce_records:
+    object: "Account"
+    contract: salesforce_account

package/recipes/flows/sap-bw-mdx.yaml

@@ -0,0 +1,14 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: sap_bw_mdx
+source: sap_bw_mdx
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/sap-bw-mdx"
+    query: "SELECT {[Measures].[Amount]} ON COLUMNS FROM [Sales]"
+    runtime_kind: "self-hosted"
+tables:
+  sap_bw_mdx_records:
+    object: "orders"
+    contract: records

package/recipes/flows/sap-bw-open-hub.yaml

@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: sap_bw_open_hub
+source: sap_bw_open_hub
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/sap-bw-open-hub"
+    runtime_kind: "self-hosted"
+tables:
+  sap_bw_open_hub_records:
+    object: "orders"
+    contract: records

package/recipes/flows/sap-c4c.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: sap_c4c
+source: sap_c4c
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/sap-c4c"
+tables:
+  sap_c4c_records:
+    object: "orders"
+    contract: records

package/recipes/flows/sap-ecc.yaml

@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: sap_ecc
+source: sap_ecc
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/sap-ecc"
+    runtime_kind: "self-hosted"
+tables:
+  sap_ecc_records:
+    object: "orders"
+    contract: records

package/recipes/flows/sap-hana.yaml

@@ -0,0 +1,14 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: sap_hana
+source: sap_hana
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/sap-hana"
+    query: 'SELECT * FROM "sales"."orders"'
+    runtime_kind: "self-hosted"
+tables:
+  sap_hana_records:
+    object: "sales.orders"
+    contract: records

package/recipes/flows/sap-table.yaml

@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: sap_table
+source: sap_table
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/sap-table"
+    runtime_kind: "self-hosted"
+tables:
+  sap_table_records:
+    object: "orders"
+    contract: records

package/recipes/flows/servicenow.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: servicenow
+source: servicenow
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/servicenow"
+tables:
+  servicenow_records:
+    object: "orders"
+    contract: records

package/recipes/flows/sftp.yaml

@@ -0,0 +1,14 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: sftp
+source: sftp
+ingest:
+  uses: adf-files
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/sftp"
+    format: "json"
+    mode: "all"
+tables:
+  sftp_records:
+    object: "orders"
+    contract: records

package/recipes/flows/sharepoint-online-list.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: sharepoint_online_list
+source: sharepoint_online_list
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/sharepoint-online-list"
+tables:
+  sharepoint_online_list_records:
+    object: "orders"
+    contract: records

package/recipes/flows/shopify.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: shopify
+source: shopify
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/shopify"
+tables:
+  shopify_records:
+    object: "orders"
+    contract: records

package/recipes/flows/snowflake.yaml

@@ -0,0 +1,15 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: snowflake
+source: snowflake
+ingest:
+  uses: adf-table
+  with:
+    resource_names:
+      staging_linked_service: "ls_${project.id}_snowflake_export"
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/snowflake"
+    source_options: { "staging_sas": "${secret:staging-sas}", "staging_path": "staging/orders" }
+tables:
+  snowflake_records:
+    object: "sales.orders"
+    contract: records

package/recipes/flows/spark.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: spark
+source: spark
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/spark"
+tables:
+  spark_records:
+    object: "sales.orders"
+    contract: records

package/recipes/flows/sqlserver.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: sqlserver
+source: sqlserver
+ingest:
+  uses: adf-copy
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/sqlserver"
+tables:
+  sqlserver_records:
+    object: "sales.dbo.orders"
+    contract: records

package/recipes/flows/square.yaml

@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: square
+source: square
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/square"
+    runtime_kind: "self-hosted"
+tables:
+  square_records:
+    object: "orders"
+    contract: records

package/recipes/flows/sybase.yaml

@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: sybase
+source: sybase
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/sybase"
+    runtime_kind: "self-hosted"
+tables:
+  sybase_records:
+    object: "orders"
+    contract: records

package/recipes/flows/teradata.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: teradata
+source: teradata
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/teradata"
+tables:
+  teradata_records:
+    object: "sales.orders"
+    contract: records

package/recipes/flows/vertica.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: vertica
+source: vertica
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/vertica"
+tables:
+  vertica_records:
+    object: "sales.orders"
+    contract: records

package/recipes/flows/web-table.yaml

@@ -0,0 +1,14 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: web_table
+source: web_table
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/web-table"
+    runtime_kind: "self-hosted"
+    source_options: { "table_index": 0 }
+tables:
+  web_table_records:
+    object: "orders"
+    contract: records

package/recipes/flows/xero.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: xero
+source: xero
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/xero"
+tables:
+  xero_records:
+    object: "orders"
+    contract: records

package/recipes/platform.yaml

@@ -0,0 +1,29 @@
+apiVersion: ingestron.io/v1
+kind: Platform
+id: native
+settings:
+  adf-copy:
+    resource_names:
+      key_vault_linked_service: "ls_${project.id}_secrets"
+      landing_linked_service: "ls_${project.id}_lake"
+      landing_dataset: "ds_${project.id}_parquet"
+    landing: abfss://landing@syntheticlake.dfs.core.windows.net/native
+    key_vault: https://synthetic-vault.vault.azure.net
+    integration_runtime: synthetic-self-hosted-runtime
+  adf-table:
+    resource_names:
+      key_vault_linked_service: "ls_${project.id}_secrets"
+      landing_linked_service: "ls_${project.id}_lake"
+      landing_dataset: "ds_${project.id}_parquet"
+    landing: abfss://landing@syntheticlake.dfs.core.windows.net/native
+    key_vault: https://synthetic-vault.vault.azure.net
+    integration_runtime: synthetic-self-hosted-runtime
+  adf-files:
+    resource_names:
+      key_vault_linked_service: "ls_${project.id}_secrets"
+      landing_linked_service: "ls_${project.id}_lake"
+      landing_files_dataset: "ds_${project.id}_copied_files"
+      landing_http_dataset: "ds_${project.id}_http_download"
+    landing: abfss://landing@syntheticlake.dfs.core.windows.net/native
+    key_vault: https://synthetic-vault.vault.azure.net
+    integration_runtime: synthetic-self-hosted-runtime

package/recipes/project.yaml

@@ -0,0 +1,10 @@
+id: native
+environments:
+  dev: {}
+  prod: {}
+imports:
+  - standard.yaml
+  - platform.yaml
+  - connections/*.yaml
+  - flows/*.yaml
+  - contracts/*.yaml

package/recipes/sample.yaml

@@ -0,0 +1,3 @@
+title: Native ADF ingestion
+summary: Configure native sources, authentication, runtime and extraction through definitions.
+order: 10

package/recipes/standard.yaml

@@ -0,0 +1,15 @@
+apiVersion: ingestron.io/v1
+kind: Standard
+id: native
+packs: [databricks@1, adf@1]
+ingest:
+  settings:
+    adf-copy:
+      start: 2026-10-01T00:00:00Z
+      every: { frequency: Hour, interval: 24 }
+    adf-table:
+      start: 2026-10-01T00:00:00Z
+      every: { frequency: Hour, interval: 24 }
+    adf-files:
+      start: 2026-10-01T00:00:00Z
+      every: { frequency: Hour, interval: 24 }

package/schemas/NOTICE.md

@@ -0,0 +1,7 @@
+# Third-party schema
+
+`odcs-v3.1.0.schema.json` is the Open Data Contract Standard v3.1.0 JSON
+Schema, unchanged, from
+<https://github.com/bitol-io/open-data-contract-standard> (`schema/`),
+copyright the Bitol project contributors, licensed under the Apache License
+2.0. Downloaded 2026-10-24.

package/schemas/connection.schema.yaml

@@ -0,0 +1,37 @@
+$schema: https://json-schema.org/draft/2020-12/schema
+title: Connection
+description: A source that flows read from.
+type: object
+additionalProperties: false
+required: [apiVersion, kind, id, connector, connectivity]
+properties:
+  apiVersion: { const: ingestron.io/v1, description: Always ingestron.io/v1. }
+  kind: { const: Connection, description: The document kind. }
+  id:
+    type: string
+    pattern: "^[a-z][a-z0-9_]*$"
+    description: Name that flows use in `source`.
+    examples:
+      - shop
+  description:
+    {
+      type: string,
+      description: "What this document is for, in a sentence.",
+      examples: [Files the web shop exports every hour.],
+    }
+  connector:
+    type: string
+    pattern: "^[a-z][a-z0-9-]*$"
+    description: Connector type, such as `files`. The pack lists which methods can read it.
+    examples:
+      - files
+  connectivity:
+    enum: [public, workspace, private]
+    enumDescriptions:
+      - The data platform reaches the source directly.
+      - The platform reaches it over private networking, such as ExpressRoute or a peered network.
+      - The platform cannot reach it; something inside the source's network must bring the data.
+    description: How the data platform reaches the source. Standards choose methods by this.
+  settings:
+    type: object
+    description: Connector settings, such as `path` for files. Write secrets as `${secret:…}`.