ingestron 0.17.7 → 1.0.0

package/project-recipes/adf/connections/postgresql.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: postgresql
+connector: postgresql
+connectivity: private
+settings:
+  server: "source.example.test"
+  database: "sales"
+  user: "reader"
+  password: "${secret:source-password}"

package/project-recipes/adf/connections/presto.yaml

@@ -0,0 +1,11 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: presto
+connector: presto
+connectivity: private
+settings:
+  host: "source.example.test"
+  port: 5432
+  user: "reader"
+  password: "${secret:source-password}"
+  catalog: "synthetic-catalog"

package/project-recipes/adf/connections/quickbooks.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: quickbooks
+connector: quickbooks
+connectivity: private
+settings:
+  company_id: "synthetic-company_id"
+  client_id: "synthetic-client"
+  client_secret: "${secret:source-client-secret}"
+  refresh_token: "${secret:source-refresh-token}"

package/project-recipes/adf/connections/rest.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: rest
+connector: rest
+connectivity: private
+settings:
+  url: "https://source.example.test"
+  authentication: OAuth2ClientCredential
+  client_id: synthetic-reader
+  client_secret: "${secret:rest-client-secret}"
+  token_endpoint: "https://login.example.test/token"
+  scope: "orders.read"

package/project-recipes/adf/connections/s3-compatible.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: s3_compatible
+connector: s3-compatible
+connectivity: private
+settings:
+  access_key: "synthetic-access_key"
+  secret_key: "${secret:source-secret-key}"
+  url: "https://source.example.test"

package/project-recipes/adf/connections/salesforce-service-cloud.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: salesforce_service_cloud
+connector: salesforce-service-cloud
+connectivity: private
+settings:
+  url: "https://source.example.test"
+  client_id: "synthetic-client"
+  client_secret: "${secret:source-client-secret}"
+  api_version: "65.0"

package/project-recipes/adf/connections/salesforce.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: salesforce
+connector: salesforce
+connectivity: private
+settings:
+  url: "https://source.example.test"
+  client_id: "synthetic-client"
+  client_secret: "${secret:source-client-secret}"
+  api_version: "65.0"

package/project-recipes/adf/connections/sap-bw-mdx.yaml

@@ -0,0 +1,11 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: sap_bw_mdx
+connector: sap-bw-mdx
+connectivity: private
+settings:
+  server: "source.example.test"
+  system_number: "00"
+  client_id: "synthetic-client"
+  user: "reader"
+  password: "${secret:source-password}"

package/project-recipes/adf/connections/sap-bw-open-hub.yaml

@@ -0,0 +1,11 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: sap_bw_open_hub
+connector: sap-bw-open-hub
+connectivity: private
+settings:
+  server: "source.example.test"
+  system_number: "00"
+  client_id: "synthetic-client"
+  user: "reader"
+  password: "${secret:source-password}"

package/project-recipes/adf/connections/sap-c4c.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: sap_c4c
+connector: sap-c4c
+connectivity: private
+settings:
+  url: "https://source.example.test"
+  user: "reader"
+  password: "${secret:source-password}"

package/project-recipes/adf/connections/sap-ecc.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: sap_ecc
+connector: sap-ecc
+connectivity: private
+settings:
+  url: "https://source.example.test"
+  user: "reader"
+  password: "${secret:source-password}"

package/project-recipes/adf/connections/sap-hana.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: sap_hana
+connector: sap-hana
+connectivity: private
+settings:
+  connection_string: "${secret:source-connection-string}"

package/project-recipes/adf/connections/sap-table.yaml

@@ -0,0 +1,11 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: sap_table
+connector: sap-table
+connectivity: private
+settings:
+  server: "source.example.test"
+  system_number: "00"
+  client_id: "synthetic-client"
+  user: "reader"
+  password: "${secret:source-password}"

package/project-recipes/adf/connections/servicenow.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: servicenow
+connector: servicenow
+connectivity: private
+settings:
+  url: "https://source.example.test"
+  user: "reader"
+  password: "${secret:source-password}"

package/project-recipes/adf/connections/sftp.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: sftp
+connector: sftp
+connectivity: private
+settings:
+  host: "source.example.test"
+  user: "reader"
+  password: "${secret:source-password}"
+  host_key: "synthetic-host_key"

package/project-recipes/adf/connections/sharepoint-online-list.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: sharepoint_online_list
+connector: sharepoint-online-list
+connectivity: private
+settings:
+  url: "https://source.example.test"
+  tenant: "synthetic-tenant"
+  client_id: "synthetic-client"
+  certificate: "${secret:source-certificate}"

package/project-recipes/adf/connections/shopify.yaml

@@ -0,0 +1,8 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: shopify
+connector: shopify
+connectivity: private
+settings:
+  host: "source.example.test"
+  access_token: "${secret:source-access-token}"

package/project-recipes/adf/connections/snowflake.yaml

@@ -0,0 +1,11 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: snowflake
+connector: snowflake
+connectivity: private
+settings:
+  account: "synthetic-account"
+  database: "sales"
+  warehouse: "synthetic-warehouse"
+  user: "reader"
+  private_key: "${secret:source-private-key}"

package/project-recipes/adf/connections/spark.yaml

@@ -0,0 +1,11 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: spark
+connector: spark
+connectivity: private
+settings:
+  host: "source.example.test"
+  port: 5432
+  user: "reader"
+  password: "${secret:source-password}"
+  http_path: "synthetic-http_path"

package/project-recipes/adf/connections/sqlserver.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: sqlserver
+connector: sqlserver
+connectivity: private
+settings:
+  server: "source.example.test"
+  user: "reader"
+  password: "${secret:source-password}"

package/project-recipes/adf/connections/square.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: square
+connector: square
+connectivity: private
+settings:
+  host: "source.example.test"
+  client_id: "synthetic-client"
+  client_secret: "${secret:source-client-secret}"
+  access_token: "${secret:source-access-token}"

package/project-recipes/adf/connections/sybase.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: sybase
+connector: sybase
+connectivity: private
+settings:
+  server: "source.example.test"
+  database: "sales"
+  user: "reader"
+  password: "${secret:source-password}"

package/project-recipes/adf/connections/teradata.yaml

@@ -0,0 +1,9 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: teradata
+connector: teradata
+connectivity: private
+settings:
+  server: "source.example.test"
+  user: "reader"
+  password: "${secret:source-password}"

package/project-recipes/adf/connections/vertica.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: vertica
+connector: vertica
+connectivity: private
+settings:
+  server: "source.example.test"
+  database: "sales"
+  user: "reader"
+  password: "${secret:source-password}"

package/project-recipes/adf/connections/web-table.yaml

@@ -0,0 +1,7 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: web_table
+connector: web-table
+connectivity: private
+settings:
+  url: "https://source.example.test"

package/project-recipes/adf/connections/xero.yaml

@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: xero
+connector: xero
+connectivity: private
+settings:
+  client_id: "${secret:source-client-id}"
+  client_secret: "${secret:source-client-secret}"
+  tenant_id: "synthetic-tenant_id"
+  refresh_token: "${secret:source-refresh-token}"

package/project-recipes/adf/contracts/records.yaml

@@ -0,0 +1,12 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: records
+version: 1.0.0
+status: draft
+schema:
+  - name: records
+    logicalType: object
+    properties:
+      - { name: invoice_id, logicalType: integer, primaryKey: true }
+      - { name: amount, logicalType: number }
+      - { name: modified_at, logicalType: timestamp }

package/project-recipes/adf/contracts/salesforce-account.yaml

@@ -0,0 +1,12 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: salesforce_account
+version: 1.0.0
+status: draft
+schema:
+  - name: accounts
+    logicalType: object
+    properties:
+      - { name: Id, logicalType: string, primaryKey: true }
+      - { name: Name, logicalType: string }
+      - { name: SystemModstamp, logicalType: timestamp }

package/project-recipes/adf/flows/adls-gen2.yaml

@@ -0,0 +1,14 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: adls_gen2
+source: adls_gen2
+ingest:
+  uses: adf-files
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/adls-gen2"
+    format: "json"
+    mode: "all"
+tables:
+  adls_gen2_records:
+    object: "exports/orders"
+    contract: records

package/project-recipes/adf/flows/amazon-rds-oracle.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: amazon_rds_oracle
+source: amazon_rds_oracle
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/amazon-rds-oracle"
+tables:
+  amazon_rds_oracle_records:
+    object: "sales.orders"
+    contract: records

package/project-recipes/adf/flows/amazon-rds-sqlserver.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: amazon_rds_sqlserver
+source: amazon_rds_sqlserver
+ingest:
+  uses: adf-copy
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/amazon-rds-sqlserver"
+tables:
+  amazon_rds_sqlserver_records:
+    object: "sales.dbo.orders"
+    contract: records

package/project-recipes/adf/flows/amazon-redshift.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: amazon_redshift
+source: amazon_redshift
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/amazon-redshift"
+tables:
+  amazon_redshift_records:
+    object: "sales.orders"
+    contract: records

package/project-recipes/adf/flows/amazon-s3.yaml

@@ -0,0 +1,14 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: amazon_s3
+source: amazon_s3
+ingest:
+  uses: adf-files
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/amazon-s3"
+    format: "json"
+    mode: "all"
+tables:
+  amazon_s3_records:
+    object: "exports/orders"
+    contract: records

package/project-recipes/adf/flows/azure-blob.yaml

@@ -0,0 +1,14 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: azure_blob
+source: azure_blob
+ingest:
+  uses: adf-files
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-blob"
+    format: "json"
+    mode: "all"
+tables:
+  azure_blob_records:
+    object: "exports/orders"
+    contract: records

package/project-recipes/adf/flows/azure-cosmos-db-mongodb.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: azure_cosmos_db_mongodb
+source: azure_cosmos_db_mongodb
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-cosmos-db-mongodb"
+tables:
+  azure_cosmos_db_mongodb_records:
+    object: "orders"
+    contract: records

package/project-recipes/adf/flows/azure-cosmos-db-nosql.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: azure_cosmos_db_nosql
+source: azure_cosmos_db_nosql
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-cosmos-db-nosql"
+tables:
+  azure_cosmos_db_nosql_records:
+    object: "orders"
+    contract: records

package/project-recipes/adf/flows/azure-cosmos-postgresql.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: azure_cosmos_postgresql
+source: azure_cosmos_postgresql
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-cosmos-postgresql"
+tables:
+  azure_cosmos_postgresql_records:
+    object: "sales.orders"
+    contract: records

package/project-recipes/adf/flows/azure-data-explorer.yaml

@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: azure_data_explorer
+source: azure_data_explorer
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-data-explorer"
+    query: "SELECT * FROM orders"
+tables:
+  azure_data_explorer_records:
+    object: "orders"
+    contract: records

package/project-recipes/adf/flows/azure-databricks-delta.yaml

@@ -0,0 +1,15 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: azure_databricks_delta
+source: azure_databricks_delta
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-databricks-delta"
+    runtime_kind: "azure"
+    integration_runtime: synthetic-azure-runtime
+    source_options: { "staging_sas": "${secret:staging-sas}", "staging_path": "staging/orders" }
+tables:
+  azure_databricks_delta_records:
+    object: "sales.orders"
+    contract: records

package/project-recipes/adf/flows/azure-files.yaml

@@ -0,0 +1,14 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: azure_files
+source: azure_files
+ingest:
+  uses: adf-files
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-files"
+    format: "json"
+    mode: "all"
+tables:
+  azure_files_records:
+    object: "orders"
+    contract: records

package/project-recipes/adf/flows/azure-mysql.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: azure_mysql
+source: azure_mysql
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-mysql"
+tables:
+  azure_mysql_records:
+    object: "orders"
+    contract: records

package/project-recipes/adf/flows/azure-postgresql.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: azure_postgresql
+source: azure_postgresql
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-postgresql"
+tables:
+  azure_postgresql_records:
+    object: "sales.orders"
+    contract: records

package/project-recipes/adf/flows/azure-sql-database.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: azure_sql_database
+source: azure_sql_database
+ingest:
+  uses: adf-copy
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-sql-database"
+tables:
+  azure_sql_database_records:
+    object: "sales.dbo.orders"
+    contract: records

package/project-recipes/adf/flows/azure-sql-managed-instance.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: azure_sql_managed_instance
+source: azure_sql_managed_instance
+ingest:
+  uses: adf-copy
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-sql-managed-instance"
+tables:
+  azure_sql_managed_instance_records:
+    object: "sales.dbo.orders"
+    contract: records

package/project-recipes/adf/flows/azure-synapse.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: azure_synapse
+source: azure_synapse
+ingest:
+  uses: adf-copy
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-synapse"
+tables:
+  azure_synapse_records:
+    object: "sales.dbo.orders"
+    contract: records

package/project-recipes/adf/flows/azure-table-storage.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: azure_table_storage
+source: azure_table_storage
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-table-storage"
+tables:
+  azure_table_storage_records:
+    object: "orders"
+    contract: records

package/project-recipes/adf/flows/cassandra.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: cassandra
+source: cassandra
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/cassandra"
+tables:
+  cassandra_records:
+    object: "sales.orders"
+    contract: records

package/project-recipes/adf/flows/dataverse.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: dataverse
+source: dataverse
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/dataverse"
+tables:
+  dataverse_records:
+    object: "orders"
+    contract: records

package/project-recipes/adf/flows/db2.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: db2
+source: db2
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/db2"
+tables:
+  db2_records:
+    object: "sales.orders"
+    contract: records

package/project-recipes/adf/flows/dynamics-365.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: dynamics_365
+source: dynamics_365
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/dynamics-365"
+tables:
+  dynamics_365_records:
+    object: "orders"
+    contract: records

package/project-recipes/adf/flows/dynamics-ax.yaml

@@ -0,0 +1,12 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: dynamics_ax
+source: dynamics_ax
+ingest:
+  uses: adf-table
+  with:
+    landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/dynamics-ax"
+tables:
+  dynamics_ax_records:
+    object: "orders"
+    contract: records