ingestron 0.17.8 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. package/NOTICE +3 -5
  2. package/README.md +26 -74
  3. package/THIRD_PARTY_NOTICES +148 -0
  4. package/dist/main.js +29708 -0
  5. package/package.json +32 -65
  6. package/project-templates/kafka/README.md +12 -0
  7. package/project-templates/kafka/connections/events.yaml +8 -0
  8. package/project-templates/kafka/contracts/page_views.yaml +13 -0
  9. package/project-templates/kafka/contracts/purchases.yaml +13 -0
  10. package/project-templates/kafka/flows/events.yaml +10 -0
  11. package/project-templates/kafka/project.yaml +10 -0
  12. package/project-templates/kafka/standard.yaml +4 -0
  13. package/project-templates/kafka/template.yaml +3 -0
  14. package/project-templates/minimal/README.md +10 -0
  15. package/project-templates/minimal/connections/shop.yaml +7 -0
  16. package/project-templates/minimal/contracts/orders.yaml +14 -0
  17. package/project-templates/minimal/flows/orders.yaml +6 -0
  18. package/project-templates/minimal/project.yaml +11 -0
  19. package/project-templates/minimal/standard.yaml +4 -0
  20. package/project-templates/minimal/template.yaml +7 -0
  21. package/project-templates/private-files/README.md +29 -0
  22. package/project-templates/private-files/connections/share.yaml +10 -0
  23. package/project-templates/private-files/contracts/sales.yaml +14 -0
  24. package/project-templates/private-files/contracts/stock.yaml +14 -0
  25. package/project-templates/private-files/flows/stores.yaml +9 -0
  26. package/project-templates/private-files/platform.yaml +11 -0
  27. package/project-templates/private-files/project.yaml +19 -0
  28. package/project-templates/private-files/standard.yaml +11 -0
  29. package/project-templates/private-files/template.yaml +10 -0
  30. package/project-templates/private-sql-server/README.md +25 -0
  31. package/project-templates/private-sql-server/connections/erp.yaml +10 -0
  32. package/project-templates/private-sql-server/contracts/customers.yaml +14 -0
  33. package/project-templates/private-sql-server/contracts/invoices.yaml +15 -0
  34. package/project-templates/private-sql-server/flows/erp.yaml +13 -0
  35. package/project-templates/private-sql-server/platform.yaml +11 -0
  36. package/project-templates/private-sql-server/project.yaml +19 -0
  37. package/project-templates/private-sql-server/standard.yaml +17 -0
  38. package/project-templates/private-sql-server/template.yaml +10 -0
  39. package/project-templates/pyspark/README.md +12 -0
  40. package/project-templates/pyspark/connections/billing.yaml +7 -0
  41. package/project-templates/pyspark/contracts/invoices.yaml +16 -0
  42. package/project-templates/pyspark/contracts/monthly_revenue.yaml +14 -0
  43. package/project-templates/pyspark/flows/billing.yaml +6 -0
  44. package/project-templates/pyspark/project.yaml +11 -0
  45. package/project-templates/pyspark/standard.yaml +4 -0
  46. package/project-templates/pyspark/template.yaml +3 -0
  47. package/project-templates/pyspark/transforms/monthly_revenue.py +10 -0
  48. package/project-templates/pyspark/transforms/monthly_revenue.yaml +10 -0
  49. package/project-templates/retail/README.md +22 -0
  50. package/project-templates/retail/connections/shop.yaml +8 -0
  51. package/project-templates/retail/contracts/customers.yaml +12 -0
  52. package/project-templates/retail/contracts/daily_sales.yaml +19 -0
  53. package/project-templates/retail/contracts/orders.yaml +16 -0
  54. package/project-templates/retail/flows/shop.yaml +18 -0
  55. package/project-templates/retail/platform.yaml +21 -0
  56. package/project-templates/retail/project.yaml +18 -0
  57. package/project-templates/retail/standard.yaml +15 -0
  58. package/project-templates/retail/template.yaml +10 -0
  59. package/project-templates/retail/transforms/daily_sales.sql +7 -0
  60. package/project-templates/retail/transforms/daily_sales.yaml +7 -0
  61. package/project-templates/salesforce/README.md +6 -0
  62. package/project-templates/salesforce/connections/salesforce.yaml +7 -0
  63. package/project-templates/salesforce/contracts/accounts.yaml +13 -0
  64. package/project-templates/salesforce/contracts/opportunities.yaml +14 -0
  65. package/project-templates/salesforce/flows/salesforce.yaml +7 -0
  66. package/project-templates/salesforce/project.yaml +9 -0
  67. package/project-templates/salesforce/standard.yaml +4 -0
  68. package/project-templates/salesforce/template.yaml +3 -0
  69. package/project-templates/sql-server/README.md +17 -0
  70. package/project-templates/sql-server/connections/erp.yaml +8 -0
  71. package/project-templates/sql-server/contracts/customer_balances.yaml +13 -0
  72. package/project-templates/sql-server/contracts/customers.yaml +13 -0
  73. package/project-templates/sql-server/contracts/invoices.yaml +14 -0
  74. package/project-templates/sql-server/flows/erp.yaml +13 -0
  75. package/project-templates/sql-server/platform.yaml +6 -0
  76. package/project-templates/sql-server/project.yaml +13 -0
  77. package/project-templates/sql-server/standard.yaml +9 -0
  78. package/project-templates/sql-server/template.yaml +3 -0
  79. package/project-templates/sql-server/transforms/customer_balances.sql +5 -0
  80. package/project-templates/sql-server/transforms/customer_balances.yaml +6 -0
  81. package/recipes/connections/adls-gen2.yaml +7 -0
  82. package/recipes/connections/amazon-rds-oracle.yaml +9 -0
  83. package/recipes/connections/amazon-rds-sqlserver.yaml +9 -0
  84. package/recipes/connections/amazon-redshift.yaml +10 -0
  85. package/recipes/connections/amazon-s3.yaml +8 -0
  86. package/recipes/connections/azure-blob.yaml +7 -0
  87. package/recipes/connections/azure-cosmos-db-mongodb.yaml +8 -0
  88. package/recipes/connections/azure-cosmos-db-nosql.yaml +8 -0
  89. package/recipes/connections/azure-cosmos-postgresql.yaml +10 -0
  90. package/recipes/connections/azure-data-explorer.yaml +8 -0
  91. package/recipes/connections/azure-databricks-delta.yaml +9 -0
  92. package/recipes/connections/azure-files.yaml +8 -0
  93. package/recipes/connections/azure-mysql.yaml +7 -0
  94. package/recipes/connections/azure-postgresql.yaml +10 -0
  95. package/recipes/connections/azure-sql-database.yaml +9 -0
  96. package/recipes/connections/azure-sql-managed-instance.yaml +9 -0
  97. package/recipes/connections/azure-synapse.yaml +9 -0
  98. package/recipes/connections/azure-table-storage.yaml +7 -0
  99. package/recipes/connections/cassandra.yaml +9 -0
  100. package/recipes/connections/dataverse.yaml +9 -0
  101. package/recipes/connections/db2.yaml +7 -0
  102. package/recipes/connections/dynamics-365.yaml +9 -0
  103. package/recipes/connections/dynamics-ax.yaml +11 -0
  104. package/recipes/connections/dynamics-crm.yaml +9 -0
  105. package/recipes/connections/fabric-lakehouse.yaml +8 -0
  106. package/recipes/connections/fabric-warehouse.yaml +9 -0
  107. package/recipes/connections/files.yaml +9 -0
  108. package/recipes/connections/ftp.yaml +9 -0
  109. package/recipes/connections/google-ads.yaml +12 -0
  110. package/recipes/connections/google-bigquery.yaml +8 -0
  111. package/recipes/connections/google-cloud-storage.yaml +8 -0
  112. package/recipes/connections/greenplum.yaml +10 -0
  113. package/recipes/connections/hdfs.yaml +9 -0
  114. package/recipes/connections/hive.yaml +11 -0
  115. package/recipes/connections/http.yaml +7 -0
  116. package/recipes/connections/hubspot.yaml +10 -0
  117. package/recipes/connections/impala.yaml +11 -0
  118. package/recipes/connections/informix.yaml +9 -0
  119. package/recipes/connections/jira.yaml +9 -0
  120. package/recipes/connections/mariadb.yaml +10 -0
  121. package/recipes/connections/microsoft-365.yaml +10 -0
  122. package/recipes/connections/microsoft-access.yaml +9 -0
  123. package/recipes/connections/mongodb-atlas.yaml +8 -0
  124. package/recipes/connections/mongodb.yaml +8 -0
  125. package/recipes/connections/mysql.yaml +10 -0
  126. package/recipes/connections/netezza.yaml +11 -0
  127. package/recipes/connections/odata.yaml +9 -0
  128. package/recipes/connections/odbc.yaml +9 -0
  129. package/recipes/connections/oracle-cloud-storage.yaml +9 -0
  130. package/recipes/connections/oracle.yaml +9 -0
  131. package/recipes/connections/postgresql.yaml +10 -0
  132. package/recipes/connections/presto.yaml +11 -0
  133. package/recipes/connections/quickbooks.yaml +10 -0
  134. package/recipes/connections/rest.yaml +12 -0
  135. package/recipes/connections/s3-compatible.yaml +9 -0
  136. package/recipes/connections/salesforce-service-cloud.yaml +10 -0
  137. package/recipes/connections/salesforce.yaml +10 -0
  138. package/recipes/connections/sap-bw-mdx.yaml +11 -0
  139. package/recipes/connections/sap-bw-open-hub.yaml +11 -0
  140. package/recipes/connections/sap-c4c.yaml +9 -0
  141. package/recipes/connections/sap-ecc.yaml +9 -0
  142. package/recipes/connections/sap-hana.yaml +7 -0
  143. package/recipes/connections/sap-table.yaml +11 -0
  144. package/recipes/connections/servicenow.yaml +9 -0
  145. package/recipes/connections/sftp.yaml +10 -0
  146. package/recipes/connections/sharepoint-online-list.yaml +10 -0
  147. package/recipes/connections/shopify.yaml +8 -0
  148. package/recipes/connections/snowflake.yaml +11 -0
  149. package/recipes/connections/spark.yaml +11 -0
  150. package/recipes/connections/sqlserver.yaml +9 -0
  151. package/recipes/connections/square.yaml +10 -0
  152. package/recipes/connections/sybase.yaml +10 -0
  153. package/recipes/connections/teradata.yaml +9 -0
  154. package/recipes/connections/vertica.yaml +10 -0
  155. package/recipes/connections/web-table.yaml +7 -0
  156. package/recipes/connections/xero.yaml +10 -0
  157. package/recipes/contracts/records.yaml +12 -0
  158. package/recipes/contracts/salesforce-account.yaml +12 -0
  159. package/recipes/flows/adls-gen2.yaml +14 -0
  160. package/recipes/flows/amazon-rds-oracle.yaml +12 -0
  161. package/recipes/flows/amazon-rds-sqlserver.yaml +12 -0
  162. package/recipes/flows/amazon-redshift.yaml +12 -0
  163. package/recipes/flows/amazon-s3.yaml +14 -0
  164. package/recipes/flows/azure-blob.yaml +14 -0
  165. package/recipes/flows/azure-cosmos-db-mongodb.yaml +12 -0
  166. package/recipes/flows/azure-cosmos-db-nosql.yaml +12 -0
  167. package/recipes/flows/azure-cosmos-postgresql.yaml +12 -0
  168. package/recipes/flows/azure-data-explorer.yaml +13 -0
  169. package/recipes/flows/azure-databricks-delta.yaml +15 -0
  170. package/recipes/flows/azure-files.yaml +14 -0
  171. package/recipes/flows/azure-mysql.yaml +12 -0
  172. package/recipes/flows/azure-postgresql.yaml +12 -0
  173. package/recipes/flows/azure-sql-database.yaml +12 -0
  174. package/recipes/flows/azure-sql-managed-instance.yaml +12 -0
  175. package/recipes/flows/azure-synapse.yaml +12 -0
  176. package/recipes/flows/azure-table-storage.yaml +12 -0
  177. package/recipes/flows/cassandra.yaml +12 -0
  178. package/recipes/flows/dataverse.yaml +12 -0
  179. package/recipes/flows/db2.yaml +12 -0
  180. package/recipes/flows/dynamics-365.yaml +12 -0
  181. package/recipes/flows/dynamics-ax.yaml +12 -0
  182. package/recipes/flows/dynamics-crm.yaml +12 -0
  183. package/recipes/flows/fabric-lakehouse.yaml +14 -0
  184. package/recipes/flows/fabric-warehouse.yaml +14 -0
  185. package/recipes/flows/files.yaml +14 -0
  186. package/recipes/flows/ftp.yaml +14 -0
  187. package/recipes/flows/google-ads.yaml +13 -0
  188. package/recipes/flows/google-bigquery.yaml +12 -0
  189. package/recipes/flows/google-cloud-storage.yaml +14 -0
  190. package/recipes/flows/greenplum.yaml +12 -0
  191. package/recipes/flows/hdfs.yaml +14 -0
  192. package/recipes/flows/hive.yaml +12 -0
  193. package/recipes/flows/http.yaml +18 -0
  194. package/recipes/flows/hubspot.yaml +12 -0
  195. package/recipes/flows/impala.yaml +12 -0
  196. package/recipes/flows/informix.yaml +13 -0
  197. package/recipes/flows/jira.yaml +12 -0
  198. package/recipes/flows/mariadb.yaml +13 -0
  199. package/recipes/flows/microsoft-365.yaml +19 -0
  200. package/recipes/flows/microsoft-access.yaml +13 -0
  201. package/recipes/flows/mongodb-atlas.yaml +12 -0
  202. package/recipes/flows/mongodb.yaml +12 -0
  203. package/recipes/flows/mysql.yaml +12 -0
  204. package/recipes/flows/netezza.yaml +13 -0
  205. package/recipes/flows/odata.yaml +12 -0
  206. package/recipes/flows/odbc.yaml +13 -0
  207. package/recipes/flows/oracle-cloud-storage.yaml +14 -0
  208. package/recipes/flows/oracle.yaml +12 -0
  209. package/recipes/flows/postgresql.yaml +18 -0
  210. package/recipes/flows/presto.yaml +12 -0
  211. package/recipes/flows/quickbooks.yaml +12 -0
  212. package/recipes/flows/rest.yaml +21 -0
  213. package/recipes/flows/s3-compatible.yaml +14 -0
  214. package/recipes/flows/salesforce-service-cloud.yaml +12 -0
  215. package/recipes/flows/salesforce.yaml +13 -0
  216. package/recipes/flows/sap-bw-mdx.yaml +14 -0
  217. package/recipes/flows/sap-bw-open-hub.yaml +13 -0
  218. package/recipes/flows/sap-c4c.yaml +12 -0
  219. package/recipes/flows/sap-ecc.yaml +13 -0
  220. package/recipes/flows/sap-hana.yaml +14 -0
  221. package/recipes/flows/sap-table.yaml +13 -0
  222. package/recipes/flows/servicenow.yaml +12 -0
  223. package/recipes/flows/sftp.yaml +14 -0
  224. package/recipes/flows/sharepoint-online-list.yaml +12 -0
  225. package/recipes/flows/shopify.yaml +12 -0
  226. package/recipes/flows/snowflake.yaml +15 -0
  227. package/recipes/flows/spark.yaml +12 -0
  228. package/recipes/flows/sqlserver.yaml +12 -0
  229. package/recipes/flows/square.yaml +13 -0
  230. package/recipes/flows/sybase.yaml +13 -0
  231. package/recipes/flows/teradata.yaml +12 -0
  232. package/recipes/flows/vertica.yaml +12 -0
  233. package/recipes/flows/web-table.yaml +14 -0
  234. package/recipes/flows/xero.yaml +12 -0
  235. package/recipes/platform.yaml +29 -0
  236. package/recipes/project.yaml +10 -0
  237. package/recipes/sample.yaml +3 -0
  238. package/recipes/standard.yaml +15 -0
  239. package/schemas/NOTICE.md +7 -0
  240. package/schemas/connection.schema.yaml +37 -0
  241. package/schemas/flow.schema.yaml +125 -0
  242. package/schemas/odcs-v3.1.0.schema.json +2808 -0
  243. package/schemas/platform.schema.yaml +260 -0
  244. package/schemas/standard.schema.yaml +211 -0
  245. package/schemas/transform.schema.yaml +108 -0
  246. package/templates/apply-contract.py.liquid +58 -0
  247. package/templates/change-data.sql.liquid +20 -0
  248. package/templates/materialised-view.py.liquid +28 -0
  249. package/templates/materialised-view.sql.liquid +4 -0
  250. package/templates/read-files.liquid +7 -0
  251. package/templates/streaming-table.sql.liquid +3 -0
  252. package/templates/table.liquid +17 -0
  253. package/CHANGELOG.md +0 -162
  254. package/SECURITY.md +0 -32
  255. package/THIRD_PARTY_NOTICES.md +0 -3678
  256. package/build/cli/cli/index.d.ts +0 -2
  257. package/build/cli/cli/index.js +0 -685
  258. package/build/cli/cli/operation-worker.d.ts +0 -1
  259. package/build/cli/cli/operation-worker.js +0 -5
  260. package/build/cli/cli/output.d.ts +0 -3
  261. package/build/cli/cli/output.js +0 -246
  262. package/build/cli/cli/plugin-browser.d.ts +0 -14
  263. package/build/cli/cli/plugin-browser.js +0 -74
  264. package/build/cli/cli/plugin-resolution.d.ts +0 -126
  265. package/build/cli/cli/plugin-resolution.js +0 -270
  266. package/build/cli/cli/presentation.d.ts +0 -12
  267. package/build/cli/cli/presentation.js +0 -57
  268. package/build/cli/cli/workflow-commands.d.ts +0 -15
  269. package/build/cli/cli/workflow-commands.js +0 -430
  270. package/build/cli/mcp/index.d.ts +0 -3
  271. package/build/cli/mcp/index.js +0 -67
  272. package/build/cli/version.d.ts +0 -2
  273. package/build/cli/version.js +0 -14
package/recipes/flows/adls-gen2.yaml ADDED
@@ -0,0 +1,14 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: adls_gen2
4
+ source: adls_gen2
5
+ ingest:
6
+ uses: adf-files
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/adls-gen2"
9
+ format: "json"
10
+ mode: "all"
11
+ tables:
12
+ adls_gen2_records:
13
+ object: "exports/orders"
14
+ contract: records
package/recipes/flows/amazon-rds-oracle.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: amazon_rds_oracle
4
+ source: amazon_rds_oracle
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/amazon-rds-oracle"
9
+ tables:
10
+ amazon_rds_oracle_records:
11
+ object: "sales.orders"
12
+ contract: records
package/recipes/flows/amazon-rds-sqlserver.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: amazon_rds_sqlserver
4
+ source: amazon_rds_sqlserver
5
+ ingest:
6
+ uses: adf-copy
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/amazon-rds-sqlserver"
9
+ tables:
10
+ amazon_rds_sqlserver_records:
11
+ object: "sales.dbo.orders"
12
+ contract: records
package/recipes/flows/amazon-redshift.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: amazon_redshift
4
+ source: amazon_redshift
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/amazon-redshift"
9
+ tables:
10
+ amazon_redshift_records:
11
+ object: "sales.orders"
12
+ contract: records
package/recipes/flows/amazon-s3.yaml ADDED
@@ -0,0 +1,14 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: amazon_s3
4
+ source: amazon_s3
5
+ ingest:
6
+ uses: adf-files
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/amazon-s3"
9
+ format: "json"
10
+ mode: "all"
11
+ tables:
12
+ amazon_s3_records:
13
+ object: "exports/orders"
14
+ contract: records
package/recipes/flows/azure-blob.yaml ADDED
@@ -0,0 +1,14 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: azure_blob
4
+ source: azure_blob
5
+ ingest:
6
+ uses: adf-files
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-blob"
9
+ format: "json"
10
+ mode: "all"
11
+ tables:
12
+ azure_blob_records:
13
+ object: "exports/orders"
14
+ contract: records
package/recipes/flows/azure-cosmos-db-mongodb.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: azure_cosmos_db_mongodb
4
+ source: azure_cosmos_db_mongodb
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-cosmos-db-mongodb"
9
+ tables:
10
+ azure_cosmos_db_mongodb_records:
11
+ object: "orders"
12
+ contract: records
package/recipes/flows/azure-cosmos-db-nosql.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: azure_cosmos_db_nosql
4
+ source: azure_cosmos_db_nosql
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-cosmos-db-nosql"
9
+ tables:
10
+ azure_cosmos_db_nosql_records:
11
+ object: "orders"
12
+ contract: records
package/recipes/flows/azure-cosmos-postgresql.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: azure_cosmos_postgresql
4
+ source: azure_cosmos_postgresql
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-cosmos-postgresql"
9
+ tables:
10
+ azure_cosmos_postgresql_records:
11
+ object: "sales.orders"
12
+ contract: records
package/recipes/flows/azure-data-explorer.yaml ADDED
@@ -0,0 +1,13 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: azure_data_explorer
4
+ source: azure_data_explorer
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-data-explorer"
9
+ query: "SELECT * FROM orders"
10
+ tables:
11
+ azure_data_explorer_records:
12
+ object: "orders"
13
+ contract: records
package/recipes/flows/azure-databricks-delta.yaml ADDED
@@ -0,0 +1,15 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: azure_databricks_delta
4
+ source: azure_databricks_delta
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-databricks-delta"
9
+ runtime_kind: "azure"
10
+ integration_runtime: synthetic-azure-runtime
11
+ source_options: { "staging_sas": "${secret:staging-sas}", "staging_path": "staging/orders" }
12
+ tables:
13
+ azure_databricks_delta_records:
14
+ object: "sales.orders"
15
+ contract: records
package/recipes/flows/azure-files.yaml ADDED
@@ -0,0 +1,14 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: azure_files
4
+ source: azure_files
5
+ ingest:
6
+ uses: adf-files
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-files"
9
+ format: "json"
10
+ mode: "all"
11
+ tables:
12
+ azure_files_records:
13
+ object: "orders"
14
+ contract: records
package/recipes/flows/azure-mysql.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: azure_mysql
4
+ source: azure_mysql
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-mysql"
9
+ tables:
10
+ azure_mysql_records:
11
+ object: "orders"
12
+ contract: records
package/recipes/flows/azure-postgresql.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: azure_postgresql
4
+ source: azure_postgresql
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-postgresql"
9
+ tables:
10
+ azure_postgresql_records:
11
+ object: "sales.orders"
12
+ contract: records
package/recipes/flows/azure-sql-database.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: azure_sql_database
4
+ source: azure_sql_database
5
+ ingest:
6
+ uses: adf-copy
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-sql-database"
9
+ tables:
10
+ azure_sql_database_records:
11
+ object: "sales.dbo.orders"
12
+ contract: records
package/recipes/flows/azure-sql-managed-instance.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: azure_sql_managed_instance
4
+ source: azure_sql_managed_instance
5
+ ingest:
6
+ uses: adf-copy
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-sql-managed-instance"
9
+ tables:
10
+ azure_sql_managed_instance_records:
11
+ object: "sales.dbo.orders"
12
+ contract: records
package/recipes/flows/azure-synapse.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: azure_synapse
4
+ source: azure_synapse
5
+ ingest:
6
+ uses: adf-copy
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-synapse"
9
+ tables:
10
+ azure_synapse_records:
11
+ object: "sales.dbo.orders"
12
+ contract: records
package/recipes/flows/azure-table-storage.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: azure_table_storage
4
+ source: azure_table_storage
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/azure-table-storage"
9
+ tables:
10
+ azure_table_storage_records:
11
+ object: "orders"
12
+ contract: records
package/recipes/flows/cassandra.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: cassandra
4
+ source: cassandra
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/cassandra"
9
+ tables:
10
+ cassandra_records:
11
+ object: "sales.orders"
12
+ contract: records
package/recipes/flows/dataverse.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: dataverse
4
+ source: dataverse
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/dataverse"
9
+ tables:
10
+ dataverse_records:
11
+ object: "orders"
12
+ contract: records
package/recipes/flows/db2.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: db2
4
+ source: db2
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/db2"
9
+ tables:
10
+ db2_records:
11
+ object: "sales.orders"
12
+ contract: records
package/recipes/flows/dynamics-365.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: dynamics_365
4
+ source: dynamics_365
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/dynamics-365"
9
+ tables:
10
+ dynamics_365_records:
11
+ object: "orders"
12
+ contract: records
package/recipes/flows/dynamics-ax.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: dynamics_ax
4
+ source: dynamics_ax
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/dynamics-ax"
9
+ tables:
10
+ dynamics_ax_records:
11
+ object: "orders"
12
+ contract: records
package/recipes/flows/dynamics-crm.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: dynamics_crm
4
+ source: dynamics_crm
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/dynamics-crm"
9
+ tables:
10
+ dynamics_crm_records:
11
+ object: "orders"
12
+ contract: records
package/recipes/flows/fabric-lakehouse.yaml ADDED
@@ -0,0 +1,14 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: fabric_lakehouse
4
+ source: fabric_lakehouse
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/fabric-lakehouse"
9
+ runtime_kind: "azure"
10
+ integration_runtime: synthetic-azure-runtime
11
+ tables:
12
+ fabric_lakehouse_records:
13
+ object: "orders"
14
+ contract: records
package/recipes/flows/fabric-warehouse.yaml ADDED
@@ -0,0 +1,14 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: fabric_warehouse
4
+ source: fabric_warehouse
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/fabric-warehouse"
9
+ runtime_kind: "azure"
10
+ integration_runtime: synthetic-azure-runtime
11
+ tables:
12
+ fabric_warehouse_records:
13
+ object: "sales.orders"
14
+ contract: records
package/recipes/flows/files.yaml ADDED
@@ -0,0 +1,14 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: files
4
+ source: files
5
+ ingest:
6
+ uses: adf-files
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/files"
9
+ format: "json"
10
+ mode: "all"
11
+ tables:
12
+ files_records:
13
+ object: "orders"
14
+ contract: records
package/recipes/flows/ftp.yaml ADDED
@@ -0,0 +1,14 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: ftp
4
+ source: ftp
5
+ ingest:
6
+ uses: adf-files
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/ftp"
9
+ format: "json"
10
+ mode: "all"
11
+ tables:
12
+ ftp_records:
13
+ object: "orders"
14
+ contract: records
package/recipes/flows/google-ads.yaml ADDED
@@ -0,0 +1,13 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: google_ads
4
+ source: google_ads
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/google-ads"
9
+ query: "SELECT customer.id, customer.descriptive_name FROM customer"
10
+ tables:
11
+ google_ads_records:
12
+ object: "orders"
13
+ contract: records
package/recipes/flows/google-bigquery.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: google_bigquery
4
+ source: google_bigquery
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/google-bigquery"
9
+ tables:
10
+ google_bigquery_records:
11
+ object: "sales.orders"
12
+ contract: records
package/recipes/flows/google-cloud-storage.yaml ADDED
@@ -0,0 +1,14 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: google_cloud_storage
4
+ source: google_cloud_storage
5
+ ingest:
6
+ uses: adf-files
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/google-cloud-storage"
9
+ format: "json"
10
+ mode: "all"
11
+ tables:
12
+ google_cloud_storage_records:
13
+ object: "exports/orders"
14
+ contract: records
package/recipes/flows/greenplum.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: greenplum
4
+ source: greenplum
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/greenplum"
9
+ tables:
10
+ greenplum_records:
11
+ object: "sales.orders"
12
+ contract: records
package/recipes/flows/hdfs.yaml ADDED
@@ -0,0 +1,14 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: hdfs
4
+ source: hdfs
5
+ ingest:
6
+ uses: adf-files
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/hdfs"
9
+ format: "json"
10
+ mode: "all"
11
+ tables:
12
+ hdfs_records:
13
+ object: "orders"
14
+ contract: records
package/recipes/flows/hive.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: hive
4
+ source: hive
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/hive"
9
+ tables:
10
+ hive_records:
11
+ object: "sales.orders"
12
+ contract: records
package/recipes/flows/http.yaml ADDED
@@ -0,0 +1,18 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: http
4
+ source: http
5
+ ingest:
6
+ uses: adf-files
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/http"
9
+ format: "json"
10
+ mode: "all"
11
+ files: "response.json"
12
+ source_options:
13
+ request_method: POST
14
+ request_body: '{"status":"active"}'
15
+ tables:
16
+ http_records:
17
+ object: "orders.json"
18
+ contract: records
package/recipes/flows/hubspot.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: hubspot
4
+ source: hubspot
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/hubspot"
9
+ tables:
10
+ hubspot_records:
11
+ object: "orders"
12
+ contract: records
package/recipes/flows/impala.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: impala
4
+ source: impala
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/impala"
9
+ tables:
10
+ impala_records:
11
+ object: "sales.orders"
12
+ contract: records
package/recipes/flows/informix.yaml ADDED
@@ -0,0 +1,13 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: informix
4
+ source: informix
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/informix"
9
+ runtime_kind: "self-hosted"
10
+ tables:
11
+ informix_records:
12
+ object: "orders"
13
+ contract: records
package/recipes/flows/jira.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: jira
4
+ source: jira
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/jira"
9
+ tables:
10
+ jira_records:
11
+ object: "sales.orders"
12
+ contract: records
package/recipes/flows/mariadb.yaml ADDED
@@ -0,0 +1,13 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: mariadb
4
+ source: mariadb
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/mariadb"
9
+ query: "SELECT * FROM orders"
10
+ tables:
11
+ mariadb_records:
12
+ object: "orders"
13
+ contract: records
package/recipes/flows/microsoft-365.yaml ADDED
@@ -0,0 +1,19 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: microsoft_365
4
+ source: microsoft_365
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ resource_names:
9
+ json_landing_linked_service: "ls_${project.id}_office_landing"
10
+ json_landing_dataset: "ds_${project.id}_office_json"
11
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/microsoft-365"
12
+ runtime_kind: "azure"
13
+ integration_runtime: synthetic-azure-runtime
14
+ format: "json"
15
+ source_options: { "requires_date_filter": true, "date_filter_column": "CreatedDateTime" }
16
+ tables:
17
+ microsoft_365_records:
18
+ object: "BasicDataSet_v0.Message_v1"
19
+ contract: records
package/recipes/flows/microsoft-access.yaml ADDED
@@ -0,0 +1,13 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: microsoft_access
4
+ source: microsoft_access
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/microsoft-access"
9
+ runtime_kind: "self-hosted"
10
+ tables:
11
+ microsoft_access_records:
12
+ object: "orders"
13
+ contract: records
package/recipes/flows/mongodb-atlas.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: mongodb_atlas
4
+ source: mongodb_atlas
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/mongodb-atlas"
9
+ tables:
10
+ mongodb_atlas_records:
11
+ object: "orders"
12
+ contract: records
package/recipes/flows/mongodb.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: mongodb
4
+ source: mongodb
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/mongodb"
9
+ tables:
10
+ mongodb_records:
11
+ object: "orders"
12
+ contract: records
package/recipes/flows/mysql.yaml ADDED
@@ -0,0 +1,12 @@
1
+ apiVersion: ingestron.io/v1
2
+ kind: Flow
3
+ id: mysql
4
+ source: mysql
5
+ ingest:
6
+ uses: adf-table
7
+ with:
8
+ landing: "abfss://landing@syntheticlake.dfs.core.windows.net/native/mysql"
9
+ tables:
10
+ mysql_records:
11
+ object: "orders"
12
+ contract: records