ingestron 0.17.7 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. package/NOTICE +3 -5
  2. package/README.md +26 -74
  3. package/THIRD_PARTY_NOTICES +148 -0
  4. package/dist/main.js +29188 -0
  5. package/package.json +32 -65
  6. package/project-recipes/adf/connections/adls-gen2.yaml +7 -0
  7. package/project-recipes/adf/connections/amazon-rds-oracle.yaml +9 -0
  8. package/project-recipes/adf/connections/amazon-rds-sqlserver.yaml +9 -0
  9. package/project-recipes/adf/connections/amazon-redshift.yaml +10 -0
  10. package/project-recipes/adf/connections/amazon-s3.yaml +8 -0
  11. package/project-recipes/adf/connections/azure-blob.yaml +7 -0
  12. package/project-recipes/adf/connections/azure-cosmos-db-mongodb.yaml +8 -0
  13. package/project-recipes/adf/connections/azure-cosmos-db-nosql.yaml +8 -0
  14. package/project-recipes/adf/connections/azure-cosmos-postgresql.yaml +10 -0
  15. package/project-recipes/adf/connections/azure-data-explorer.yaml +8 -0
  16. package/project-recipes/adf/connections/azure-databricks-delta.yaml +9 -0
  17. package/project-recipes/adf/connections/azure-files.yaml +8 -0
  18. package/project-recipes/adf/connections/azure-mysql.yaml +7 -0
  19. package/project-recipes/adf/connections/azure-postgresql.yaml +10 -0
  20. package/project-recipes/adf/connections/azure-sql-database.yaml +9 -0
  21. package/project-recipes/adf/connections/azure-sql-managed-instance.yaml +9 -0
  22. package/project-recipes/adf/connections/azure-synapse.yaml +9 -0
  23. package/project-recipes/adf/connections/azure-table-storage.yaml +7 -0
  24. package/project-recipes/adf/connections/cassandra.yaml +9 -0
  25. package/project-recipes/adf/connections/dataverse.yaml +9 -0
  26. package/project-recipes/adf/connections/db2.yaml +7 -0
  27. package/project-recipes/adf/connections/dynamics-365.yaml +9 -0
  28. package/project-recipes/adf/connections/dynamics-ax.yaml +11 -0
  29. package/project-recipes/adf/connections/dynamics-crm.yaml +9 -0
  30. package/project-recipes/adf/connections/fabric-lakehouse.yaml +8 -0
  31. package/project-recipes/adf/connections/fabric-warehouse.yaml +9 -0
  32. package/project-recipes/adf/connections/files.yaml +9 -0
  33. package/project-recipes/adf/connections/ftp.yaml +9 -0
  34. package/project-recipes/adf/connections/google-ads.yaml +12 -0
  35. package/project-recipes/adf/connections/google-bigquery.yaml +8 -0
  36. package/project-recipes/adf/connections/google-cloud-storage.yaml +8 -0
  37. package/project-recipes/adf/connections/greenplum.yaml +10 -0
  38. package/project-recipes/adf/connections/hdfs.yaml +9 -0
  39. package/project-recipes/adf/connections/hive.yaml +11 -0
  40. package/project-recipes/adf/connections/http.yaml +7 -0
  41. package/project-recipes/adf/connections/hubspot.yaml +10 -0
  42. package/project-recipes/adf/connections/impala.yaml +11 -0
  43. package/project-recipes/adf/connections/informix.yaml +9 -0
  44. package/project-recipes/adf/connections/jira.yaml +9 -0
  45. package/project-recipes/adf/connections/mariadb.yaml +10 -0
  46. package/project-recipes/adf/connections/microsoft-365.yaml +10 -0
  47. package/project-recipes/adf/connections/microsoft-access.yaml +9 -0
  48. package/project-recipes/adf/connections/mongodb-atlas.yaml +8 -0
  49. package/project-recipes/adf/connections/mongodb.yaml +8 -0
  50. package/project-recipes/adf/connections/mysql.yaml +10 -0
  51. package/project-recipes/adf/connections/netezza.yaml +11 -0
  52. package/project-recipes/adf/connections/odata.yaml +9 -0
  53. package/project-recipes/adf/connections/odbc.yaml +9 -0
  54. package/project-recipes/adf/connections/oracle-cloud-storage.yaml +9 -0
  55. package/project-recipes/adf/connections/oracle.yaml +9 -0
  56. package/project-recipes/adf/connections/postgresql.yaml +10 -0
  57. package/project-recipes/adf/connections/presto.yaml +11 -0
  58. package/project-recipes/adf/connections/quickbooks.yaml +10 -0
  59. package/project-recipes/adf/connections/rest.yaml +12 -0
  60. package/project-recipes/adf/connections/s3-compatible.yaml +9 -0
  61. package/project-recipes/adf/connections/salesforce-service-cloud.yaml +10 -0
  62. package/project-recipes/adf/connections/salesforce.yaml +10 -0
  63. package/project-recipes/adf/connections/sap-bw-mdx.yaml +11 -0
  64. package/project-recipes/adf/connections/sap-bw-open-hub.yaml +11 -0
  65. package/project-recipes/adf/connections/sap-c4c.yaml +9 -0
  66. package/project-recipes/adf/connections/sap-ecc.yaml +9 -0
  67. package/project-recipes/adf/connections/sap-hana.yaml +7 -0
  68. package/project-recipes/adf/connections/sap-table.yaml +11 -0
  69. package/project-recipes/adf/connections/servicenow.yaml +9 -0
  70. package/project-recipes/adf/connections/sftp.yaml +10 -0
  71. package/project-recipes/adf/connections/sharepoint-online-list.yaml +10 -0
  72. package/project-recipes/adf/connections/shopify.yaml +8 -0
  73. package/project-recipes/adf/connections/snowflake.yaml +11 -0
  74. package/project-recipes/adf/connections/spark.yaml +11 -0
  75. package/project-recipes/adf/connections/sqlserver.yaml +9 -0
  76. package/project-recipes/adf/connections/square.yaml +10 -0
  77. package/project-recipes/adf/connections/sybase.yaml +10 -0
  78. package/project-recipes/adf/connections/teradata.yaml +9 -0
  79. package/project-recipes/adf/connections/vertica.yaml +10 -0
  80. package/project-recipes/adf/connections/web-table.yaml +7 -0
  81. package/project-recipes/adf/connections/xero.yaml +10 -0
  82. package/project-recipes/adf/contracts/records.yaml +12 -0
  83. package/project-recipes/adf/contracts/salesforce-account.yaml +12 -0
  84. package/project-recipes/adf/flows/adls-gen2.yaml +14 -0
  85. package/project-recipes/adf/flows/amazon-rds-oracle.yaml +12 -0
  86. package/project-recipes/adf/flows/amazon-rds-sqlserver.yaml +12 -0
  87. package/project-recipes/adf/flows/amazon-redshift.yaml +12 -0
  88. package/project-recipes/adf/flows/amazon-s3.yaml +14 -0
  89. package/project-recipes/adf/flows/azure-blob.yaml +14 -0
  90. package/project-recipes/adf/flows/azure-cosmos-db-mongodb.yaml +12 -0
  91. package/project-recipes/adf/flows/azure-cosmos-db-nosql.yaml +12 -0
  92. package/project-recipes/adf/flows/azure-cosmos-postgresql.yaml +12 -0
  93. package/project-recipes/adf/flows/azure-data-explorer.yaml +13 -0
  94. package/project-recipes/adf/flows/azure-databricks-delta.yaml +15 -0
  95. package/project-recipes/adf/flows/azure-files.yaml +14 -0
  96. package/project-recipes/adf/flows/azure-mysql.yaml +12 -0
  97. package/project-recipes/adf/flows/azure-postgresql.yaml +12 -0
  98. package/project-recipes/adf/flows/azure-sql-database.yaml +12 -0
  99. package/project-recipes/adf/flows/azure-sql-managed-instance.yaml +12 -0
  100. package/project-recipes/adf/flows/azure-synapse.yaml +12 -0
  101. package/project-recipes/adf/flows/azure-table-storage.yaml +12 -0
  102. package/project-recipes/adf/flows/cassandra.yaml +12 -0
  103. package/project-recipes/adf/flows/dataverse.yaml +12 -0
  104. package/project-recipes/adf/flows/db2.yaml +12 -0
  105. package/project-recipes/adf/flows/dynamics-365.yaml +12 -0
  106. package/project-recipes/adf/flows/dynamics-ax.yaml +12 -0
  107. package/project-recipes/adf/flows/dynamics-crm.yaml +12 -0
  108. package/project-recipes/adf/flows/fabric-lakehouse.yaml +14 -0
  109. package/project-recipes/adf/flows/fabric-warehouse.yaml +14 -0
  110. package/project-recipes/adf/flows/files.yaml +14 -0
  111. package/project-recipes/adf/flows/ftp.yaml +14 -0
  112. package/project-recipes/adf/flows/google-ads.yaml +13 -0
  113. package/project-recipes/adf/flows/google-bigquery.yaml +12 -0
  114. package/project-recipes/adf/flows/google-cloud-storage.yaml +14 -0
  115. package/project-recipes/adf/flows/greenplum.yaml +12 -0
  116. package/project-recipes/adf/flows/hdfs.yaml +14 -0
  117. package/project-recipes/adf/flows/hive.yaml +12 -0
  118. package/project-recipes/adf/flows/http.yaml +18 -0
  119. package/project-recipes/adf/flows/hubspot.yaml +12 -0
  120. package/project-recipes/adf/flows/impala.yaml +12 -0
  121. package/project-recipes/adf/flows/informix.yaml +13 -0
  122. package/project-recipes/adf/flows/jira.yaml +12 -0
  123. package/project-recipes/adf/flows/mariadb.yaml +13 -0
  124. package/project-recipes/adf/flows/microsoft-365.yaml +19 -0
  125. package/project-recipes/adf/flows/microsoft-access.yaml +13 -0
  126. package/project-recipes/adf/flows/mongodb-atlas.yaml +12 -0
  127. package/project-recipes/adf/flows/mongodb.yaml +12 -0
  128. package/project-recipes/adf/flows/mysql.yaml +12 -0
  129. package/project-recipes/adf/flows/netezza.yaml +13 -0
  130. package/project-recipes/adf/flows/odata.yaml +12 -0
  131. package/project-recipes/adf/flows/odbc.yaml +13 -0
  132. package/project-recipes/adf/flows/oracle-cloud-storage.yaml +14 -0
  133. package/project-recipes/adf/flows/oracle.yaml +12 -0
  134. package/project-recipes/adf/flows/postgresql.yaml +18 -0
  135. package/project-recipes/adf/flows/presto.yaml +12 -0
  136. package/project-recipes/adf/flows/quickbooks.yaml +12 -0
  137. package/project-recipes/adf/flows/rest.yaml +21 -0
  138. package/project-recipes/adf/flows/s3-compatible.yaml +14 -0
  139. package/project-recipes/adf/flows/salesforce-service-cloud.yaml +12 -0
  140. package/project-recipes/adf/flows/salesforce.yaml +13 -0
  141. package/project-recipes/adf/flows/sap-bw-mdx.yaml +14 -0
  142. package/project-recipes/adf/flows/sap-bw-open-hub.yaml +13 -0
  143. package/project-recipes/adf/flows/sap-c4c.yaml +12 -0
  144. package/project-recipes/adf/flows/sap-ecc.yaml +13 -0
  145. package/project-recipes/adf/flows/sap-hana.yaml +14 -0
  146. package/project-recipes/adf/flows/sap-table.yaml +13 -0
  147. package/project-recipes/adf/flows/servicenow.yaml +12 -0
  148. package/project-recipes/adf/flows/sftp.yaml +14 -0
  149. package/project-recipes/adf/flows/sharepoint-online-list.yaml +12 -0
  150. package/project-recipes/adf/flows/shopify.yaml +12 -0
  151. package/project-recipes/adf/flows/snowflake.yaml +15 -0
  152. package/project-recipes/adf/flows/spark.yaml +12 -0
  153. package/project-recipes/adf/flows/sqlserver.yaml +12 -0
  154. package/project-recipes/adf/flows/square.yaml +13 -0
  155. package/project-recipes/adf/flows/sybase.yaml +13 -0
  156. package/project-recipes/adf/flows/teradata.yaml +12 -0
  157. package/project-recipes/adf/flows/vertica.yaml +12 -0
  158. package/project-recipes/adf/flows/web-table.yaml +14 -0
  159. package/project-recipes/adf/flows/xero.yaml +12 -0
  160. package/project-recipes/adf/platform.yaml +29 -0
  161. package/project-recipes/adf/project.yaml +10 -0
  162. package/project-recipes/adf/sample.yaml +3 -0
  163. package/project-recipes/adf/standard.yaml +15 -0
  164. package/project-templates/kafka/README.md +12 -0
  165. package/project-templates/kafka/connections/events.yaml +8 -0
  166. package/project-templates/kafka/contracts/page_views.yaml +13 -0
  167. package/project-templates/kafka/contracts/purchases.yaml +13 -0
  168. package/project-templates/kafka/flows/events.yaml +10 -0
  169. package/project-templates/kafka/project.yaml +10 -0
  170. package/project-templates/kafka/standard.yaml +4 -0
  171. package/project-templates/kafka/template.yaml +3 -0
  172. package/project-templates/minimal/README.md +10 -0
  173. package/project-templates/minimal/connections/shop.yaml +7 -0
  174. package/project-templates/minimal/contracts/orders.yaml +14 -0
  175. package/project-templates/minimal/flows/orders.yaml +6 -0
  176. package/project-templates/minimal/project.yaml +11 -0
  177. package/project-templates/minimal/standard.yaml +4 -0
  178. package/project-templates/minimal/template.yaml +7 -0
  179. package/project-templates/private-files/README.md +29 -0
  180. package/project-templates/private-files/connections/share.yaml +10 -0
  181. package/project-templates/private-files/contracts/sales.yaml +14 -0
  182. package/project-templates/private-files/contracts/stock.yaml +14 -0
  183. package/project-templates/private-files/flows/stores.yaml +9 -0
  184. package/project-templates/private-files/platform.yaml +11 -0
  185. package/project-templates/private-files/project.yaml +19 -0
  186. package/project-templates/private-files/standard.yaml +11 -0
  187. package/project-templates/private-files/template.yaml +10 -0
  188. package/project-templates/private-sql-server/README.md +25 -0
  189. package/project-templates/private-sql-server/connections/erp.yaml +10 -0
  190. package/project-templates/private-sql-server/contracts/customers.yaml +14 -0
  191. package/project-templates/private-sql-server/contracts/invoices.yaml +15 -0
  192. package/project-templates/private-sql-server/flows/erp.yaml +13 -0
  193. package/project-templates/private-sql-server/platform.yaml +11 -0
  194. package/project-templates/private-sql-server/project.yaml +19 -0
  195. package/project-templates/private-sql-server/standard.yaml +17 -0
  196. package/project-templates/private-sql-server/template.yaml +10 -0
  197. package/project-templates/pyspark/README.md +12 -0
  198. package/project-templates/pyspark/connections/billing.yaml +7 -0
  199. package/project-templates/pyspark/contracts/invoices.yaml +16 -0
  200. package/project-templates/pyspark/contracts/monthly_revenue.yaml +14 -0
  201. package/project-templates/pyspark/flows/billing.yaml +6 -0
  202. package/project-templates/pyspark/project.yaml +11 -0
  203. package/project-templates/pyspark/standard.yaml +4 -0
  204. package/project-templates/pyspark/template.yaml +3 -0
  205. package/project-templates/pyspark/transforms/monthly_revenue.py +10 -0
  206. package/project-templates/pyspark/transforms/monthly_revenue.yaml +10 -0
  207. package/project-templates/retail/README.md +22 -0
  208. package/project-templates/retail/connections/shop.yaml +8 -0
  209. package/project-templates/retail/contracts/customers.yaml +12 -0
  210. package/project-templates/retail/contracts/daily_sales.yaml +19 -0
  211. package/project-templates/retail/contracts/orders.yaml +16 -0
  212. package/project-templates/retail/flows/shop.yaml +18 -0
  213. package/project-templates/retail/platform.yaml +21 -0
  214. package/project-templates/retail/project.yaml +18 -0
  215. package/project-templates/retail/standard.yaml +15 -0
  216. package/project-templates/retail/template.yaml +10 -0
  217. package/project-templates/retail/transforms/daily_sales.sql +7 -0
  218. package/project-templates/retail/transforms/daily_sales.yaml +7 -0
  219. package/project-templates/salesforce/README.md +6 -0
  220. package/project-templates/salesforce/connections/salesforce.yaml +7 -0
  221. package/project-templates/salesforce/contracts/accounts.yaml +13 -0
  222. package/project-templates/salesforce/contracts/opportunities.yaml +14 -0
  223. package/project-templates/salesforce/flows/salesforce.yaml +7 -0
  224. package/project-templates/salesforce/project.yaml +9 -0
  225. package/project-templates/salesforce/standard.yaml +4 -0
  226. package/project-templates/salesforce/template.yaml +3 -0
  227. package/project-templates/sql-server/README.md +17 -0
  228. package/project-templates/sql-server/connections/erp.yaml +8 -0
  229. package/project-templates/sql-server/contracts/customer_balances.yaml +13 -0
  230. package/project-templates/sql-server/contracts/customers.yaml +13 -0
  231. package/project-templates/sql-server/contracts/invoices.yaml +14 -0
  232. package/project-templates/sql-server/flows/erp.yaml +13 -0
  233. package/project-templates/sql-server/platform.yaml +6 -0
  234. package/project-templates/sql-server/project.yaml +13 -0
  235. package/project-templates/sql-server/standard.yaml +9 -0
  236. package/project-templates/sql-server/template.yaml +3 -0
  237. package/project-templates/sql-server/transforms/customer_balances.sql +5 -0
  238. package/project-templates/sql-server/transforms/customer_balances.yaml +6 -0
  239. package/schemas/NOTICE.md +7 -0
  240. package/schemas/connection.schema.yaml +37 -0
  241. package/schemas/flow.schema.yaml +124 -0
  242. package/schemas/odcs-v3.1.0.schema.json +2808 -0
  243. package/schemas/platform.schema.yaml +260 -0
  244. package/schemas/standard.schema.yaml +211 -0
  245. package/schemas/transform.schema.yaml +108 -0
  246. package/templates/apply-contract.py.liquid +58 -0
  247. package/templates/change-data.sql.liquid +20 -0
  248. package/templates/materialised-view.py.liquid +28 -0
  249. package/templates/materialised-view.sql.liquid +4 -0
  250. package/templates/read-files.liquid +7 -0
  251. package/templates/streaming-table.sql.liquid +3 -0
  252. package/templates/table.liquid +17 -0
  253. package/CHANGELOG.md +0 -148
  254. package/SECURITY.md +0 -32
  255. package/THIRD_PARTY_NOTICES.md +0 -3678
  256. package/build/cli/cli/index.d.ts +0 -2
  257. package/build/cli/cli/index.js +0 -685
  258. package/build/cli/cli/operation-worker.d.ts +0 -1
  259. package/build/cli/cli/operation-worker.js +0 -5
  260. package/build/cli/cli/output.d.ts +0 -3
  261. package/build/cli/cli/output.js +0 -226
  262. package/build/cli/cli/plugin-browser.d.ts +0 -14
  263. package/build/cli/cli/plugin-browser.js +0 -74
  264. package/build/cli/cli/plugin-resolution.d.ts +0 -72
  265. package/build/cli/cli/plugin-resolution.js +0 -216
  266. package/build/cli/cli/presentation.d.ts +0 -12
  267. package/build/cli/cli/presentation.js +0 -57
  268. package/build/cli/cli/workflow-commands.d.ts +0 -15
  269. package/build/cli/cli/workflow-commands.js +0 -415
  270. package/build/cli/mcp/index.d.ts +0 -3
  271. package/build/cli/mcp/index.js +0 -67
  272. package/build/cli/version.d.ts +0 -2
  273. package/build/cli/version.js +0 -14
@@ -1,2 +0,0 @@
1
- #!/usr/bin/env node
2
- export {};
@@ -1,685 +0,0 @@
1
- #!/usr/bin/env node
2
- import { Command } from "commander";
3
- import * as prompts from "@clack/prompts";
4
- import { resolve, basename, dirname } from "node:path";
5
- import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
6
- import { parseDocument } from "yaml";
7
- import { execute, executeAsync, operationSchemas, } from "@ingestron/core";
8
- import { fence } from "@ingestron/core/adapter";
9
- import { Configuration } from "@ingestron/core/adapter";
10
- import { check, Problem } from "@ingestron/core/adapter";
11
- import { version } from "../version.js";
12
- import { terminal, exitCode } from "./output.js";
13
- import { decorate, banner, backgroundOperation } from "./presentation.js";
14
- import { executePluginOperation } from "./plugin-resolution.js";
15
- import { workflowCommands } from "./workflow-commands.js";
16
- const app = new Command()
17
- .name("ingestron")
18
- .description("Configure flows once. Generate reviewable native data engineering assets offline.")
19
- .version(version)
20
- .option("--project <directory>", "Project root", ".")
21
- .option("-e, --environment <name>", "Selected environment", "dev")
22
- .option("--env <name>", "Alias for --environment")
23
- .option("--json", "Versioned JSON output; progress goes to stderr")
24
- .option("--plain", "Plain noninteractive output; no styling or animation")
25
- .option("--no-color", "Disable terminal colour and decoration")
26
- .option("--no-progress", "Disable progress animation")
27
- .option("--verbose", "Show full result details in human-readable output")
28
- .option("--no-input", "Never ask interactive questions")
29
- .option("--dry-run", "Preview local file changes without applying");
30
- const opts = () => app.opts();
31
- const presentation = () => ({
32
- tty: !!process.stdout.isTTY,
33
- plain: opts().plain,
34
- colour: opts().color,
35
- progress: opts().progress,
36
- json: !!opts().json,
37
- });
38
- const context = (root) => ({
39
- root: root ?? resolve(opts().project),
40
- environment: opts().env ?? opts().environment,
41
- allowWrite: true,
42
- allowNetwork: true,
43
- allowExecute: true,
44
- });
45
- function print(result, args) {
46
- process.exitCode = exitCode(result);
47
- if (opts().json || process.argv.includes("--json"))
48
- process.stdout.write(JSON.stringify(result) + "\n");
49
- else
50
- (result.ok ? process.stdout : process.stderr).write(decorate(terminal(result, !!opts().verbose, args), result.ok, presentation()) + "\n");
51
- }
52
- // Commander failures must obey the same JSON contract as operation failures.
53
- // Help/version remain successful envelopes in JSON mode; no progress goes to stdout.
54
- let helpText = "";
55
- app.exitOverride();
56
- app.configureOutput({
57
- writeOut: (text) => {
58
- if (process.argv.includes("--json"))
59
- helpText += text;
60
- else
61
- process.stdout.write(text.startsWith("Usage: ingestron [")
62
- ? banner(text, presentation())
63
- : text);
64
- },
65
- writeErr: (text) => {
66
- if (text.startsWith("Usage:")) {
67
- if (process.argv.includes("--json"))
68
- helpText += text;
69
- else
70
- process.stdout.write(text.startsWith("Usage: ingestron [")
71
- ? banner(text, presentation())
72
- : text);
73
- }
74
- },
75
- outputError: () => { },
76
- });
77
- const run = async (name, args = {}, ctx = context()) => {
78
- const result = ["build", "plan", "generate", "packages_install"].includes(name)
79
- ? await backgroundOperation(ctx, name, args, `${name === "packages_install" ? "Installing plugin" : "Preparing native assets"}`, presentation())
80
- : await executeAsync(ctx, name, args);
81
- print(result, args);
82
- };
83
- // Structured host boundary: request bytes are data, never shell arguments.
84
- app
85
- .command("operation <name>")
86
- .description("Invoke a versioned core operation with a JSON request on stdin")
87
- .option("--allow-write", "Permit this operation to write project files")
88
- .option("--allow-network", "Permit this operation to download packages")
89
- .option("--allow-execute", "Permit this operation to execute a local runtime")
90
- .action(async (name, flags) => {
91
- check(Object.hasOwn(operationSchemas, name), "OPERATION", "Unknown operation");
92
- let bytes = 0;
93
- const chunks = [];
94
- for await (const chunk of process.stdin) {
95
- const value = Buffer.from(chunk);
96
- bytes += value.length;
97
- check(bytes <= 1024 * 1024, "LIMIT", "Operation request exceeds 1 MiB");
98
- chunks.push(value);
99
- }
100
- let request;
101
- try {
102
- request = JSON.parse(Buffer.concat(chunks).toString("utf8") || "{}");
103
- }
104
- catch {
105
- throw new Problem("INPUT", "Expected a JSON operation request on stdin");
106
- }
107
- const result = await executePluginOperation({
108
- ...context(),
109
- allowWrite: !!flags.allowWrite && !opts().dryRun,
110
- allowNetwork: !!flags.allowNetwork && !opts().dryRun,
111
- allowExecute: !!flags.allowExecute && !opts().dryRun,
112
- }, name, request);
113
- print(result, request);
114
- });
115
- function author(name, args, ctx = context()) {
116
- const result = execute(ctx, name, args);
117
- return print(!result.ok || opts().dryRun
118
- ? result
119
- : execute(ctx, "apply", { proposal: result.result }));
120
- }
121
- function readData(file) {
122
- const doc = parseDocument(new Configuration(context().root).text(file), {
123
- uniqueKeys: true,
124
- });
125
- check(!doc.errors.length, "YAML", doc.errors[0]?.message ?? "Invalid YAML");
126
- return doc.toJS();
127
- }
128
- function save(file, value) {
129
- const absolute = fence(resolve(opts().project), file);
130
- mkdirSync(dirname(absolute), { recursive: true });
131
- writeFileSync(absolute, JSON.stringify(value, null, 2) + "\n");
132
- }
133
- const canPrompt = () => !!process.stdin.isTTY &&
134
- opts().input !== false &&
135
- !opts().json &&
136
- !opts().plain &&
137
- process.env.TERM !== "dumb" &&
138
- !process.env.CI;
139
- async function answer(value) {
140
- const result = await value;
141
- if (prompts.isCancel(result)) {
142
- process.exitCode = 130;
143
- throw new Problem("CANCELLED", "Cancelled");
144
- }
145
- return result;
146
- }
147
- app
148
- .command("init [directory]")
149
- .description("Create a project; install a provider when ready to build")
150
- .option("--id <id>", "Project identifier")
151
- .option("--provider <reference>", "Installed repository or exact reference, e.g. owner/provider@1.0.0")
152
- .option("--environments <names>", "Comma-separated environments (default: dev,test,prod; --env selects one)")
153
- .action(async (directory = ".", command) => {
154
- const root = resolve(opts().project, directory);
155
- const id = command.id ??
156
- (canPrompt()
157
- ? await answer(prompts.text({
158
- message: "Project identifier",
159
- initialValue: basename(root),
160
- }))
161
- : basename(root));
162
- author("initialise", {
163
- id,
164
- provider: command.provider,
165
- environments: (command.environments ??
166
- (opts().env !== undefined ||
167
- app.getOptionValueSource("environment") === "cli"
168
- ? context().environment
169
- : "dev,test,prod"))
170
- .split(",")
171
- .map((name) => name.trim()),
172
- }, context(root));
173
- });
174
- const flow = app
175
- .command("flow")
176
- .description("Create and inspect ingestion/transformation flows");
177
- flow
178
- .command("add <id>")
179
- .option("--kind <kind>", "ingestion or transformation", "ingestion")
180
- .option("--contract <id>", "Contract ID or project-relative file")
181
- .option("--source <id>", "Named source; otherwise inferred from contract provenance")
182
- .option("--standard <name>", "Provider ingestion standard, e.g. snapshot-with-history@v1")
183
- .option("--provider <configuration>", "Provider configuration for this flow")
184
- .option("--source-kind <kind>", "Provider source kind")
185
- .option("--format <format>", "Provider-supported file format (for example csv, json, parquet)")
186
- .option("--source-binding <id>", "Existing or new environment source binding")
187
- .option("--input-dataset <id>", "Published source dataset for a native SQL transformation")
188
- .action(async (id, command) => {
189
- let standard = command.standard;
190
- if (command.kind === "ingestion" && !standard && canPrompt())
191
- standard = await answer(prompts.text({
192
- message: "Standard from the installed provider (name@version)",
193
- validate: (value) => value?.trim() ? undefined : "Enter a provider standard",
194
- }));
195
- author("flow_create", {
196
- ...(command.contract ? { contract: command.contract } : {}),
197
- ...(command.source ? { source: command.source } : {}),
198
- id,
199
- kind: command.kind,
200
- ...(standard ? { standard } : {}),
201
- ...(command.provider ? { provider: command.provider } : {}),
202
- ...(command.sourceKind ? { sourceKind: command.sourceKind } : {}),
203
- ...(command.format ? { format: command.format } : {}),
204
- ...(command.inputDataset ? { inputDataset: command.inputDataset } : {}),
205
- ...(command.sourceBinding
206
- ? { sourceBinding: command.sourceBinding }
207
- : {}),
208
- });
209
- });
210
- flow
211
- .command("configure <id>")
212
- .requiredOption("--patch <file>")
213
- .action((id, command) => author("flow_configure", { flow: id, patch: readData(command.patch) }));
214
- app
215
- .command("step")
216
- .command("add <id>")
217
- .requiredOption("--flow <id>")
218
- .requiredOption("--uses <activity>")
219
- .option("--after <id>")
220
- .option("--with <file>")
221
- .action((id, command) => author("step_add", {
222
- flow: command.flow,
223
- step: {
224
- id,
225
- uses: command.uses,
226
- with: command.with ? readData(command.with) : {},
227
- },
228
- ...(command.after ? { after: command.after } : {}),
229
- }));
230
- flow
231
- .command("list")
232
- .description("List configured flows, table names and published outputs")
233
- .action(() => run("inspect"));
234
- const table = app
235
- .command("table")
236
- .description("Declare ingestion tables once per flow");
237
- table
238
- .command("add <id>")
239
- .requiredOption("--flow <id>", "Owning ingestion flow")
240
- .requiredOption("--contract <file>", "Source ODCS file inside the project")
241
- .action((id, command) => author("table_add", {
242
- id,
243
- flow: command.flow,
244
- contract: command.contract,
245
- }));
246
- table
247
- .command("configure <id>")
248
- .requiredOption("--flow <id>", "Owning flow")
249
- .requiredOption("--patch <file>", "YAML/JSON override patch")
250
- .action((id, command) => author("table_configure", {
251
- id,
252
- flow: command.flow,
253
- patch: readData(command.patch),
254
- }));
255
- table
256
- .command("list")
257
- .description("List configured table names grouped by flow")
258
- .action(() => run("tables"));
259
- const dataset = app.command("dataset");
260
- dataset.command("list").action(() => run("datasets"));
261
- dataset
262
- .command("add <id>")
263
- .requiredOption("--flow <id>", "Transformation flow")
264
- .requiredOption("--contract <file>", "Output ODCS file")
265
- .requiredOption("--from <port>", "Producing port, e.g. steps.ready.outputs.table")
266
- .action((id, command) => author("dataset_add", {
267
- id,
268
- flow: command.flow,
269
- contract: command.contract,
270
- from: command.from,
271
- }));
272
- const config = app
273
- .command("config")
274
- .description("Configure existing resources and non-secret values");
275
- config
276
- .command("set <path>")
277
- .requiredOption("--value <value>", "YAML scalar or mapping")
278
- .action((path, command) => author("config_set", { path, value: parseDocument(command.value).toJS() }));
279
- config
280
- .command("explain <path>")
281
- .action((path) => run("config_explain", { path }));
282
- config.command("fill").action(async () => {
283
- const result = execute(context(), "resolve", {});
284
- if (!result.ok) {
285
- print(result);
286
- return;
287
- }
288
- check(canPrompt(), "INPUT", "config fill requires an interactive terminal; use config set or doctor with --no-input/--json");
289
- const profile = result.result.project.environments[context().environment];
290
- const fields = [];
291
- const walk = (value, path) => {
292
- if (value && typeof value === "object" && value.$env)
293
- fields.push({ path, variable: value.$env });
294
- else if (value && typeof value === "object")
295
- for (const [k, child] of Object.entries(value))
296
- walk(child, path ? path + "." + k : k);
297
- };
298
- walk(profile.bindings, "bindings");
299
- walk(profile.values, "values");
300
- for (const field of fields) {
301
- const value = await answer(prompts.text({
302
- message: `${field.path} (${field.variable})`,
303
- placeholder: "Existing resource value; leave blank to keep pending",
304
- }));
305
- if (value)
306
- author("config_set", { path: field.path, value });
307
- }
308
- });
309
- const environments = app.command("environments");
310
- environments.command("list").action(() => run("environments"));
311
- environments
312
- .command("add <name>")
313
- .description("Copy an existing environment for review and configuration")
314
- .requiredOption("--from <environment>", "Existing environment to copy")
315
- .action((name, command) => author("environment_add", { name, from: command.from }));
316
- for (const kind of ["activities", "standards", "providers"]) {
317
- const group = app
318
- .command(kind)
319
- .description(`Inspect current ${kind} and their supported capabilities`);
320
- group.command("list").action(() => run("catalogue", { kind }));
321
- group.command("explain <id>").action((id) => run("catalogue", { kind, id }));
322
- }
323
- const connections = app
324
- .command("connections")
325
- .description("Validate and prepare project-defined connector flows without source access");
326
- connections
327
- .command("add <id>")
328
- .description("Register an installed connector as a project connection")
329
- .requiredOption("--package <alias>", "Configured connector package alias")
330
- .requiredOption("--source-id <id>", "Non-secret source identity")
331
- .requiredOption("--tenant-id <id>", "Non-secret tenant identity")
332
- .option("--binding <id>", "Environment binding for runtime secrets")
333
- .option("--settings <file>", "Project-relative YAML/JSON settings file")
334
- .action((id, options) => author("connection_add", {
335
- id,
336
- package: options.package,
337
- sourceId: options.sourceId,
338
- tenantId: options.tenantId,
339
- ...(options.binding ? { binding: options.binding } : {}),
340
- settings: options.settings ? readData(options.settings) : {},
341
- }));
342
- connections
343
- .command("flow-add <id>")
344
- .description("Create a connection-backed ingestion flow")
345
- .requiredOption("--provider <configuration>", "Configured provider")
346
- .requiredOption("--connection <id>", "Configured connection")
347
- .requiredOption("--table <id>", "Output table")
348
- .requiredOption("--contract <file>", "Project-relative ODCS contract")
349
- .option("--source <file>", "Project-relative YAML/JSON table source settings")
350
- .option("--source-path <path>", "Files connector source path")
351
- .option("--format <format>", "Files connector format")
352
- .option("--execution <file>", "Project-relative provider execution settings")
353
- .action((id, options) => {
354
- check(!!options.source !== !!options.sourcePath, "OPTION", "Choose a source settings file or a source path");
355
- check(!options.sourcePath || options.format, "OPTION", "A file source needs its format");
356
- author("connection_flow_add", {
357
- id,
358
- provider: options.provider,
359
- connection: options.connection,
360
- table: options.table,
361
- contract: options.contract,
362
- source: options.source
363
- ? readData(options.source)
364
- : { path: options.sourcePath, format: options.format },
365
- ...(options.execution ? { execution: readData(options.execution) } : {}),
366
- });
367
- });
368
- connections
369
- .command("validate <flow>")
370
- .action((flow) => run("connection_prepare", { flow, validateOnly: true }));
371
- connections
372
- .command("prepare <flow>")
373
- .requiredOption("--out <file>", "New provider result JSON file")
374
- .action((flow, command) => {
375
- const result = execute(context(), "connection_prepare", { flow });
376
- if (result.ok) {
377
- const file = fence(context().root, command.out);
378
- mkdirSync(dirname(file), { recursive: true });
379
- writeFileSync(file, JSON.stringify(result.result, null, 2) + "\n", {
380
- flag: "wx",
381
- });
382
- }
383
- print(result);
384
- });
385
- connections
386
- .command("contracts <flow>")
387
- .requiredOption("--review <file>", "Approved generated review")
388
- .requiredOption("--out <file>", "New provider result file")
389
- .action((flow, command) => {
390
- const result = execute(context(), "connection_prepare", {
391
- flow,
392
- review: command.review,
393
- });
394
- if (result.ok) {
395
- const file = fence(context().root, command.out);
396
- mkdirSync(dirname(file), { recursive: true });
397
- writeFileSync(file, JSON.stringify(result.result, null, 2) + "\n", {
398
- flag: "wx",
399
- });
400
- }
401
- print(result);
402
- });
403
- const providerCommands = app.commands.find((command) => command.name() === "providers");
404
- providerCommands.configureHelp({
405
- visibleCommands: (command) => command.commands.filter((child) => !["install", "update"].includes(child.name())),
406
- });
407
- for (const update of [false, true])
408
- providerCommands
409
- .command(`${update ? "update" : "install"} <reference>`)
410
- .description("Install a provider version into the project cache; select it in project.yaml")
411
- .option("--from-git <directory>", "Local Git repository")
412
- .option("--frozen", "Require an existing immutable lock")
413
- .action((reference, command) => run("packages_install", {
414
- reference,
415
- update,
416
- frozen: !!command.frozen,
417
- ...(command.fromGit ? { fromGit: command.fromGit } : {}),
418
- }));
419
- providerCommands
420
- .command("commands <configuration>")
421
- .description("List provider-owned commands and input schemas without executing plugin code")
422
- .action((configuration) => run("provider_commands", { configuration }));
423
- const provider = app
424
- .command("provider")
425
- .description("Install an execution provider or run its offline commands")
426
- .arguments("<configuration> <command...>")
427
- .option("--input <file>", "Project-relative JSON input file")
428
- .option("--out <file>", "Save the provider result JSON to a new project file")
429
- .action((configuration, words, command) => providerAction("provider_command", { configuration, command: words.join(" ") }, command));
430
- provider
431
- .command("exec <configuration> <command...>")
432
- .description("Run a configured provider command; inspect providers commands first")
433
- .option("--input <file>", "Project-relative JSON input file")
434
- .option("--out <file>", "Save the provider result JSON to a new project file")
435
- .action((configuration, words, command) => providerAction("provider_command", { configuration, command: words.join(" ") }, command));
436
- function providerAction(operation, args, command) {
437
- const input = command.input
438
- ? JSON.parse(readFileSync(new Configuration(context().root).path(command.input), "utf8"))
439
- : {};
440
- const result = execute(context(), operation, { ...args, input });
441
- if (result.ok && command.out) {
442
- const file = fence(context().root, command.out);
443
- mkdirSync(dirname(file), { recursive: true });
444
- writeFileSync(file, JSON.stringify(result.result, null, 2) + "\n", {
445
- flag: "wx",
446
- });
447
- if (!opts().json && !opts().verbose) {
448
- process.stdout.write(`Saved provider result to ${command.out}\n`);
449
- return;
450
- }
451
- }
452
- print(result);
453
- }
454
- providerCommands
455
- .command("export")
456
- .description("Review/export files returned by an offline provider; no script execution")
457
- .requiredOption("--result <file>", "Saved provider result JSON")
458
- .requiredOption("--out <directory>", "New artifact directory")
459
- .action((command) => author("provider_export", { result: command.result, out: command.out }));
460
- providerCommands
461
- .command("migrate")
462
- .requiredOption("--from <package>", "Project package key")
463
- .requiredOption("--to <reference>", "Installed exact provider reference")
464
- .action((command) => author("provider_migrate", { package: command.from, to: command.to }));
465
- const packages = app.command("packages");
466
- packages.command("list").action(() => run("packages_list"));
467
- for (const update of [false, true])
468
- packages
469
- .command(`${update ? "update" : "install"} <reference>`)
470
- .option("--from-git <directory>", "Explicit local Git source for offline installation")
471
- .option("--frozen", "Require existing immutable lock entry")
472
- .action((reference, command) => run("packages_install", {
473
- reference,
474
- update,
475
- ...(command.fromGit ? { fromGit: command.fromGit } : {}),
476
- frozen: !!command.frozen,
477
- }));
478
- app
479
- .command("validate")
480
- .option("--mode <mode>", "draft or strict", "strict")
481
- .option("--flow <id>")
482
- .option("--profile <name>", "Execution profile within the selected flow")
483
- .option("--table <id>")
484
- .option("--step <id>")
485
- .action((command) => run("validate", {
486
- mode: command.mode,
487
- ...(command.flow ? { flow: command.flow } : {}),
488
- ...(command.profile ? { profile: command.profile } : {}),
489
- ...(command.table ? { table: command.table } : {}),
490
- ...(command.step ? { step: command.step } : {}),
491
- }));
492
- app
493
- .command("doctor")
494
- .description("Check local configuration and missing inputs; no platform connection")
495
- .action(() => run("doctor"));
496
- app
497
- .command("resolve")
498
- .option("--flow <id>")
499
- .option("--profile <name>")
500
- .action((command) => run("resolve", command));
501
- app
502
- .command("plan")
503
- .option("--flow <id>")
504
- .option("--profile <name>", "Execution profile within the selected flow")
505
- .option("--table <id>")
506
- .option("--step <id>")
507
- .option("--out <file>", "Save a digest-bound JSON plan")
508
- .action((command) => {
509
- const result = execute(context(), "plan", {
510
- ...(command.flow ? { flow: command.flow } : {}),
511
- ...(command.profile ? { profile: command.profile } : {}),
512
- ...(command.table ? { table: command.table } : {}),
513
- ...(command.step ? { step: command.step } : {}),
514
- });
515
- if (result.ok && command.out)
516
- save(command.out, result.result);
517
- print(result);
518
- });
519
- app
520
- .command("diff")
521
- .description("Compare local plans; no remote deployment drift lookup")
522
- .requiredOption("--plan <file>", "Previously reviewed plan")
523
- .action((command) => run("diff", { plan: command.plan }));
524
- app
525
- .command("generate")
526
- .requiredOption("--plan <file>", "Saved plan JSON")
527
- .requiredOption("--out <directory>", "Owned output directory inside project")
528
- .option("--ownership <mode>", "managed (regenerate safely) or team (manual handover)", "managed")
529
- .action((command) => run("generate", {
530
- plan: command.plan,
531
- out: command.out,
532
- ownership: command.ownership,
533
- }));
534
- app
535
- .command("validate-output <directory>")
536
- .description("Check local output ownership and syntax; no target validation")
537
- .action((directory) => run("validate_output", { directory }));
538
- app.command("schema <name>").action((name) => run("schema", { name }));
539
- const contract = app.command("contract");
540
- const activity = app.command("activity");
541
- activity
542
- .command("validate <file>")
543
- .description("Validate an activity manifest and inventory source files; no rendering or execution")
544
- .action((file) => run("activity_inspect", { path: file }));
545
- contract
546
- .command("validate <file>")
547
- .action((file) => run("contract_validate", { path: file }));
548
- app
549
- .command("mcp")
550
- .command("serve")
551
- .option("--allow-write", "Allow explicit reviewed local changes")
552
- .option("--allow-execute", "Allow explicit customer-side source execution; requires --allow-write")
553
- .option("--allow-network", "Allow GitHub package version lookup and fetching; no platform access")
554
- .action(async (command) => {
555
- const { serve } = await import("../mcp/index.js");
556
- await serve(resolve(opts().project), !!command.allowWrite, context().environment, !!command.allowNetwork, !!command.allowExecute);
557
- });
558
- workflowCommands(app, {
559
- context,
560
- run,
561
- author,
562
- print,
563
- readData,
564
- canPrompt,
565
- answer,
566
- perform: (operation, args, label) => backgroundOperation(context(), operation, args, label, presentation()),
567
- });
568
- app.hook("preAction", (_root, command) => {
569
- if (!opts().dryRun)
570
- return;
571
- const parts = [];
572
- for (let current = command; current && current !== app; current = current.parent)
573
- parts.unshift(current.name());
574
- check(new Set([
575
- "init",
576
- "source add",
577
- "source configure",
578
- "source prepare",
579
- "source import",
580
- "plugin configure",
581
- "connections add",
582
- "connections flow-add",
583
- "plugin migrate",
584
- "config environment add",
585
- "flow table add",
586
- "flow table configure",
587
- "flow step add",
588
- "flow output add",
589
- "advanced export",
590
- "flow add",
591
- "flow configure",
592
- "step add",
593
- "table add",
594
- "table configure",
595
- "dataset add",
596
- "config set",
597
- "config fill",
598
- "environments add",
599
- "providers migrate",
600
- "providers export",
601
- "contract draft",
602
- "contract scaffold",
603
- "contract map",
604
- "plugin scaffold",
605
- "plugin pack-add",
606
- ]).has(parts.join(" ")), "OPTION", "--dry-run is supported only by project authoring commands; this command has not run");
607
- });
608
- async function parseCommands() {
609
- const parsed = app.parseOptions(process.argv.slice(2));
610
- const requested = [...parsed.operands, ...parsed.unknown].find((arg) => !arg.startsWith("-"));
611
- const known = new Set(app.commands.flatMap((command) => [command.name(), ...command.aliases()]));
612
- if (existsSync(resolve(context().root, "project.yaml"))) {
613
- const catalogue = execute(context(), "plugin_commands", {});
614
- if (!catalogue.ok) {
615
- if (requested && !known.has(requested)) {
616
- print(catalogue);
617
- return;
618
- }
619
- }
620
- else {
621
- for (const binding of catalogue.result.bindings) {
622
- check(!known.has(binding.namespace), "COMMAND_NAMESPACE", `Plugin namespace conflicts with core command ${binding.namespace}`);
623
- let namespace = app.commands.find((command) => command.name() === binding.namespace);
624
- if (namespace)
625
- continue; // Same locked package, multiple selectable configurations.
626
- namespace = app
627
- .command(binding.namespace)
628
- .description(`Installed ${binding.provider} offline operations`);
629
- for (const definition of binding.commands) {
630
- const words = definition.name.split(" ");
631
- let parent = namespace;
632
- for (const word of words.slice(0, -1)) {
633
- let group = parent.commands.find((command) => command.name() === word);
634
- if (!group)
635
- group = parent.command(word).description(`${word} operations`);
636
- parent = group;
637
- }
638
- parent
639
- .command(words.at(-1))
640
- .description(definition.description)
641
- .option("--target <configuration>", "Provider configuration; required when multiple bindings match")
642
- .option("--input <file>", "Project-relative JSON input file")
643
- .option("--out <file>", "Save result JSON to a new project file")
644
- .addHelpText("after", `\nInput schema:\n${JSON.stringify(definition.inputSchema, null, 2)}\n${(definition.examples ?? []).join("\n")}`)
645
- .action((options) => providerAction("plugin_command", {
646
- namespace: binding.namespace,
647
- command: definition.name,
648
- ...(options.target ? { target: options.target } : {}),
649
- }, options));
650
- }
651
- }
652
- }
653
- }
654
- await app.parseAsync();
655
- }
656
- await parseCommands().catch((error) => {
657
- if (error.code === "commander.helpDisplayed" ||
658
- error.code === "commander.version" ||
659
- error.code === "commander.help") {
660
- process.exitCode = 0;
661
- if (process.argv.includes("--json"))
662
- print({
663
- apiVersion: "ingestron.operation/v1",
664
- operation: error.code === "commander.version" ? "version" : "help",
665
- ok: true,
666
- result: { text: helpText },
667
- diagnostics: [],
668
- });
669
- return;
670
- }
671
- print({
672
- apiVersion: "ingestron.operation/v1",
673
- operation: "command",
674
- ok: false,
675
- diagnostics: [
676
- {
677
- code: error instanceof Problem ? error.code : "COMMAND",
678
- message: error instanceof Error
679
- ? error.message.replace(/^error: /, "")
680
- : "Command failed",
681
- ...(error instanceof Problem && error.hint ? { hint: error.hint } : {}),
682
- },
683
- ],
684
- });
685
- });