ingestron 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/main.js +770 -376
- package/package.json +3 -2
- package/schemas/flow.schema.yaml +16 -0
- package/schemas/standard.schema.yaml +5 -0
- package/schemas/transform.schema.yaml +15 -0
package/package.json
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ingestron",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "Declare data ingestion in reviewed YAML definitions and generate Databricks
|
|
3
|
+
"version": "1.2.0",
|
|
4
|
+
"description": "Declare data ingestion in reviewed YAML definitions and generate Databricks, Azure Data Factory and Synapse serverless SQL files under your standards.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ingestion",
|
|
7
7
|
"databricks",
|
|
8
8
|
"azure-data-factory",
|
|
9
|
+
"azure-synapse",
|
|
9
10
|
"data-engineering",
|
|
10
11
|
"yaml",
|
|
11
12
|
"cli"
|
package/schemas/flow.schema.yaml
CHANGED
|
@@ -51,12 +51,15 @@ properties:
|
|
|
51
51
|
{ $ref: "#/$defs/expectations", description: Your own checks on this table's rows. }
|
|
52
52
|
ingest: { $ref: "#/$defs/with", description: Method settings for this table only. }
|
|
53
53
|
store: { $ref: "#/$defs/with", description: Method settings for this table only. }
|
|
54
|
+
serve: { $ref: "#/$defs/with", description: Method settings for this table only. }
|
|
54
55
|
examples:
|
|
55
56
|
- orders:
|
|
56
57
|
object: orders
|
|
57
58
|
contract: shop.orders
|
|
58
59
|
ingest: { $ref: "#/$defs/ingest", description: How data leaves the source. }
|
|
59
60
|
store: { $ref: "#/$defs/store", description: How data is kept in the platform. }
|
|
61
|
+
serve:
|
|
62
|
+
{ $ref: "#/$defs/serve", description: Where the data is made available after its last step. }
|
|
60
63
|
$defs:
|
|
61
64
|
with:
|
|
62
65
|
type: object
|
|
@@ -123,3 +126,16 @@ $defs:
|
|
|
123
126
|
with:
|
|
124
127
|
type: object
|
|
125
128
|
description: Method settings for every table of this flow.
|
|
129
|
+
serve:
|
|
130
|
+
type: object
|
|
131
|
+
additionalProperties: false
|
|
132
|
+
properties:
|
|
133
|
+
uses:
|
|
134
|
+
type: string
|
|
135
|
+
pattern: ^[a-z][a-z0-9-]*$
|
|
136
|
+
description: A serve method the standard allows, such as views over the data. Leave out to let the standard choose; nothing is served unless something chooses.
|
|
137
|
+
examples:
|
|
138
|
+
- synapse-serve
|
|
139
|
+
with:
|
|
140
|
+
type: object
|
|
141
|
+
description: Method settings for every table of this flow.
|
|
@@ -82,6 +82,11 @@ properties:
|
|
|
82
82
|
$ref: "#/$defs/rules",
|
|
83
83
|
description: "How tables are stored: method settings, rules, and allowed and fixed settings.",
|
|
84
84
|
}
|
|
85
|
+
serve:
|
|
86
|
+
{
|
|
87
|
+
$ref: "#/$defs/rules",
|
|
88
|
+
description: "Where tables are made available after their last step: method settings, rules, and allowed and fixed settings. Nothing is served unless something chooses.",
|
|
89
|
+
}
|
|
85
90
|
$defs:
|
|
86
91
|
method: { type: string, pattern: "^[a-z][a-z0-9-]*$" }
|
|
87
92
|
naming:
|
|
@@ -79,6 +79,8 @@ properties:
|
|
|
79
79
|
description: Your own checks on the table's rows.
|
|
80
80
|
transform: { $ref: "#/$defs/transform", description: How the code runs. }
|
|
81
81
|
store: { $ref: "#/$defs/store", description: How the result is kept. }
|
|
82
|
+
serve:
|
|
83
|
+
{ $ref: "#/$defs/serve", description: Where the result is made available after it is kept. }
|
|
82
84
|
$defs:
|
|
83
85
|
transform:
|
|
84
86
|
type: object
|
|
@@ -106,3 +108,16 @@ $defs:
|
|
|
106
108
|
with:
|
|
107
109
|
type: object
|
|
108
110
|
description: Settings for the method, over the standard's.
|
|
111
|
+
serve:
|
|
112
|
+
type: object
|
|
113
|
+
additionalProperties: false
|
|
114
|
+
properties:
|
|
115
|
+
uses:
|
|
116
|
+
type: string
|
|
117
|
+
pattern: ^[a-z][a-z0-9-]*$
|
|
118
|
+
description: A serve method the standard allows, such as views over the data. Leave out to let the standard choose; nothing is served unless something chooses.
|
|
119
|
+
examples:
|
|
120
|
+
- synapse-serve
|
|
121
|
+
with:
|
|
122
|
+
type: object
|
|
123
|
+
description: Settings for the method, over the standard's.
|