@dataform/core 3.0.0 → 3.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/configs.proto CHANGED
@@ -43,7 +43,6 @@ message WorkflowSettings {
43
43
 
44
44
  // Optional. User-defined variables that are made available to project code
45
45
  // during compilation. An object containing a list of "key": value pairs.
46
- // Example: `{ "name": "wrench", "mass": "1.3kg", "count": "3" }`.
47
46
  map<string, string> vars = 6;
48
47
 
49
48
  // Optional. The suffix to append to all Google Cloud project references.
@@ -93,6 +92,37 @@ message ActionConfig {
93
92
 
94
93
  // A list of BigQuery policy tags that will be applied to the column.
95
94
  repeated string bigquery_policy_tags = 3;
95
+
96
+ // A list of tags for this column which will be applied.
97
+ repeated string tags = 4;
98
+ }
99
+
100
+ // Options for shorthand specifying assertions, useable for some table-based
101
+ // action types.
102
+ message TableAssertionsConfig {
103
+ // Column(s) which constitute the dataset's unique key index.
104
+ // If set, the resulting assertion will fail if there is more than one row
105
+ // in the dataset with the same values for all of these column(s).
106
+ repeated string unique_key = 1;
107
+
108
+ // Combinations of column(s), each of which should constitute a unique key
109
+ // index for the dataset. If set, the resulting assertion(s) will fail if
110
+ // there is more than one row in the dataset with the same values for all of
111
+ // the column(s) in the unique key(s).
112
+ message UniqueKey {
113
+ repeated string unique_key = 1;
114
+ }
115
+ repeated UniqueKey unique_keys = 2;
116
+
117
+ // Column(s) which may never be `NULL`.
118
+ // If set, the resulting assertion will fail if any row contains `NULL`
119
+ // values for these column(s).
120
+ repeated string non_null = 3;
121
+
122
+ // General condition(s) which should hold true for all rows in the dataset.
123
+ // If set, the resulting assertion will fail if any row violates any of
124
+ // these condition(s).
125
+ repeated string row_conditions = 4;
96
126
  }
97
127
 
98
128
  message TableConfig {
@@ -150,27 +180,28 @@ message ActionConfig {
150
180
  // https://cloud.google.com/dataform/docs/partitions-clusters.
151
181
  repeated string cluster_by = 15;
152
182
 
153
- // Key-value pairs for BigQuery labels. If the label name contains special
154
- // characters, e.g. hyphens, then quote its name, e.g. `labels: {
155
- // "label-name": "value" }`.
183
+ // Key-value pairs for BigQuery labels.
156
184
  map<string, string> labels = 16;
157
185
 
158
- // Key-value pairs of additional options to pass to the BigQuery API.
159
- //
160
- // Some options, for example, partitionExpirationDays, have dedicated
161
- // type/validity checked fields. For such options, use the dedicated
162
- // fields.
163
- //
164
- // String values must be encapsulated in double-quotes, for example:
165
- // additionalOptions: {numeric_option: "5", string_option: '"string-value"'}
166
- //
167
- // If the option name contains special characters, encapsulate the name in
168
- // quotes, for example: additionalOptions: { "option-name": "value" }.
186
+ // Key-value pairs of additional options to pass to the BigQuery API. Some
187
+ // options, for example, partitionExpirationDays, have dedicated
188
+ // type/validity checked fields. For such options, use the dedicated fields.
169
189
  map<string, string> additional_options = 17;
170
190
 
171
191
  // When set to true, assertions dependent upon any dependency will
172
192
  // be add as dedpendency to this action
173
193
  bool depend_on_dependency_assertions = 18;
194
+
195
+ // Assertions to be run on the dataset.
196
+ // If configured, relevant assertions will automatically be created and run
197
+ // as a dependency of this dataset.
198
+ TableAssertionsConfig assertions = 19;
199
+
200
+ // If true, this indicates that the action only depends on data from
201
+ // explicitly-declared dependencies. Otherwise if false, it indicates that
202
+ // the action depends on data from a source which has not been declared as
203
+ // a dependency.
204
+ bool hermetic = 20;
174
205
  }
175
206
 
176
207
  message ViewConfig {
@@ -214,27 +245,28 @@ message ActionConfig {
214
245
  // Descriptions of columns within the table.
215
246
  repeated ColumnDescriptor columns = 12;
216
247
 
217
- // Key-value pairs for BigQuery labels. If the label name contains special
218
- // characters, e.g. hyphens, then quote its name, e.g. `labels: {
219
- // "label-name": "value" }`.
248
+ // Key-value pairs for BigQuery labels.
220
249
  map<string, string> labels = 13;
221
250
 
222
- // Key-value pairs of additional options to pass to the BigQuery API.
223
- //
224
- // Some options, for example, partitionExpirationDays, have dedicated
225
- // type/validity checked fields. For such options, use the dedicated
226
- // fields.
227
- //
228
- // String values must be encapsulated in double-quotes, for example:
229
- // additionalOptions: {numeric_option: "5", string_option: '"string-value"'}
230
- //
231
- // If the option name contains special characters, encapsulate the name in
232
- // quotes, for example: additionalOptions: { "option-name": "value" }.
251
+ // Key-value pairs of additional options to pass to the BigQuery API. Some
252
+ // options, for example, partitionExpirationDays, have dedicated
253
+ // type/validity checked fields. For such options, use the dedicated fields.
233
254
  map<string, string> additional_options = 14;
234
255
 
235
256
  // When set to true, assertions dependent upon any dependency will
236
257
  // be add as dedpendency to this action
237
258
  bool depend_on_dependency_assertions = 15;
259
+
260
+ // If true, this indicates that the action only depends on data from
261
+ // explicitly-declared dependencies. Otherwise if false, it indicates that
262
+ // the action depends on data from a source which has not been declared as
263
+ // a dependency.
264
+ bool hermetic = 16;
265
+
266
+ // Assertions to be run on the dataset.
267
+ // If configured, relevant assertions will automatically be created and run
268
+ // as a dependency of this dataset.
269
+ TableAssertionsConfig assertions = 17;
238
270
  }
239
271
 
240
272
  message IncrementalTableConfig {
@@ -306,27 +338,28 @@ message ActionConfig {
306
338
  // https://cloud.google.com/dataform/docs/partitions-clusters.
307
339
  repeated string cluster_by = 18;
308
340
 
309
- // Key-value pairs for BigQuery labels. If the label name contains special
310
- // characters, e.g. hyphens, then quote its name, e.g. `labels: {
311
- // "label-name": "value" }`.
341
+ // Key-value pairs for BigQuery labels.
312
342
  map<string, string> labels = 19;
313
343
 
314
- // Key-value pairs of additional options to pass to the BigQuery API.
315
- //
316
- // Some options, for example, partitionExpirationDays, have dedicated
317
- // type/validity checked fields. For such options, use the dedicated
318
- // fields.
319
- //
320
- // String values must be encapsulated in double-quotes, for example:
321
- // additionalOptions: {numeric_option: "5", string_option: '"string-value"'}
322
- //
323
- // If the option name contains special characters, encapsulate the name in
324
- // quotes, for example: additionalOptions: { "option-name": "value" }.
344
+ // Key-value pairs of additional options to pass to the BigQuery API. Some
345
+ // options, for example, partitionExpirationDays, have dedicated
346
+ // type/validity checked fields. For such options, use the dedicated fields.
325
347
  map<string, string> additional_options = 20;
326
348
 
327
349
  // When set to true, assertions dependent upon any dependency will
328
350
  // be add as dedpendency to this action
329
351
  bool depend_on_dependency_assertions = 21;
352
+
353
+ // Assertions to be run on the dataset.
354
+ // If configured, relevant assertions will automatically be created and run
355
+ // as a dependency of this dataset.
356
+ TableAssertionsConfig assertions = 22;
357
+
358
+ // If true, this indicates that the action only depends on data from
359
+ // explicitly-declared dependencies. Otherwise if false, it indicates that
360
+ // the action depends on data from a source which has not been declared as
361
+ // a dependency.
362
+ bool hermetic = 23;
330
363
  }
331
364
 
332
365
  message AssertionConfig {
@@ -355,6 +388,16 @@ message ActionConfig {
355
388
 
356
389
  // Description of the assertion.
357
390
  string description = 8;
391
+
392
+ // If true, this indicates that the action only depends on data from
393
+ // explicitly-declared dependencies. Otherwise if false, it indicates that
394
+ // the action depends on data from a source which has not been declared as
395
+ // a dependency.
396
+ bool hermetic = 9;
397
+
398
+ // If true, assertions dependent upon any of the dependencies are added as
399
+ // dependencies as well.
400
+ bool depend_on_dependency_assertions = 10;
358
401
  }
359
402
 
360
403
  message OperationConfig {
@@ -396,6 +439,12 @@ message ActionConfig {
396
439
  // When set to true, assertions dependent upon any dependency will
397
440
  // be add as dedpendency to this action
398
441
  bool depend_on_dependency_assertions = 11;
442
+
443
+ // If true, this indicates that the action only depends on data from
444
+ // explicitly-declared dependencies. Otherwise if false, it indicates that
445
+ // the action depends on data from a source which has not been declared as
446
+ // a dependency.
447
+ bool hermetic = 12;
399
448
  }
400
449
 
401
450
  message DeclarationConfig {
@@ -449,6 +498,28 @@ message ActionConfig {
449
498
  // TODO(ekrekr): add a notebook runtime field definition.
450
499
  }
451
500
 
501
+ message DataPreparationConfig {
502
+ // The name of the data preparation.
503
+ string name = 1;
504
+
505
+ // Targets of actions that this action is dependent on.
506
+ repeated Target dependency_targets = 2;
507
+
508
+ // Path to the source file that the contents of the action is loaded from.
509
+ string filename = 3;
510
+
511
+ // A list of user-defined tags with which the action should be labeled.
512
+ repeated string tags = 4;
513
+
514
+ // If set to true, this action will not be executed. However, the action can
515
+ // still be depended upon. Useful for temporarily turning off broken
516
+ // actions.
517
+ bool disabled = 7;
518
+
519
+ // Description of the data preparation.
520
+ string description = 8;
521
+ }
522
+
452
523
  oneof action {
453
524
  TableConfig table = 1;
454
525
  ViewConfig view = 2;
@@ -457,6 +528,7 @@ message ActionConfig {
457
528
  OperationConfig operation = 5;
458
529
  DeclarationConfig declaration = 6;
459
530
  NotebookConfig notebook = 7;
531
+ DataPreparationConfig data_preparation = 8;
460
532
  }
461
533
  }
462
534
 
package/core.proto CHANGED
@@ -13,6 +13,7 @@
13
13
  // limitations under the License.
14
14
 
15
15
  syntax = "proto3";
16
+ import "data_preparation.proto";
16
17
 
17
18
  package dataform;
18
19
 
@@ -261,6 +262,29 @@ message NotebookRuntimeOptions {
261
262
  }
262
263
  }
263
264
 
265
+ // Data Preparation Related entries
266
+ message DataPreparation {
267
+ // Used ONLY as an identifiers. All outputs should be listed in the targets
268
+ // section.
269
+ Target target = 8;
270
+ Target canonical_target = 9;
271
+
272
+ // Data preparations can have more than 1 output
273
+ repeated Target targets = 1;
274
+
275
+ repeated Target canonical_targets = 2;
276
+
277
+ repeated string tags = 3;
278
+
279
+ repeated Target dependency_targets = 4;
280
+
281
+ string file_name = 5;
282
+
283
+ bool disabled = 6;
284
+
285
+ DataPreparationDefinition data_preparation = 7;
286
+ }
287
+
264
288
  message CompiledGraph {
265
289
  ProjectConfig project_config = 4;
266
290
 
@@ -272,6 +296,7 @@ message CompiledGraph {
272
296
  // it is not used at runtime.
273
297
  repeated Test tests = 8;
274
298
  repeated Notebook notebooks = 12;
299
+ repeated DataPreparation data_preparations = 13;
275
300
 
276
301
  GraphErrors graph_errors = 7;
277
302
 
@@ -0,0 +1,188 @@
1
+ // Copyright 2023 Google LLC
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+
15
+ syntax = "proto3";
16
+
17
+ package dataform;
18
+
19
+ option java_package = "com.dataform.protos";
20
+ option java_multiple_files = true;
21
+
22
+ option go_package = "github.com/dataform-co/dataform/protos/dataform";
23
+
24
+ message DataPreparationDefinition {
25
+ repeated DataPreparationNode nodes = 1;
26
+ DataPreparationGenerated generated = 2;
27
+ }
28
+
29
+ message DataPreparationNode {
30
+ string id = 1;
31
+ DataPreparationNodeSource source = 2;
32
+ repeated DataPreparationNodeStep steps = 3;
33
+ DataPreparationNodeGenerated generated = 4;
34
+ // Destination BigQuery table(s) are defined within the data preparation
35
+ optional DataPreparationNodeDestination destination = 5;
36
+ }
37
+
38
+ message DataPreparationNodeSource {
39
+ oneof source {
40
+ string node_id = 1;
41
+ TableReference table = 2;
42
+ Join join = 3;
43
+ }
44
+ }
45
+
46
+ message DataPreparationNodeDestination {
47
+ oneof destination {
48
+ TableReference table = 1;
49
+ }
50
+ }
51
+
52
+ message DataPreparationNodeStep {
53
+ string id = 1;
54
+ string description = 2;
55
+ oneof definition {
56
+ ColumnStep column_step = 3;
57
+ FilterStep filter_step = 4;
58
+ }
59
+ DataPreparationNodeStepGenerated generated = 5;
60
+ }
61
+
62
+ message ColumnStep {
63
+ string column_name = 1;
64
+ Expression expression = 2;
65
+ }
66
+
67
+ message FilterStep {
68
+ Expression expression = 1;
69
+ }
70
+
71
+ message Expression {
72
+ oneof expression {
73
+ string sql = 1;
74
+ }
75
+ }
76
+
77
+ message Join {
78
+ string left_node_id = 1;
79
+ string right_node_id = 2;
80
+ JoinType join_type = 3;
81
+ JoinCondition join_condition = 4;
82
+ }
83
+
84
+ enum JoinType {
85
+ JOIN_TYPE_UNSPECIFIED = 0;
86
+ JOIN_TYPE_INNER = 1;
87
+ JOIN_TYPE_FULL_OUTER = 2;
88
+ JOIN_TYPE_LEFT = 3;
89
+ JOIN_TYPE_RIGHT = 4;
90
+ }
91
+
92
+ message JoinCondition {
93
+ oneof condition {
94
+ Expression expression = 1;
95
+ JoinKeys keys = 2;
96
+ }
97
+ }
98
+
99
+ message JoinKeys {
100
+ repeated JoinKey keys = 1;
101
+ }
102
+
103
+ message JoinKey {
104
+ string left_column = 1;
105
+ string right_column = 2;
106
+ }
107
+
108
+ message TableReference {
109
+ string project = 1;
110
+ string dataset = 2;
111
+ string table = 3;
112
+ }
113
+
114
+ message DataPreparationGenerated {
115
+ repeated DataPreparationValidationError validation_errors = 1;
116
+ optional string location = 2;
117
+ }
118
+
119
+ message DataPreparationNodeGenerated {
120
+ repeated DataPreparationSection sections = 1;
121
+ repeated string sources = 2;
122
+ repeated DataPreparationValidationError validation_errors = 3;
123
+ optional DataPreparationSchema output_schema = 4;
124
+ DataPreparationNodeSourceGenerated source_generated = 5;
125
+ optional DataPreparationNodeDestinationGenerated destination_generated = 6;
126
+ }
127
+
128
+ message DataPreparationSection {
129
+ DataPreparationSectionType type = 1;
130
+ string label = 2;
131
+ }
132
+
133
+ enum DataPreparationSectionType {
134
+ SECTION_TYPE_UNSPECIFIED = 0;
135
+ SECTION_UNPARSEABLE = 1;
136
+ SECTION_SOURCE_TABLE = 2;
137
+ SECTION_SQL = 3;
138
+ SECTION_DESTINATION_TABLE = 4;
139
+ }
140
+
141
+ message DataPreparationNodeSourceGenerated {
142
+ optional DataPreparationNodeSourceSourceSchema source_schema = 4;
143
+ }
144
+
145
+ message DataPreparationNodeSourceSourceSchema {
146
+ oneof source_schema {
147
+ DataPreparationSchema node_schema = 1;
148
+ DataPreparationSchema table_schema = 2;
149
+ JoinSchema join_schema = 3;
150
+ }
151
+ }
152
+
153
+ message JoinSchema {
154
+ DataPreparationSchema left_schema = 1;
155
+ DataPreparationSchema right_schema = 2;
156
+ }
157
+
158
+ message DataPreparationNodeDestinationGenerated {
159
+ optional DataPreparationSchema schema = 1;
160
+ }
161
+
162
+ message DataPreparationNodeStepGenerated {
163
+ repeated string source_columns = 1;
164
+ repeated DataPreparationValidationError validation_errors = 2;
165
+ }
166
+
167
+ message DataPreparationSchema {
168
+ repeated DataPreparationSchemaField field = 1;
169
+ }
170
+
171
+ message DataPreparationSchemaField {
172
+ string name = 1;
173
+ optional string type = 2;
174
+ optional string mode = 3;
175
+ repeated DataPreparationSchemaField fields = 4;
176
+ }
177
+
178
+ message DataPreparationValidationError {
179
+ DataPreparationValidationErrorLevel level = 1;
180
+ string description = 2;
181
+ }
182
+
183
+ enum DataPreparationValidationErrorLevel {
184
+ LEVEL_UNSPECIFIED = 0;
185
+ LEVEL_WARN = 1;
186
+ LEVEL_ERROR = 2;
187
+ LEVEL_FATAL = 3;
188
+ }
package/package.json CHANGED
@@ -32,7 +32,7 @@
32
32
  "publishConfig": {
33
33
  "access": "public"
34
34
  },
35
- "version": "3.0.0",
35
+ "version": "3.0.2",
36
36
  "name": "@dataform/core",
37
37
  "description": "Dataform core API.",
38
38
  "main": "bundle.js"