@dataform/core 3.0.0-alpha.2 → 3.0.0-alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/configs.proto ADDED
@@ -0,0 +1,429 @@
1
+ // Copyright 2023 Google LLC
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+
15
+ syntax = "proto3";
16
+
17
+ package dataform;
18
+
19
+ option java_package = "com.dataform.protos";
20
+ option java_outer_classname = "ConfigsMeta";
21
+ option java_multiple_files = true;
22
+
23
+ option go_package = "github.com/dataform-co/dataform/protos/dataform";
24
+
25
+ // Workflow Settings defines the contents of the `workflow_settings.yaml`
26
+ // configuration file.
27
+ message WorkflowSettings {
28
+ // The desired dataform core version to compile against.
29
+ string dataform_core_version = 1;
30
+
31
+ // Required. The default Google Cloud project (database).
32
+ string default_project = 2;
33
+
34
+ // Required. The default dataset (schema).
35
+ string default_dataset = 3;
36
+
37
+ // Required. The default BigQuery location to use. For more information on
38
+ // BigQuery locations, see https://cloud.google.com/bigquery/docs/locations.
39
+ string default_location = 4;
40
+
41
+ // Required. The default dataset (schema) for assertions.
42
+ string default_assertion_dataset = 5;
43
+
44
+ // Optional. User-defined variables that are made available to project code
45
+ // during compilation. An object containing a list of "key": value pairs.
46
+ // Example: `{ "name": "wrench", "mass": "1.3kg", "count": "3" }`.
47
+ map<string, string> vars = 6;
48
+
49
+ // Optional. The suffix to append to all Google Cloud project references.
50
+ string project_suffix = 7;
51
+
52
+ // Optional. The suffix to append to all dataset references.
53
+ string dataset_suffix = 8;
54
+
55
+ // Optional. The prefix to append to all action names.
56
+ string name_prefix = 9;
57
+ }
58
+
59
+ // Action configs defines the contents of `actions.yaml` configuration files.
60
+ // TODO(ekrekr): consolidate these configuration options in the JS API.
61
+ message ActionConfigs {
62
+ repeated ActionConfig actions = 1;
63
+ }
64
+
65
+ // Action config defines the configuration properties of actions.
66
+ message ActionConfig {
67
+ // Target represents a unique action identifier.
68
+ message Target {
69
+ // The Google Cloud project (database) of the action.
70
+ string project = 1;
71
+
72
+ // The dataset (schema) of the action. For notebooks, this is the location.
73
+ string dataset = 2;
74
+
75
+ // The name of the action.
76
+ string name = 4;
77
+ }
78
+
79
+ message ColumnDescriptor {
80
+ // The identifier for the column, using multiple parts for nested records.
81
+ repeated string path = 1;
82
+
83
+ // A text description of the column.
84
+ string description = 2;
85
+
86
+ // A list of BigQuery policy tags that will be applied to the column.
87
+ repeated string bigquery_policy_tags = 3;
88
+ }
89
+
90
+ message TableConfig {
91
+ // The name of the table.
92
+ string name = 1;
93
+
94
+ // The dataset (schema) of the table.
95
+ string dataset = 2;
96
+
97
+ // The Google Cloud project (database) of the table.
98
+ string project = 3;
99
+
100
+ // Targets of actions that this action is dependent on.
101
+ repeated Target dependency_targets = 4;
102
+
103
+ // Path to the source file that the contents of the action is loaded from.
104
+ string filename = 5;
105
+
106
+ // A list of user-defined tags with which the action should be labeled.
107
+ repeated string tags = 6;
108
+
109
+ // If set to true, this action will not be executed. However, the action can
110
+ // still be depended upon. Useful for temporarily turning off broken
111
+ // actions.
112
+ bool disabled = 7;
113
+
114
+ // Queries to run before `query`. This can be useful for granting
115
+ // permissions.
116
+ repeated string pre_operations = 8;
117
+
118
+ // Queries to run after `query`.
119
+ repeated string post_operations = 9;
120
+
121
+ // Description of the table.
122
+ string description = 10;
123
+
124
+ // Descriptions of columns within the table.
125
+ repeated ColumnDescriptor columns = 11;
126
+
127
+ // The key by which to partition the table. Typically the name of a
128
+ // timestamp or the date column. See
129
+ // https://cloud.google.com/dataform/docs/partitions-clusters.
130
+ string partition_by = 12;
131
+
132
+ // The number of days for which BigQuery stores data in each partition.
133
+ // The setting applies to all partitions in a table, but is calculated
134
+ // independently for each partition based on the partition time.
135
+ int32 partition_expiration_days = 13;
136
+
137
+ // Declares whether the partitioned table requires a WHERE clause
138
+ // predicate filter that filters the partitioning column.
139
+ bool require_partition_filter = 14;
140
+
141
+ // The keys by which to cluster partitions by. See
142
+ // https://cloud.google.com/dataform/docs/partitions-clusters.
143
+ repeated string cluster_by = 15;
144
+
145
+ // Key-value pairs for BigQuery labels. If the label name contains special
146
+ // characters, e.g. hyphens, then quote its name, e.g. `labels: {
147
+ // "label-name": "value" }`.
148
+ map<string, string> labels = 16;
149
+
150
+ // Key-value pairs of additional options to pass to the BigQuery API.
151
+ //
152
+ // Some options, for example, partitionExpirationDays, have dedicated
153
+ // type/validity checked fields. For such options, use the dedicated
154
+ // fields.
155
+ //
156
+ // String values must be encapsulated in double-quotes, for example:
157
+ // additionalOptions: {numeric_option: "5", string_option: '"string-value"'}
158
+ //
159
+ // If the option name contains special characters, encapsulate the name in
160
+ // quotes, for example: additionalOptions: { "option-name": "value" }.
161
+ map<string, string> additional_options = 17;
162
+ }
163
+
164
+ message ViewConfig {
165
+ // The name of the view.
166
+ string name = 1;
167
+
168
+ // The dataset (schema) of the view.
169
+ string dataset = 2;
170
+
171
+ // The Google Cloud project (database) of the view.
172
+ string project = 3;
173
+
174
+ // Targets of actions that this action is dependent on.
175
+ repeated Target dependency_targets = 4;
176
+
177
+ // Path to the source file that the contents of the action is loaded from.
178
+ string filename = 5;
179
+
180
+ // A list of user-defined tags with which the action should be labeled.
181
+ repeated string tags = 6;
182
+
183
+ // If set to true, this action will not be executed. However, the action can
184
+ // still be depended upon. Useful for temporarily turning off broken
185
+ // actions.
186
+ bool disabled = 7;
187
+
188
+ // Queries to run before `query`. This can be useful for granting
189
+ // permissions.
190
+ repeated string pre_operations = 8;
191
+
192
+ // Queries to run after `query`.
193
+ repeated string post_operations = 9;
194
+
195
+ // Applies the materialized view optimization, see
196
+ // https://cloud.google.com/bigquery/docs/materialized-views-intro.
197
+ bool materialized = 10;
198
+
199
+ // Description of the view.
200
+ string description = 11;
201
+
202
+ // Descriptions of columns within the table.
203
+ repeated ColumnDescriptor columns = 12;
204
+
205
+ // Key-value pairs for BigQuery labels. If the label name contains special
206
+ // characters, e.g. hyphens, then quote its name, e.g. `labels: {
207
+ // "label-name": "value" }`.
208
+ map<string, string> labels = 13;
209
+
210
+ // Key-value pairs of additional options to pass to the BigQuery API.
211
+ //
212
+ // Some options, for example, partitionExpirationDays, have dedicated
213
+ // type/validity checked fields. For such options, use the dedicated
214
+ // fields.
215
+ //
216
+ // String values must be encapsulated in double-quotes, for example:
217
+ // additionalOptions: {numeric_option: "5", string_option: '"string-value"'}
218
+ //
219
+ // If the option name contains special characters, encapsulate the name in
220
+ // quotes, for example: additionalOptions: { "option-name": "value" }.
221
+ map<string, string> additional_options = 14;
222
+ }
223
+
224
+ message IncrementalTableConfig {
225
+ // The name of the incremental table.
226
+ string name = 1;
227
+
228
+ // The dataset (schema) of the incremental table.
229
+ string dataset = 2;
230
+
231
+ // The Google Cloud project (database) of the incremental table.
232
+ string project = 3;
233
+
234
+ // Targets of actions that this action is dependent on.
235
+ repeated Target dependency_targets = 4;
236
+
237
+ // Path to the source file that the contents of the action is loaded from.
238
+ string filename = 5;
239
+
240
+ // A list of user-defined tags with which the action should be labeled.
241
+ repeated string tags = 6;
242
+
243
+ // If set to true, this action will not be executed. However, the action can
244
+ // still be depended upon. Useful for temporarily turning off broken
245
+ // actions.
246
+ bool disabled = 7;
247
+
248
+ // Queries to run before `query`. This can be useful for granting
249
+ // permissions.
250
+ repeated string pre_operations = 8;
251
+
252
+ // Queries to run after `query`.
253
+ repeated string post_operations = 9;
254
+
255
+ // If true, prevents the dataset from being rebuilt from scratch.
256
+ bool protected = 10;
257
+
258
+ // is specified, if a row arrives whose key matches an existing row's key,
259
+ // then the existing row is overwritten with the new data.
260
+ repeated string unique_key = 11;
261
+
262
+ // Description of the incremental table.
263
+ string description = 12;
264
+
265
+ // Descriptions of columns within the table.
266
+ repeated ColumnDescriptor columns = 13;
267
+
268
+ // The key by which to partition the table. Typically the name of a
269
+ // timestamp or the date column. See
270
+ // https://cloud.google.com/dataform/docs/partitions-clusters.
271
+ string partition_by = 14;
272
+
273
+ // The number of days for which BigQuery stores data in each partition.
274
+ // The setting applies to all partitions in a table, but is calculated
275
+ // independently for each partition based on the partition time.
276
+ int32 partition_expiration_days = 15;
277
+
278
+ // Declares whether the partitioned table requires a WHERE clause
279
+ // predicate filter that filters the partitioning column.
280
+ bool require_partition_filter = 16;
281
+
282
+ // SQL-based filter for when incremental updates are applied.
283
+ string update_partition_filter = 17;
284
+
285
+ // The keys by which to cluster partitions by. See
286
+ // https://cloud.google.com/dataform/docs/partitions-clusters.
287
+ repeated string cluster_by = 18;
288
+
289
+ // Key-value pairs for BigQuery labels. If the label name contains special
290
+ // characters, e.g. hyphens, then quote its name, e.g. `labels: {
291
+ // "label-name": "value" }`.
292
+ map<string, string> labels = 19;
293
+
294
+ // Key-value pairs of additional options to pass to the BigQuery API.
295
+ //
296
+ // Some options, for example, partitionExpirationDays, have dedicated
297
+ // type/validity checked fields. For such options, use the dedicated
298
+ // fields.
299
+ //
300
+ // String values must be encapsulated in double-quotes, for example:
301
+ // additionalOptions: {numeric_option: "5", string_option: '"string-value"'}
302
+ //
303
+ // If the option name contains special characters, encapsulate the name in
304
+ // quotes, for example: additionalOptions: { "option-name": "value" }.
305
+ map<string, string> additional_options = 20;
306
+ }
307
+
308
+ message AssertionConfig {
309
+ // The name of the assertion.
310
+ string name = 1;
311
+
312
+ // The dataset (schema) of the assertion.
313
+ string dataset = 2;
314
+
315
+ // The Google Cloud project (database) of the assertion.
316
+ string project = 3;
317
+
318
+ // Targets of actions that this action is dependent on.
319
+ repeated Target dependency_targets = 4;
320
+
321
+ // Path to the source file that the contents of the action is loaded from.
322
+ string filename = 5;
323
+
324
+ // A list of user-defined tags with which the action should be labeled.
325
+ repeated string tags = 6;
326
+
327
+ // If set to true, this action will not be executed. However, the action can
328
+ // still be depended upon. Useful for temporarily turning off broken
329
+ // actions.
330
+ bool disabled = 7;
331
+
332
+ // Description of the assertion.
333
+ string description = 8;
334
+ }
335
+
336
+ message OperationConfig {
337
+ // The name of the operation.
338
+ string name = 1;
339
+
340
+ // The dataset (schema) of the operation.
341
+ string dataset = 2;
342
+
343
+ // The Google Cloud project (database) of the operation.
344
+ string project = 3;
345
+
346
+ // Targets of actions that this action is dependent on.
347
+ repeated Target dependency_targets = 4;
348
+
349
+ // Path to the source file that the contents of the action is loaded from.
350
+ string filename = 5;
351
+
352
+ // A list of user-defined tags with which the action should be labeled.
353
+ repeated string tags = 6;
354
+
355
+ // If set to true, this action will not be executed. However, the action can
356
+ // still be depended upon. Useful for temporarily turning off broken
357
+ // actions.
358
+ bool disabled = 7;
359
+
360
+ // Declares that this action creates a dataset which should be
361
+ // referenceable as a dependency target, for example by using the `ref`
362
+ // function.
363
+ bool has_output = 8;
364
+
365
+ // Description of the operation.
366
+ string description = 9;
367
+
368
+ // Descriptions of columns within the operation. Can only be set if
369
+ // hasOutput is true.
370
+ repeated ColumnDescriptor columns = 10;
371
+ }
372
+
373
+ message DeclarationConfig {
374
+ // The name of the declaration.
375
+ string name = 1;
376
+
377
+ // The dataset (schema) of the declaration.
378
+ string dataset = 2;
379
+
380
+ // The Google Cloud project (database) of the declaration.
381
+ string project = 3;
382
+
383
+ // Description of the declaration.
384
+ string description = 4;
385
+
386
+ // Descriptions of columns within the declaration.
387
+ repeated ColumnDescriptor columns = 5;
388
+ }
389
+
390
+ message NotebookConfig {
391
+ // The name of the notebook.
392
+ string name = 1;
393
+
394
+ // The Google Cloud location of the notebook.
395
+ string location = 2;
396
+
397
+ // The Google Cloud project (database) of the notebook.
398
+ string project = 3;
399
+
400
+ // Targets of actions that this action is dependent on.
401
+ repeated Target dependency_targets = 4;
402
+
403
+ // Path to the source file that the contents of the action is loaded from.
404
+ string filename = 5;
405
+
406
+ // A list of user-defined tags with which the action should be labeled.
407
+ repeated string tags = 6;
408
+
409
+ // If set to true, this action will not be executed. However, the action can
410
+ // still be depended upon. Useful for temporarily turning off broken
411
+ // actions.
412
+ bool disabled = 7;
413
+
414
+ // Description of the notebook.
415
+ string description = 8;
416
+
417
+ // TODO(ekrekr): add a notebook runtime field definition.
418
+ }
419
+
420
+ oneof action {
421
+ TableConfig table = 1;
422
+ ViewConfig view = 2;
423
+ IncrementalTableConfig incremental_table = 3;
424
+ AssertionConfig assertion = 4;
425
+ OperationConfig operation = 5;
426
+ DeclarationConfig declaration = 6;
427
+ NotebookConfig notebook = 7;
428
+ }
429
+ }
package/core.proto CHANGED
@@ -37,8 +37,6 @@ message ProjectConfig {
37
37
  string schema_suffix = 7;
38
38
  string table_prefix = 11;
39
39
 
40
- string dataform_core_version = 17;
41
-
42
40
  reserved 3, 4, 6, 8, 10, 12, 13;
43
41
  }
44
42
 
@@ -107,29 +105,13 @@ message ColumnDescriptor {
107
105
 
108
106
  string display_name = 3;
109
107
 
110
- enum DimensionType {
111
- UNKNOWN_DIMENSION = 0;
112
- CATEGORY = 1;
113
- TIMESTAMP = 2;
114
- NUMBER = 3;
115
- }
116
- DimensionType dimension_type = 4;
117
-
118
- enum Aggregation {
119
- UNKNOWN_AGGREGATION = 0;
120
- SUM = 1;
121
- DISTINCT = 2;
122
- DERIVED = 3;
123
- }
124
- Aggregation aggregation = 5;
125
-
126
- string expression = 6;
127
-
128
108
  repeated string tags = 7;
129
109
 
130
110
  // BigQuery specific settings.
131
111
 
132
112
  repeated string bigquery_policy_tags = 8;
113
+
114
+ reserved 4, 5, 6;
133
115
  }
134
116
 
135
117
  enum TableType {
@@ -137,7 +119,8 @@ enum TableType {
137
119
  TABLE = 1;
138
120
  INCREMENTAL = 2;
139
121
  VIEW = 3;
140
- INLINE = 4;
122
+
123
+ reserved 4;
141
124
  }
142
125
 
143
126
  message Table {
@@ -145,8 +128,6 @@ message Table {
145
128
  // configured in compiled graphs via the Table proto.
146
129
  TableType enum_type = 36;
147
130
 
148
- ActionConfig config = 37;
149
-
150
131
  Target target = 4;
151
132
  Target canonical_target = 32;
152
133
 
@@ -185,8 +166,6 @@ message Table {
185
166
  }
186
167
 
187
168
  message Operation {
188
- ActionConfig config = 15;
189
-
190
169
  Target target = 3;
191
170
  Target canonical_target = 13;
192
171
  repeated Target dependency_targets = 11;
@@ -208,8 +187,6 @@ message Operation {
208
187
  }
209
188
 
210
189
  message Assertion {
211
- ActionConfig config = 16;
212
-
213
190
  Target target = 8;
214
191
  Target canonical_target = 13;
215
192
 
@@ -240,8 +217,6 @@ enum ActionHermeticity {
240
217
  }
241
218
 
242
219
  message Declaration {
243
- ActionConfig config = 16;
244
-
245
220
  Target target = 2;
246
221
  Target canonical_target = 5;
247
222
 
@@ -262,11 +237,19 @@ message Test {
262
237
  }
263
238
 
264
239
  message Notebook {
265
- ActionConfig config = 1;
240
+ Target target = 1;
266
241
 
267
- Target target = 2;
242
+ Target canonical_target = 2;
243
+
244
+ repeated string tags = 3;
268
245
 
269
- string notebook_contents = 3;
246
+ repeated Target dependency_targets = 4;
247
+
248
+ string file_name = 5;
249
+
250
+ bool disabled = 6;
251
+
252
+ string notebook_contents = 7;
270
253
  }
271
254
 
272
255
  message CompiledGraph {
@@ -317,100 +300,3 @@ enum SupportedFeatures {
317
300
  UNKNOWN_FEATURE = 0;
318
301
  ARRAY_BUFFER_IPC = 1;
319
302
  }
320
-
321
- message ActionConfigs {
322
- repeated ActionConfig actions = 1;
323
- }
324
-
325
- message ActionConfig {
326
- // Target is a unique action identifier.
327
- Target target = 1;
328
-
329
- // A list of user-defined tags with which the action should be labeled.
330
- repeated string tags = 2;
331
-
332
- // Targets of actions that this action is dependent on.
333
- repeated Target dependency_targets = 3;
334
-
335
- // Filename to load the action contents from.
336
- string file_name = 4;
337
-
338
- // Type specific configs. Must correspond to the declared type.
339
- oneof action_config {
340
- TableConfig table = 5;
341
- ViewConfig view = 6;
342
- IncrementalTableConfig incremental_table = 7;
343
- AssertionConfig assertion = 8;
344
- OperationConfig operation = 9;
345
- DeclarationConfig declaration = 10;
346
- NotebookConfig notebook = 11;
347
- }
348
- }
349
-
350
- // Configuration options for BigQuery tables.
351
- message TableConfig {
352
- // If set to true, this action will not be executed. However, the action may
353
- // still be depended upon.
354
- bool disabled = 1;
355
- }
356
-
357
- // Configuration options for BigQuery views.
358
- message ViewConfig {
359
- // If set to true, this action will not be executed. However, the action may
360
- // still be depended upon.
361
- bool disabled = 1;
362
-
363
- // If set to true, will make the view materialized.
364
- // For more information, read the [BigQuery materialized view
365
- // docs](https://cloud.google.com/bigquery/docs/materialized-views-intro).
366
- bool materialized = 2;
367
- }
368
-
369
- // Configuration options for BigQuery incremental tables.
370
- message IncrementalTableConfig {
371
- // If set to true, this action will not be executed. However, the action may
372
- // still be depended upon.
373
- bool disabled = 1;
374
-
375
- // If set to true, running this action will ignore the full-refresh option.
376
- // This is useful for tables which are built from transient data, to ensure
377
- // that historical data is never lost.
378
- bool protected = 2;
379
-
380
- // Unique keys for merge criteria for incremental tables.
381
- // If configured, records with matching unique key(s) will be updated, rather
382
- // than new rows being inserted.
383
- repeated string unique_key = 3;
384
- }
385
-
386
- // Configuration options for BigQuery table assertions.
387
- message AssertionConfig {
388
- // If set to true, this action will not be executed. However, the action may
389
- // still be depended upon.
390
- bool disabled = 1;
391
- }
392
-
393
- // Configuration options for BigQuery table opertions.
394
- message OperationConfig {
395
- // If set to true, this action will not be executed. However, the action may
396
- // still be depended upon.
397
- bool disabled = 1;
398
-
399
- // Declares that this `operations` action creates a dataset which should be
400
- // referenceable using the `ref` function.
401
- bool has_output = 2;
402
- }
403
-
404
- // Configuration options for BigQuery table declarations.
405
- // TODO(ekrekr): remove this empty message, if explicit action type without a
406
- // `type` field proves to be impossible / ugly.
407
- message DeclarationConfig {}
408
-
409
- // Configuration options for Vertex AI notebooks.
410
- message NotebookConfig {
411
- // If set to true, this action will not be executed. However, the action may
412
- // still be depended upon.
413
- bool disabled = 1;
414
-
415
- // TODO(ekrekr): add a notebook runtime field definition.
416
- }
package/package.json CHANGED
@@ -41,7 +41,7 @@
41
41
  "publishConfig": {
42
42
  "access": "public"
43
43
  },
44
- "version": "3.0.0-alpha.2",
44
+ "version": "3.0.0-alpha.4",
45
45
  "name": "@dataform/core",
46
46
  "description": "Dataform core API.",
47
47
  "main": "bundle.js"