@dataform/core 3.0.0-alpha.3 → 3.0.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.d.ts +6847 -0
- package/bundle.js +2 -2
- package/configs.proto +429 -0
- package/core.proto +15 -129
- package/package.json +1 -1
package/configs.proto
ADDED
|
@@ -0,0 +1,429 @@
|
|
|
1
|
+
// Copyright 2023 Google LLC
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
|
|
15
|
+
syntax = "proto3";
|
|
16
|
+
|
|
17
|
+
package dataform;
|
|
18
|
+
|
|
19
|
+
option java_package = "com.dataform.protos";
|
|
20
|
+
option java_outer_classname = "ConfigsMeta";
|
|
21
|
+
option java_multiple_files = true;
|
|
22
|
+
|
|
23
|
+
option go_package = "github.com/dataform-co/dataform/protos/dataform";
|
|
24
|
+
|
|
25
|
+
// Workflow Settings defines the contents of the `workflow_settings.yaml`
|
|
26
|
+
// configuration file.
|
|
27
|
+
message WorkflowSettings {
|
|
28
|
+
// The desired dataform core version to compile against.
|
|
29
|
+
string dataform_core_version = 1;
|
|
30
|
+
|
|
31
|
+
// Required. The default Google Cloud project (database).
|
|
32
|
+
string default_project = 2;
|
|
33
|
+
|
|
34
|
+
// Required. The default dataset (schema).
|
|
35
|
+
string default_dataset = 3;
|
|
36
|
+
|
|
37
|
+
// Required. The default BigQuery location to use. For more information on
|
|
38
|
+
// BigQuery locations, see https://cloud.google.com/bigquery/docs/locations.
|
|
39
|
+
string default_location = 4;
|
|
40
|
+
|
|
41
|
+
// Required. The default dataset (schema) for assertions.
|
|
42
|
+
string default_assertion_dataset = 5;
|
|
43
|
+
|
|
44
|
+
// Optional. User-defined variables that are made available to project code
|
|
45
|
+
// during compilation. An object containing a list of "key": value pairs.
|
|
46
|
+
// Example: `{ "name": "wrench", "mass": "1.3kg", "count": "3" }`.
|
|
47
|
+
map<string, string> vars = 6;
|
|
48
|
+
|
|
49
|
+
// Optional. The suffix to append to all Google Cloud project references.
|
|
50
|
+
string project_suffix = 7;
|
|
51
|
+
|
|
52
|
+
// Optional. The suffix to append to all dataset references.
|
|
53
|
+
string dataset_suffix = 8;
|
|
54
|
+
|
|
55
|
+
// Optional. The prefix to append to all action names.
|
|
56
|
+
string name_prefix = 9;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Action configs defines the contents of `actions.yaml` configuration files.
|
|
60
|
+
// TODO(ekrekr): consolidate these configuration options in the JS API.
|
|
61
|
+
message ActionConfigs {
|
|
62
|
+
repeated ActionConfig actions = 1;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Action config defines the configuration properties of actions.
|
|
66
|
+
message ActionConfig {
|
|
67
|
+
// Target represents a unique action identifier.
|
|
68
|
+
message Target {
|
|
69
|
+
// The Google Cloud project (database) of the action.
|
|
70
|
+
string project = 1;
|
|
71
|
+
|
|
72
|
+
// The dataset (schema) of the action. For notebooks, this is the location.
|
|
73
|
+
string dataset = 2;
|
|
74
|
+
|
|
75
|
+
// The name of the action.
|
|
76
|
+
string name = 4;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
message ColumnDescriptor {
|
|
80
|
+
// The identifier for the column, using multiple parts for nested records.
|
|
81
|
+
repeated string path = 1;
|
|
82
|
+
|
|
83
|
+
// A text description of the column.
|
|
84
|
+
string description = 2;
|
|
85
|
+
|
|
86
|
+
// A list of BigQuery policy tags that will be applied to the column.
|
|
87
|
+
repeated string bigquery_policy_tags = 3;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
message TableConfig {
|
|
91
|
+
// The name of the table.
|
|
92
|
+
string name = 1;
|
|
93
|
+
|
|
94
|
+
// The dataset (schema) of the table.
|
|
95
|
+
string dataset = 2;
|
|
96
|
+
|
|
97
|
+
// The Google Cloud project (database) of the table.
|
|
98
|
+
string project = 3;
|
|
99
|
+
|
|
100
|
+
// Targets of actions that this action is dependent on.
|
|
101
|
+
repeated Target dependency_targets = 4;
|
|
102
|
+
|
|
103
|
+
// Path to the source file that the contents of the action is loaded from.
|
|
104
|
+
string filename = 5;
|
|
105
|
+
|
|
106
|
+
// A list of user-defined tags with which the action should be labeled.
|
|
107
|
+
repeated string tags = 6;
|
|
108
|
+
|
|
109
|
+
// If set to true, this action will not be executed. However, the action can
|
|
110
|
+
// still be depended upon. Useful for temporarily turning off broken
|
|
111
|
+
// actions.
|
|
112
|
+
bool disabled = 7;
|
|
113
|
+
|
|
114
|
+
// Queries to run before `query`. This can be useful for granting
|
|
115
|
+
// permissions.
|
|
116
|
+
repeated string pre_operations = 8;
|
|
117
|
+
|
|
118
|
+
// Queries to run after `query`.
|
|
119
|
+
repeated string post_operations = 9;
|
|
120
|
+
|
|
121
|
+
// Description of the table.
|
|
122
|
+
string description = 10;
|
|
123
|
+
|
|
124
|
+
// Descriptions of columns within the table.
|
|
125
|
+
repeated ColumnDescriptor columns = 11;
|
|
126
|
+
|
|
127
|
+
// The key by which to partition the table. Typically the name of a
|
|
128
|
+
// timestamp or the date column. See
|
|
129
|
+
// https://cloud.google.com/dataform/docs/partitions-clusters.
|
|
130
|
+
string partition_by = 12;
|
|
131
|
+
|
|
132
|
+
// The number of days for which BigQuery stores data in each partition.
|
|
133
|
+
// The setting applies to all partitions in a table, but is calculated
|
|
134
|
+
// independently for each partition based on the partition time.
|
|
135
|
+
int32 partition_expiration_days = 13;
|
|
136
|
+
|
|
137
|
+
// Declares whether the partitioned table requires a WHERE clause
|
|
138
|
+
// predicate filter that filters the partitioning column.
|
|
139
|
+
bool require_partition_filter = 14;
|
|
140
|
+
|
|
141
|
+
// The keys by which to cluster partitions by. See
|
|
142
|
+
// https://cloud.google.com/dataform/docs/partitions-clusters.
|
|
143
|
+
repeated string cluster_by = 15;
|
|
144
|
+
|
|
145
|
+
// Key-value pairs for BigQuery labels. If the label name contains special
|
|
146
|
+
// characters, e.g. hyphens, then quote its name, e.g. `labels: {
|
|
147
|
+
// "label-name": "value" }`.
|
|
148
|
+
map<string, string> labels = 16;
|
|
149
|
+
|
|
150
|
+
// Key-value pairs of additional options to pass to the BigQuery API.
|
|
151
|
+
//
|
|
152
|
+
// Some options, for example, partitionExpirationDays, have dedicated
|
|
153
|
+
// type/validity checked fields. For such options, use the dedicated
|
|
154
|
+
// fields.
|
|
155
|
+
//
|
|
156
|
+
// String values must be encapsulated in double-quotes, for example:
|
|
157
|
+
// additionalOptions: {numeric_option: "5", string_option: '"string-value"'}
|
|
158
|
+
//
|
|
159
|
+
// If the option name contains special characters, encapsulate the name in
|
|
160
|
+
// quotes, for example: additionalOptions: { "option-name": "value" }.
|
|
161
|
+
map<string, string> additional_options = 17;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
message ViewConfig {
|
|
165
|
+
// The name of the view.
|
|
166
|
+
string name = 1;
|
|
167
|
+
|
|
168
|
+
// The dataset (schema) of the view.
|
|
169
|
+
string dataset = 2;
|
|
170
|
+
|
|
171
|
+
// The Google Cloud project (database) of the view.
|
|
172
|
+
string project = 3;
|
|
173
|
+
|
|
174
|
+
// Targets of actions that this action is dependent on.
|
|
175
|
+
repeated Target dependency_targets = 4;
|
|
176
|
+
|
|
177
|
+
// Path to the source file that the contents of the action is loaded from.
|
|
178
|
+
string filename = 5;
|
|
179
|
+
|
|
180
|
+
// A list of user-defined tags with which the action should be labeled.
|
|
181
|
+
repeated string tags = 6;
|
|
182
|
+
|
|
183
|
+
// If set to true, this action will not be executed. However, the action can
|
|
184
|
+
// still be depended upon. Useful for temporarily turning off broken
|
|
185
|
+
// actions.
|
|
186
|
+
bool disabled = 7;
|
|
187
|
+
|
|
188
|
+
// Queries to run before `query`. This can be useful for granting
|
|
189
|
+
// permissions.
|
|
190
|
+
repeated string pre_operations = 8;
|
|
191
|
+
|
|
192
|
+
// Queries to run after `query`.
|
|
193
|
+
repeated string post_operations = 9;
|
|
194
|
+
|
|
195
|
+
// Applies the materialized view optimization, see
|
|
196
|
+
// https://cloud.google.com/bigquery/docs/materialized-views-intro.
|
|
197
|
+
bool materialized = 10;
|
|
198
|
+
|
|
199
|
+
// Description of the view.
|
|
200
|
+
string description = 11;
|
|
201
|
+
|
|
202
|
+
// Descriptions of columns within the table.
|
|
203
|
+
repeated ColumnDescriptor columns = 12;
|
|
204
|
+
|
|
205
|
+
// Key-value pairs for BigQuery labels. If the label name contains special
|
|
206
|
+
// characters, e.g. hyphens, then quote its name, e.g. `labels: {
|
|
207
|
+
// "label-name": "value" }`.
|
|
208
|
+
map<string, string> labels = 13;
|
|
209
|
+
|
|
210
|
+
// Key-value pairs of additional options to pass to the BigQuery API.
|
|
211
|
+
//
|
|
212
|
+
// Some options, for example, partitionExpirationDays, have dedicated
|
|
213
|
+
// type/validity checked fields. For such options, use the dedicated
|
|
214
|
+
// fields.
|
|
215
|
+
//
|
|
216
|
+
// String values must be encapsulated in double-quotes, for example:
|
|
217
|
+
// additionalOptions: {numeric_option: "5", string_option: '"string-value"'}
|
|
218
|
+
//
|
|
219
|
+
// If the option name contains special characters, encapsulate the name in
|
|
220
|
+
// quotes, for example: additionalOptions: { "option-name": "value" }.
|
|
221
|
+
map<string, string> additional_options = 14;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
message IncrementalTableConfig {
|
|
225
|
+
// The name of the incremental table.
|
|
226
|
+
string name = 1;
|
|
227
|
+
|
|
228
|
+
// The dataset (schema) of the incremental table.
|
|
229
|
+
string dataset = 2;
|
|
230
|
+
|
|
231
|
+
// The Google Cloud project (database) of the incremental table.
|
|
232
|
+
string project = 3;
|
|
233
|
+
|
|
234
|
+
// Targets of actions that this action is dependent on.
|
|
235
|
+
repeated Target dependency_targets = 4;
|
|
236
|
+
|
|
237
|
+
// Path to the source file that the contents of the action is loaded from.
|
|
238
|
+
string filename = 5;
|
|
239
|
+
|
|
240
|
+
// A list of user-defined tags with which the action should be labeled.
|
|
241
|
+
repeated string tags = 6;
|
|
242
|
+
|
|
243
|
+
// If set to true, this action will not be executed. However, the action can
|
|
244
|
+
// still be depended upon. Useful for temporarily turning off broken
|
|
245
|
+
// actions.
|
|
246
|
+
bool disabled = 7;
|
|
247
|
+
|
|
248
|
+
// Queries to run before `query`. This can be useful for granting
|
|
249
|
+
// permissions.
|
|
250
|
+
repeated string pre_operations = 8;
|
|
251
|
+
|
|
252
|
+
// Queries to run after `query`.
|
|
253
|
+
repeated string post_operations = 9;
|
|
254
|
+
|
|
255
|
+
// If true, prevents the dataset from being rebuilt from scratch.
|
|
256
|
+
bool protected = 10;
|
|
257
|
+
|
|
258
|
+
// is specified, if a row arrives whose key matches an existing row's key,
|
|
259
|
+
// then the existing row is overwritten with the new data.
|
|
260
|
+
repeated string unique_key = 11;
|
|
261
|
+
|
|
262
|
+
// Description of the incremental table.
|
|
263
|
+
string description = 12;
|
|
264
|
+
|
|
265
|
+
// Descriptions of columns within the table.
|
|
266
|
+
repeated ColumnDescriptor columns = 13;
|
|
267
|
+
|
|
268
|
+
// The key by which to partition the table. Typically the name of a
|
|
269
|
+
// timestamp or the date column. See
|
|
270
|
+
// https://cloud.google.com/dataform/docs/partitions-clusters.
|
|
271
|
+
string partition_by = 14;
|
|
272
|
+
|
|
273
|
+
// The number of days for which BigQuery stores data in each partition.
|
|
274
|
+
// The setting applies to all partitions in a table, but is calculated
|
|
275
|
+
// independently for each partition based on the partition time.
|
|
276
|
+
int32 partition_expiration_days = 15;
|
|
277
|
+
|
|
278
|
+
// Declares whether the partitioned table requires a WHERE clause
|
|
279
|
+
// predicate filter that filters the partitioning column.
|
|
280
|
+
bool require_partition_filter = 16;
|
|
281
|
+
|
|
282
|
+
// SQL-based filter for when incremental updates are applied.
|
|
283
|
+
string update_partition_filter = 17;
|
|
284
|
+
|
|
285
|
+
// The keys by which to cluster partitions by. See
|
|
286
|
+
// https://cloud.google.com/dataform/docs/partitions-clusters.
|
|
287
|
+
repeated string cluster_by = 18;
|
|
288
|
+
|
|
289
|
+
// Key-value pairs for BigQuery labels. If the label name contains special
|
|
290
|
+
// characters, e.g. hyphens, then quote its name, e.g. `labels: {
|
|
291
|
+
// "label-name": "value" }`.
|
|
292
|
+
map<string, string> labels = 19;
|
|
293
|
+
|
|
294
|
+
// Key-value pairs of additional options to pass to the BigQuery API.
|
|
295
|
+
//
|
|
296
|
+
// Some options, for example, partitionExpirationDays, have dedicated
|
|
297
|
+
// type/validity checked fields. For such options, use the dedicated
|
|
298
|
+
// fields.
|
|
299
|
+
//
|
|
300
|
+
// String values must be encapsulated in double-quotes, for example:
|
|
301
|
+
// additionalOptions: {numeric_option: "5", string_option: '"string-value"'}
|
|
302
|
+
//
|
|
303
|
+
// If the option name contains special characters, encapsulate the name in
|
|
304
|
+
// quotes, for example: additionalOptions: { "option-name": "value" }.
|
|
305
|
+
map<string, string> additional_options = 20;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
message AssertionConfig {
|
|
309
|
+
// The name of the assertion.
|
|
310
|
+
string name = 1;
|
|
311
|
+
|
|
312
|
+
// The dataset (schema) of the assertion.
|
|
313
|
+
string dataset = 2;
|
|
314
|
+
|
|
315
|
+
// The Google Cloud project (database) of the assertion.
|
|
316
|
+
string project = 3;
|
|
317
|
+
|
|
318
|
+
// Targets of actions that this action is dependent on.
|
|
319
|
+
repeated Target dependency_targets = 4;
|
|
320
|
+
|
|
321
|
+
// Path to the source file that the contents of the action is loaded from.
|
|
322
|
+
string filename = 5;
|
|
323
|
+
|
|
324
|
+
// A list of user-defined tags with which the action should be labeled.
|
|
325
|
+
repeated string tags = 6;
|
|
326
|
+
|
|
327
|
+
// If set to true, this action will not be executed. However, the action can
|
|
328
|
+
// still be depended upon. Useful for temporarily turning off broken
|
|
329
|
+
// actions.
|
|
330
|
+
bool disabled = 7;
|
|
331
|
+
|
|
332
|
+
// Description of the assertion.
|
|
333
|
+
string description = 8;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
message OperationConfig {
|
|
337
|
+
// The name of the operation.
|
|
338
|
+
string name = 1;
|
|
339
|
+
|
|
340
|
+
// The dataset (schema) of the operation.
|
|
341
|
+
string dataset = 2;
|
|
342
|
+
|
|
343
|
+
// The Google Cloud project (database) of the operation.
|
|
344
|
+
string project = 3;
|
|
345
|
+
|
|
346
|
+
// Targets of actions that this action is dependent on.
|
|
347
|
+
repeated Target dependency_targets = 4;
|
|
348
|
+
|
|
349
|
+
// Path to the source file that the contents of the action is loaded from.
|
|
350
|
+
string filename = 5;
|
|
351
|
+
|
|
352
|
+
// A list of user-defined tags with which the action should be labeled.
|
|
353
|
+
repeated string tags = 6;
|
|
354
|
+
|
|
355
|
+
// If set to true, this action will not be executed. However, the action can
|
|
356
|
+
// still be depended upon. Useful for temporarily turning off broken
|
|
357
|
+
// actions.
|
|
358
|
+
bool disabled = 7;
|
|
359
|
+
|
|
360
|
+
// Declares that this action creates a dataset which should be
|
|
361
|
+
// referenceable as a dependency target, for example by using the `ref`
|
|
362
|
+
// function.
|
|
363
|
+
bool has_output = 8;
|
|
364
|
+
|
|
365
|
+
// Description of the operation.
|
|
366
|
+
string description = 9;
|
|
367
|
+
|
|
368
|
+
// Descriptions of columns within the operation. Can only be set if
|
|
369
|
+
// hasOutput is true.
|
|
370
|
+
repeated ColumnDescriptor columns = 10;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
message DeclarationConfig {
|
|
374
|
+
// The name of the declaration.
|
|
375
|
+
string name = 1;
|
|
376
|
+
|
|
377
|
+
// The dataset (schema) of the declaration.
|
|
378
|
+
string dataset = 2;
|
|
379
|
+
|
|
380
|
+
// The Google Cloud project (database) of the declaration.
|
|
381
|
+
string project = 3;
|
|
382
|
+
|
|
383
|
+
// Description of the declaration.
|
|
384
|
+
string description = 4;
|
|
385
|
+
|
|
386
|
+
// Descriptions of columns within the declaration.
|
|
387
|
+
repeated ColumnDescriptor columns = 5;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
message NotebookConfig {
|
|
391
|
+
// The name of the notebook.
|
|
392
|
+
string name = 1;
|
|
393
|
+
|
|
394
|
+
// The Google Cloud location of the notebook.
|
|
395
|
+
string location = 2;
|
|
396
|
+
|
|
397
|
+
// The Google Cloud project (database) of the notebook.
|
|
398
|
+
string project = 3;
|
|
399
|
+
|
|
400
|
+
// Targets of actions that this action is dependent on.
|
|
401
|
+
repeated Target dependency_targets = 4;
|
|
402
|
+
|
|
403
|
+
// Path to the source file that the contents of the action is loaded from.
|
|
404
|
+
string filename = 5;
|
|
405
|
+
|
|
406
|
+
// A list of user-defined tags with which the action should be labeled.
|
|
407
|
+
repeated string tags = 6;
|
|
408
|
+
|
|
409
|
+
// If set to true, this action will not be executed. However, the action can
|
|
410
|
+
// still be depended upon. Useful for temporarily turning off broken
|
|
411
|
+
// actions.
|
|
412
|
+
bool disabled = 7;
|
|
413
|
+
|
|
414
|
+
// Description of the notebook.
|
|
415
|
+
string description = 8;
|
|
416
|
+
|
|
417
|
+
// TODO(ekrekr): add a notebook runtime field definition.
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
oneof action {
|
|
421
|
+
TableConfig table = 1;
|
|
422
|
+
ViewConfig view = 2;
|
|
423
|
+
IncrementalTableConfig incremental_table = 3;
|
|
424
|
+
AssertionConfig assertion = 4;
|
|
425
|
+
OperationConfig operation = 5;
|
|
426
|
+
DeclarationConfig declaration = 6;
|
|
427
|
+
NotebookConfig notebook = 7;
|
|
428
|
+
}
|
|
429
|
+
}
|
package/core.proto
CHANGED
|
@@ -37,8 +37,6 @@ message ProjectConfig {
|
|
|
37
37
|
string schema_suffix = 7;
|
|
38
38
|
string table_prefix = 11;
|
|
39
39
|
|
|
40
|
-
string dataform_core_version = 17;
|
|
41
|
-
|
|
42
40
|
reserved 3, 4, 6, 8, 10, 12, 13;
|
|
43
41
|
}
|
|
44
42
|
|
|
@@ -107,29 +105,13 @@ message ColumnDescriptor {
|
|
|
107
105
|
|
|
108
106
|
string display_name = 3;
|
|
109
107
|
|
|
110
|
-
enum DimensionType {
|
|
111
|
-
UNKNOWN_DIMENSION = 0;
|
|
112
|
-
CATEGORY = 1;
|
|
113
|
-
TIMESTAMP = 2;
|
|
114
|
-
NUMBER = 3;
|
|
115
|
-
}
|
|
116
|
-
DimensionType dimension_type = 4;
|
|
117
|
-
|
|
118
|
-
enum Aggregation {
|
|
119
|
-
UNKNOWN_AGGREGATION = 0;
|
|
120
|
-
SUM = 1;
|
|
121
|
-
DISTINCT = 2;
|
|
122
|
-
DERIVED = 3;
|
|
123
|
-
}
|
|
124
|
-
Aggregation aggregation = 5;
|
|
125
|
-
|
|
126
|
-
string expression = 6;
|
|
127
|
-
|
|
128
108
|
repeated string tags = 7;
|
|
129
109
|
|
|
130
110
|
// BigQuery specific settings.
|
|
131
111
|
|
|
132
112
|
repeated string bigquery_policy_tags = 8;
|
|
113
|
+
|
|
114
|
+
reserved 4, 5, 6;
|
|
133
115
|
}
|
|
134
116
|
|
|
135
117
|
enum TableType {
|
|
@@ -137,7 +119,8 @@ enum TableType {
|
|
|
137
119
|
TABLE = 1;
|
|
138
120
|
INCREMENTAL = 2;
|
|
139
121
|
VIEW = 3;
|
|
140
|
-
|
|
122
|
+
|
|
123
|
+
reserved 4;
|
|
141
124
|
}
|
|
142
125
|
|
|
143
126
|
message Table {
|
|
@@ -145,8 +128,6 @@ message Table {
|
|
|
145
128
|
// configured in compiled graphs via the Table proto.
|
|
146
129
|
TableType enum_type = 36;
|
|
147
130
|
|
|
148
|
-
ActionConfig config = 37;
|
|
149
|
-
|
|
150
131
|
Target target = 4;
|
|
151
132
|
Target canonical_target = 32;
|
|
152
133
|
|
|
@@ -185,8 +166,6 @@ message Table {
|
|
|
185
166
|
}
|
|
186
167
|
|
|
187
168
|
message Operation {
|
|
188
|
-
ActionConfig config = 15;
|
|
189
|
-
|
|
190
169
|
Target target = 3;
|
|
191
170
|
Target canonical_target = 13;
|
|
192
171
|
repeated Target dependency_targets = 11;
|
|
@@ -208,8 +187,6 @@ message Operation {
|
|
|
208
187
|
}
|
|
209
188
|
|
|
210
189
|
message Assertion {
|
|
211
|
-
ActionConfig config = 16;
|
|
212
|
-
|
|
213
190
|
Target target = 8;
|
|
214
191
|
Target canonical_target = 13;
|
|
215
192
|
|
|
@@ -240,8 +217,6 @@ enum ActionHermeticity {
|
|
|
240
217
|
}
|
|
241
218
|
|
|
242
219
|
message Declaration {
|
|
243
|
-
ActionConfig config = 16;
|
|
244
|
-
|
|
245
220
|
Target target = 2;
|
|
246
221
|
Target canonical_target = 5;
|
|
247
222
|
|
|
@@ -262,11 +237,19 @@ message Test {
|
|
|
262
237
|
}
|
|
263
238
|
|
|
264
239
|
message Notebook {
|
|
265
|
-
|
|
240
|
+
Target target = 1;
|
|
266
241
|
|
|
267
|
-
Target
|
|
242
|
+
Target canonical_target = 2;
|
|
243
|
+
|
|
244
|
+
repeated string tags = 3;
|
|
268
245
|
|
|
269
|
-
|
|
246
|
+
repeated Target dependency_targets = 4;
|
|
247
|
+
|
|
248
|
+
string file_name = 5;
|
|
249
|
+
|
|
250
|
+
bool disabled = 6;
|
|
251
|
+
|
|
252
|
+
string notebook_contents = 7;
|
|
270
253
|
}
|
|
271
254
|
|
|
272
255
|
message CompiledGraph {
|
|
@@ -317,100 +300,3 @@ enum SupportedFeatures {
|
|
|
317
300
|
UNKNOWN_FEATURE = 0;
|
|
318
301
|
ARRAY_BUFFER_IPC = 1;
|
|
319
302
|
}
|
|
320
|
-
|
|
321
|
-
message ActionConfigs {
|
|
322
|
-
repeated ActionConfig actions = 1;
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
message ActionConfig {
|
|
326
|
-
// Target is a unique action identifier.
|
|
327
|
-
Target target = 1;
|
|
328
|
-
|
|
329
|
-
// A list of user-defined tags with which the action should be labeled.
|
|
330
|
-
repeated string tags = 2;
|
|
331
|
-
|
|
332
|
-
// Targets of actions that this action is dependent on.
|
|
333
|
-
repeated Target dependency_targets = 3;
|
|
334
|
-
|
|
335
|
-
// Filename to load the action contents from.
|
|
336
|
-
string file_name = 4;
|
|
337
|
-
|
|
338
|
-
// Type specific configs. Must correspond to the declared type.
|
|
339
|
-
oneof action_config {
|
|
340
|
-
TableConfig table = 5;
|
|
341
|
-
ViewConfig view = 6;
|
|
342
|
-
IncrementalTableConfig incremental_table = 7;
|
|
343
|
-
AssertionConfig assertion = 8;
|
|
344
|
-
OperationConfig operation = 9;
|
|
345
|
-
DeclarationConfig declaration = 10;
|
|
346
|
-
NotebookConfig notebook = 11;
|
|
347
|
-
}
|
|
348
|
-
}
|
|
349
|
-
|
|
350
|
-
// Configuration options for BigQuery tables.
|
|
351
|
-
message TableConfig {
|
|
352
|
-
// If set to true, this action will not be executed. However, the action may
|
|
353
|
-
// still be depended upon.
|
|
354
|
-
bool disabled = 1;
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
// Configuration options for BigQuery views.
|
|
358
|
-
message ViewConfig {
|
|
359
|
-
// If set to true, this action will not be executed. However, the action may
|
|
360
|
-
// still be depended upon.
|
|
361
|
-
bool disabled = 1;
|
|
362
|
-
|
|
363
|
-
// If set to true, will make the view materialized.
|
|
364
|
-
// For more information, read the [BigQuery materialized view
|
|
365
|
-
// docs](https://cloud.google.com/bigquery/docs/materialized-views-intro).
|
|
366
|
-
bool materialized = 2;
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
// Configuration options for BigQuery incremental tables.
|
|
370
|
-
message IncrementalTableConfig {
|
|
371
|
-
// If set to true, this action will not be executed. However, the action may
|
|
372
|
-
// still be depended upon.
|
|
373
|
-
bool disabled = 1;
|
|
374
|
-
|
|
375
|
-
// If set to true, running this action will ignore the full-refresh option.
|
|
376
|
-
// This is useful for tables which are built from transient data, to ensure
|
|
377
|
-
// that historical data is never lost.
|
|
378
|
-
bool protected = 2;
|
|
379
|
-
|
|
380
|
-
// Unique keys for merge criteria for incremental tables.
|
|
381
|
-
// If configured, records with matching unique key(s) will be updated, rather
|
|
382
|
-
// than new rows being inserted.
|
|
383
|
-
repeated string unique_key = 3;
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
// Configuration options for BigQuery table assertions.
|
|
387
|
-
message AssertionConfig {
|
|
388
|
-
// If set to true, this action will not be executed. However, the action may
|
|
389
|
-
// still be depended upon.
|
|
390
|
-
bool disabled = 1;
|
|
391
|
-
}
|
|
392
|
-
|
|
393
|
-
// Configuration options for BigQuery table opertions.
|
|
394
|
-
message OperationConfig {
|
|
395
|
-
// If set to true, this action will not be executed. However, the action may
|
|
396
|
-
// still be depended upon.
|
|
397
|
-
bool disabled = 1;
|
|
398
|
-
|
|
399
|
-
// Declares that this `operations` action creates a dataset which should be
|
|
400
|
-
// referenceable using the `ref` function.
|
|
401
|
-
bool has_output = 2;
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
// Configuration options for BigQuery table declarations.
|
|
405
|
-
// TODO(ekrekr): remove this empty message, if explicit action type without a
|
|
406
|
-
// `type` field proves to be impossible / ugly.
|
|
407
|
-
message DeclarationConfig {}
|
|
408
|
-
|
|
409
|
-
// Configuration options for Vertex AI notebooks.
|
|
410
|
-
message NotebookConfig {
|
|
411
|
-
// If set to true, this action will not be executed. However, the action may
|
|
412
|
-
// still be depended upon.
|
|
413
|
-
bool disabled = 1;
|
|
414
|
-
|
|
415
|
-
// TODO(ekrekr): add a notebook runtime field definition.
|
|
416
|
-
}
|