@dataform/core 2.8.4 → 3.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.js +17 -1
- package/core.proto +143 -55
- package/package.json +2 -4
package/core.proto
CHANGED
|
@@ -1,4 +1,18 @@
|
|
|
1
|
-
|
|
1
|
+
// Copyright 2023 Google LLC
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
|
|
15
|
+
syntax = "proto3";
|
|
2
16
|
|
|
3
17
|
package dataform;
|
|
4
18
|
|
|
@@ -19,29 +33,13 @@ message ProjectConfig {
|
|
|
19
33
|
|
|
20
34
|
map<string, string> vars = 14;
|
|
21
35
|
|
|
22
|
-
int32 concurrent_query_limit = 13;
|
|
23
|
-
int32 idempotent_action_retries = 8;
|
|
24
|
-
|
|
25
36
|
string database_suffix = 15;
|
|
26
37
|
string schema_suffix = 7;
|
|
27
38
|
string table_prefix = 11;
|
|
28
39
|
|
|
40
|
+
string dataform_core_version = 17;
|
|
29
41
|
|
|
30
|
-
|
|
31
|
-
// Please use 'default_database' instead.
|
|
32
|
-
string gcloud_project_id = 6;
|
|
33
|
-
// This setting no longer has any effect.
|
|
34
|
-
bool use_run_cache = 10;
|
|
35
|
-
|
|
36
|
-
reserved 3, 4, 12;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
message GenerateIndexConfig {
|
|
40
|
-
CompileConfig compile_config = 1;
|
|
41
|
-
repeated string include_paths = 2;
|
|
42
|
-
repeated string definition_paths = 3;
|
|
43
|
-
|
|
44
|
-
reserved 4;
|
|
42
|
+
reserved 3, 4, 6, 8, 10, 12, 13;
|
|
45
43
|
}
|
|
46
44
|
|
|
47
45
|
message CompileConfig {
|
|
@@ -57,10 +55,7 @@ message CompileConfig {
|
|
|
57
55
|
// Override compilation timeout settings.
|
|
58
56
|
int32 timeout_millis = 6;
|
|
59
57
|
|
|
60
|
-
|
|
61
|
-
bool use_main = 9;
|
|
62
|
-
|
|
63
|
-
reserved 2, 4, 5, 7;
|
|
58
|
+
reserved 2, 4, 5, 7, 9;
|
|
64
59
|
}
|
|
65
60
|
|
|
66
61
|
message Target {
|
|
@@ -79,29 +74,6 @@ message BigQueryOptions {
|
|
|
79
74
|
map<string, string> additional_options = 7;
|
|
80
75
|
}
|
|
81
76
|
|
|
82
|
-
message RedshiftOptions {
|
|
83
|
-
string dist_key = 1;
|
|
84
|
-
string dist_style = 2;
|
|
85
|
-
repeated string sort_keys = 3;
|
|
86
|
-
string sort_style = 4;
|
|
87
|
-
|
|
88
|
-
bool bind = 5 [deprecated = true];
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
message SnowflakeOptions {
|
|
92
|
-
bool secure = 1;
|
|
93
|
-
bool transient = 2;
|
|
94
|
-
repeated string cluster_by = 3;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
message SQLDataWarehouseOptions {
|
|
98
|
-
string distribution = 1;
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
message PrestoOptions {
|
|
102
|
-
repeated string partition_by = 3;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
77
|
message GraphErrors {
|
|
106
78
|
repeated CompilationError compilation_errors = 1;
|
|
107
79
|
|
|
@@ -129,7 +101,8 @@ message ColumnDescriptor {
|
|
|
129
101
|
string description = 1;
|
|
130
102
|
|
|
131
103
|
// For nested records, the path might look like e.g. ["record", "column"].
|
|
132
|
-
// For simple columns, this will always contain a single entry e.g.
|
|
104
|
+
// For simple columns, this will always contain a single entry e.g.
|
|
105
|
+
// ["column"].
|
|
133
106
|
repeated string path = 2;
|
|
134
107
|
|
|
135
108
|
string display_name = 3;
|
|
@@ -168,6 +141,12 @@ enum TableType {
|
|
|
168
141
|
}
|
|
169
142
|
|
|
170
143
|
message Table {
|
|
144
|
+
// For legacy compatability reasons, both incremental tables and views are
|
|
145
|
+
// configured in compiled graphs via the Table proto.
|
|
146
|
+
TableType enum_type = 36;
|
|
147
|
+
|
|
148
|
+
ActionConfig config = 37;
|
|
149
|
+
|
|
171
150
|
Target target = 4;
|
|
172
151
|
Target canonical_target = 32;
|
|
173
152
|
|
|
@@ -180,7 +159,6 @@ message Table {
|
|
|
180
159
|
string query = 5;
|
|
181
160
|
bool protected = 9;
|
|
182
161
|
bool materialized = 35;
|
|
183
|
-
TableType enum_type = 36;
|
|
184
162
|
|
|
185
163
|
ActionDescriptor action_descriptor = 24;
|
|
186
164
|
|
|
@@ -198,19 +176,17 @@ message Table {
|
|
|
198
176
|
repeated string incremental_post_ops = 29;
|
|
199
177
|
|
|
200
178
|
// Warehouse specific features.
|
|
201
|
-
RedshiftOptions redshift = 21;
|
|
202
179
|
BigQueryOptions bigquery = 22;
|
|
203
|
-
SnowflakeOptions snowflake = 33;
|
|
204
|
-
SQLDataWarehouseOptions sql_data_warehouse = 25;
|
|
205
|
-
PrestoOptions presto = 34;
|
|
206
180
|
|
|
207
181
|
// Generated.
|
|
208
182
|
string file_name = 18;
|
|
209
183
|
|
|
210
|
-
reserved 1,2, 7, 12, 16;
|
|
184
|
+
reserved 1, 2, 7, 12, 16;
|
|
211
185
|
}
|
|
212
186
|
|
|
213
187
|
message Operation {
|
|
188
|
+
ActionConfig config = 15;
|
|
189
|
+
|
|
214
190
|
Target target = 3;
|
|
215
191
|
Target canonical_target = 13;
|
|
216
192
|
repeated Target dependency_targets = 11;
|
|
@@ -232,6 +208,8 @@ message Operation {
|
|
|
232
208
|
}
|
|
233
209
|
|
|
234
210
|
message Assertion {
|
|
211
|
+
ActionConfig config = 16;
|
|
212
|
+
|
|
235
213
|
Target target = 8;
|
|
236
214
|
Target canonical_target = 13;
|
|
237
215
|
|
|
@@ -262,6 +240,7 @@ enum ActionHermeticity {
|
|
|
262
240
|
}
|
|
263
241
|
|
|
264
242
|
message Declaration {
|
|
243
|
+
ActionConfig config = 16;
|
|
265
244
|
|
|
266
245
|
Target target = 2;
|
|
267
246
|
Target canonical_target = 5;
|
|
@@ -282,6 +261,14 @@ message Test {
|
|
|
282
261
|
string file_name = 4;
|
|
283
262
|
}
|
|
284
263
|
|
|
264
|
+
message Notebook {
|
|
265
|
+
ActionConfig config = 1;
|
|
266
|
+
|
|
267
|
+
Target target = 2;
|
|
268
|
+
|
|
269
|
+
string notebook_contents = 3;
|
|
270
|
+
}
|
|
271
|
+
|
|
285
272
|
message CompiledGraph {
|
|
286
273
|
ProjectConfig project_config = 4;
|
|
287
274
|
|
|
@@ -289,7 +276,10 @@ message CompiledGraph {
|
|
|
289
276
|
repeated Operation operations = 2;
|
|
290
277
|
repeated Assertion assertions = 3;
|
|
291
278
|
repeated Declaration declarations = 9;
|
|
279
|
+
// TODO(ekrekr): move the Test action outside of the compiled graph, because
|
|
280
|
+
// it is not used at runtime.
|
|
292
281
|
repeated Test tests = 8;
|
|
282
|
+
repeated Notebook notebooks = 12;
|
|
293
283
|
|
|
294
284
|
GraphErrors graph_errors = 7;
|
|
295
285
|
|
|
@@ -320,9 +310,107 @@ message CompileExecutionResponse {
|
|
|
320
310
|
CompiledGraph compiled_graph = 1;
|
|
321
311
|
}
|
|
322
312
|
|
|
323
|
-
// This feature list is added to when making potentilly backwards breaking
|
|
324
|
-
// caller of Dataform Core know whether it supports the
|
|
313
|
+
// This feature list is added to when making potentilly backwards breaking
|
|
314
|
+
// changes. It lets the caller of Dataform Core know whether it supports the
|
|
315
|
+
// change.
|
|
325
316
|
enum SupportedFeatures {
|
|
326
317
|
UNKNOWN_FEATURE = 0;
|
|
327
318
|
ARRAY_BUFFER_IPC = 1;
|
|
328
319
|
}
|
|
320
|
+
|
|
321
|
+
message ActionConfigs {
|
|
322
|
+
repeated ActionConfig actions = 1;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
message ActionConfig {
|
|
326
|
+
// Target is a unique action identifier.
|
|
327
|
+
Target target = 1;
|
|
328
|
+
|
|
329
|
+
// A list of user-defined tags with which the action should be labeled.
|
|
330
|
+
repeated string tags = 2;
|
|
331
|
+
|
|
332
|
+
// Targets of actions that this action is dependent on.
|
|
333
|
+
repeated Target dependency_targets = 3;
|
|
334
|
+
|
|
335
|
+
// Filename to load the action contents from.
|
|
336
|
+
string file_name = 4;
|
|
337
|
+
|
|
338
|
+
// Type specific configs. Must correspond to the declared type.
|
|
339
|
+
oneof action_config {
|
|
340
|
+
TableConfig table = 5;
|
|
341
|
+
ViewConfig view = 6;
|
|
342
|
+
IncrementalTableConfig incremental_table = 7;
|
|
343
|
+
AssertionConfig assertion = 8;
|
|
344
|
+
OperationConfig operation = 9;
|
|
345
|
+
DeclarationConfig declaration = 10;
|
|
346
|
+
NotebookConfig notebook = 11;
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
// Configuration options for BigQuery tables.
|
|
351
|
+
message TableConfig {
|
|
352
|
+
// If set to true, this action will not be executed. However, the action may
|
|
353
|
+
// still be depended upon.
|
|
354
|
+
bool disabled = 1;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// Configuration options for BigQuery views.
|
|
358
|
+
message ViewConfig {
|
|
359
|
+
// If set to true, this action will not be executed. However, the action may
|
|
360
|
+
// still be depended upon.
|
|
361
|
+
bool disabled = 1;
|
|
362
|
+
|
|
363
|
+
// If set to true, will make the view materialized.
|
|
364
|
+
// For more information, read the [BigQuery materialized view
|
|
365
|
+
// docs](https://cloud.google.com/bigquery/docs/materialized-views-intro).
|
|
366
|
+
bool materialized = 2;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
// Configuration options for BigQuery incremental tables.
|
|
370
|
+
message IncrementalTableConfig {
|
|
371
|
+
// If set to true, this action will not be executed. However, the action may
|
|
372
|
+
// still be depended upon.
|
|
373
|
+
bool disabled = 1;
|
|
374
|
+
|
|
375
|
+
// If set to true, running this action will ignore the full-refresh option.
|
|
376
|
+
// This is useful for tables which are built from transient data, to ensure
|
|
377
|
+
// that historical data is never lost.
|
|
378
|
+
bool protected = 2;
|
|
379
|
+
|
|
380
|
+
// Unique keys for merge criteria for incremental tables.
|
|
381
|
+
// If configured, records with matching unique key(s) will be updated, rather
|
|
382
|
+
// than new rows being inserted.
|
|
383
|
+
repeated string unique_key = 3;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
// Configuration options for BigQuery table assertions.
|
|
387
|
+
message AssertionConfig {
|
|
388
|
+
// If set to true, this action will not be executed. However, the action may
|
|
389
|
+
// still be depended upon.
|
|
390
|
+
bool disabled = 1;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// Configuration options for BigQuery table opertions.
|
|
394
|
+
message OperationConfig {
|
|
395
|
+
// If set to true, this action will not be executed. However, the action may
|
|
396
|
+
// still be depended upon.
|
|
397
|
+
bool disabled = 1;
|
|
398
|
+
|
|
399
|
+
// Declares that this `operations` action creates a dataset which should be
|
|
400
|
+
// referenceable using the `ref` function.
|
|
401
|
+
bool has_output = 2;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
// Configuration options for BigQuery table declarations.
|
|
405
|
+
// TODO(ekrekr): remove this empty message, if explicit action type without a
|
|
406
|
+
// `type` field proves to be impossible / ugly.
|
|
407
|
+
message DeclarationConfig {}
|
|
408
|
+
|
|
409
|
+
// Configuration options for Vertex AI notebooks.
|
|
410
|
+
message NotebookConfig {
|
|
411
|
+
// If set to true, this action will not be executed. However, the action may
|
|
412
|
+
// still be depended upon.
|
|
413
|
+
bool disabled = 1;
|
|
414
|
+
|
|
415
|
+
// TODO(ekrekr): add a notebook runtime field definition.
|
|
416
|
+
}
|
package/package.json
CHANGED
|
@@ -4,7 +4,6 @@
|
|
|
4
4
|
"**/object-path": "^0.11.5",
|
|
5
5
|
"**/http-proxy": "^1.18.1",
|
|
6
6
|
"**/serialize-javascript": "^3.1.0",
|
|
7
|
-
"**/lodash": "^4.17.21",
|
|
8
7
|
"**/ssri": "^7.1.1",
|
|
9
8
|
"**/postcss": "^8.2.10",
|
|
10
9
|
"**/browserslist": "^4.16.5",
|
|
@@ -28,7 +27,7 @@
|
|
|
28
27
|
"**/markdown-it": "^12.3.2"
|
|
29
28
|
},
|
|
30
29
|
"homepage": "https://github.com/dataform-co/dataform",
|
|
31
|
-
"license": "
|
|
30
|
+
"license": "Apache-2.0",
|
|
32
31
|
"keywords": [
|
|
33
32
|
"dataform",
|
|
34
33
|
"etl",
|
|
@@ -37,13 +36,12 @@
|
|
|
37
36
|
"data-modelling",
|
|
38
37
|
"sql",
|
|
39
38
|
"bigquery",
|
|
40
|
-
"redshift",
|
|
41
39
|
"dataops"
|
|
42
40
|
],
|
|
43
41
|
"publishConfig": {
|
|
44
42
|
"access": "public"
|
|
45
43
|
},
|
|
46
|
-
"version": "
|
|
44
|
+
"version": "3.0.0-alpha.1",
|
|
47
45
|
"name": "@dataform/core",
|
|
48
46
|
"description": "Dataform core API.",
|
|
49
47
|
"main": "bundle.js"
|