@dataform/core 3.0.61 → 3.0.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.d.ts +197 -117
- package/bundle.js +1 -1
- package/configs.proto +25 -17
- package/core.proto +40 -2
- package/package.json +4 -2
package/configs.proto
CHANGED
|
@@ -81,24 +81,10 @@ message WorkflowSettings {
|
|
|
81
81
|
|
|
82
82
|
// Optional. If set to true, unit tests will be included in the compiled graph.
|
|
83
83
|
bool include_tests_in_compiled_graph = 16;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
message DefaultIcebergConfig {
|
|
87
|
-
// Optional. Bucket name used to construct a storage URI when creating an
|
|
88
|
-
// Iceberg table.
|
|
89
|
-
string bucket_name = 1;
|
|
90
|
-
|
|
91
|
-
// Optional. Table folder root used to construct a storage URI when creating
|
|
92
|
-
// an Iceberg table.
|
|
93
|
-
string table_folder_root = 2;
|
|
94
84
|
|
|
95
|
-
// Optional.
|
|
96
|
-
//
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
// Optional. The connection specifying the credentials to be used to read and
|
|
100
|
-
// write to external storage, such as Cloud Storage.
|
|
101
|
-
string connection = 4;
|
|
85
|
+
// Optional. If set to true, governance controls (e.g. Data Policies) will be
|
|
86
|
+
// preserved on BigQuery tables.
|
|
87
|
+
bool preserve_governance_controls = 18;
|
|
102
88
|
}
|
|
103
89
|
|
|
104
90
|
// Action configs defines the contents of `actions.yaml` configuration files.
|
|
@@ -294,6 +280,10 @@ message ActionConfig {
|
|
|
294
280
|
// If unset, the value from workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.
|
|
295
281
|
// Dataform CLI only (GCP Dataform support pending).
|
|
296
282
|
string reservation = 24;
|
|
283
|
+
|
|
284
|
+
// Optional. If true, preserves existing BigQuery column governance controls
|
|
285
|
+
// (e.g. data policy) when recreating a table.
|
|
286
|
+
optional bool preserve_governance_controls = 25;
|
|
297
287
|
}
|
|
298
288
|
|
|
299
289
|
message Metadata {
|
|
@@ -396,6 +386,18 @@ message ActionConfig {
|
|
|
396
386
|
SYNCHRONIZE = 3;
|
|
397
387
|
}
|
|
398
388
|
|
|
389
|
+
enum IncrementalStrategy {
|
|
390
|
+
// Strategy is unspecified. Defaults to MERGE if uniqueKey is configured,
|
|
391
|
+
// or APPEND otherwise.
|
|
392
|
+
INCREMENTAL_STRATEGY_UNSPECIFIED = 0;
|
|
393
|
+
// MERGE strategy. Matches rows based on uniqueKey to update existing rows
|
|
394
|
+
// or insert new ones.
|
|
395
|
+
INCREMENTAL_STRATEGY_MERGE = 1;
|
|
396
|
+
// INSERT_OVERWRITE strategy. Overwrites entire partitions in the target
|
|
397
|
+
// table that are present in the new staging data.
|
|
398
|
+
INCREMENTAL_STRATEGY_INSERT_OVERWRITE = 2;
|
|
399
|
+
}
|
|
400
|
+
|
|
399
401
|
message IncrementalTableConfig {
|
|
400
402
|
// The name of the incremental table.
|
|
401
403
|
string name = 1;
|
|
@@ -499,6 +501,12 @@ message ActionConfig {
|
|
|
499
501
|
// If unset, the value from workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.
|
|
500
502
|
// Dataform CLI only (GCP Dataform support pending).
|
|
501
503
|
string reservation = 27;
|
|
504
|
+
|
|
505
|
+
// Optional. The incremental strategy to use when updating the table.
|
|
506
|
+
IncrementalStrategy incremental_strategy = 28;
|
|
507
|
+
|
|
508
|
+
// Array of SQL-based predicates for when incremental updates are applied.
|
|
509
|
+
repeated string incremental_predicates = 29;
|
|
502
510
|
}
|
|
503
511
|
|
|
504
512
|
message AssertionConfig {
|
package/core.proto
CHANGED
|
@@ -16,8 +16,8 @@ syntax = "proto3";
|
|
|
16
16
|
|
|
17
17
|
package dataform;
|
|
18
18
|
|
|
19
|
-
import "configs.proto";
|
|
20
19
|
import "extension.proto";
|
|
20
|
+
|
|
21
21
|
import "google/protobuf/struct.proto";
|
|
22
22
|
|
|
23
23
|
option java_package = "com.dataform.protos";
|
|
@@ -52,9 +52,29 @@ message ProjectConfig {
|
|
|
52
52
|
|
|
53
53
|
bool include_tests_in_compiled_graph = 22;
|
|
54
54
|
|
|
55
|
+
bool preserve_governance_controls = 24;
|
|
56
|
+
|
|
55
57
|
reserved 3, 4, 6, 8, 10, 12, 13;
|
|
56
58
|
}
|
|
57
59
|
|
|
60
|
+
message DefaultIcebergConfig {
|
|
61
|
+
// Optional. Bucket name used to construct a storage URI when creating an
|
|
62
|
+
// Iceberg table.
|
|
63
|
+
string bucket_name = 1;
|
|
64
|
+
|
|
65
|
+
// Optional. Table folder root used to construct a storage URI when creating
|
|
66
|
+
// an Iceberg table.
|
|
67
|
+
string table_folder_root = 2;
|
|
68
|
+
|
|
69
|
+
// Optional. Table folder subpath used to construct a storage URI when
|
|
70
|
+
// creating an Iceberg table.
|
|
71
|
+
string table_folder_subpath = 3;
|
|
72
|
+
|
|
73
|
+
// Optional. The connection specifying the credentials to be used to read and
|
|
74
|
+
// write to external storage, such as Cloud Storage.
|
|
75
|
+
string connection = 4;
|
|
76
|
+
}
|
|
77
|
+
|
|
58
78
|
message CompileConfig {
|
|
59
79
|
// Required.
|
|
60
80
|
string project_dir = 1;
|
|
@@ -65,7 +85,8 @@ message CompileConfig {
|
|
|
65
85
|
// Project configuration overrides.
|
|
66
86
|
ProjectConfig project_config_override = 3;
|
|
67
87
|
|
|
68
|
-
// Override compilation timeout
|
|
88
|
+
// Override top-level compilation timeout (the single compile worker that
|
|
89
|
+
// runs once per invocation).
|
|
69
90
|
int32 timeout_millis = 6;
|
|
70
91
|
|
|
71
92
|
Extension extension = 10;
|
|
@@ -105,6 +126,8 @@ message BigQueryOptions {
|
|
|
105
126
|
TableFormat table_format = 9;
|
|
106
127
|
FileFormat file_format = 10;
|
|
107
128
|
string storage_uri = 11;
|
|
129
|
+
repeated string incremental_predicates = 12;
|
|
130
|
+
bool preserve_governance_controls = 13;
|
|
108
131
|
}
|
|
109
132
|
|
|
110
133
|
message GraphErrors {
|
|
@@ -189,6 +212,18 @@ enum OnSchemaChange {
|
|
|
189
212
|
SYNCHRONIZE = 3;
|
|
190
213
|
}
|
|
191
214
|
|
|
215
|
+
enum IncrementalStrategy {
|
|
216
|
+
// Strategy is unspecified. Defaults to MERGE if uniqueKey is configured,
|
|
217
|
+
// or APPEND otherwise.
|
|
218
|
+
INCREMENTAL_STRATEGY_UNSPECIFIED = 0;
|
|
219
|
+
// MERGE strategy. Matches rows based on uniqueKey to update existing row
|
|
220
|
+
// or insert new ones.
|
|
221
|
+
MERGE = 1;
|
|
222
|
+
// INSERT_OVERWRITE strategy. Overwrites entire partitions in the target
|
|
223
|
+
// table that are present in the new staging data.
|
|
224
|
+
INSERT_OVERWRITE = 2;
|
|
225
|
+
}
|
|
226
|
+
|
|
192
227
|
message Table {
|
|
193
228
|
// For legacy compatability reasons, both incremental tables and views are
|
|
194
229
|
// configured in compiled graphs via the Table proto.
|
|
@@ -225,6 +260,9 @@ message Table {
|
|
|
225
260
|
|
|
226
261
|
string jit_code = 38;
|
|
227
262
|
|
|
263
|
+
// The incremental strategy to use when updating the incremental table.
|
|
264
|
+
IncrementalStrategy incremental_strategy = 39;
|
|
265
|
+
|
|
228
266
|
// Warehouse specific features.
|
|
229
267
|
BigQueryOptions bigquery = 22;
|
|
230
268
|
|
package/package.json
CHANGED
|
@@ -17,7 +17,9 @@
|
|
|
17
17
|
"**/node-fetch": "^2.6.7",
|
|
18
18
|
"**/markdown-it": "^12.3.2",
|
|
19
19
|
"**/wrap-ansi": "7.0.0",
|
|
20
|
-
"**/string-width": "4.1.0"
|
|
20
|
+
"**/string-width": "4.1.0",
|
|
21
|
+
"**/@eslint-community/eslint-utils": "4.9.1",
|
|
22
|
+
"**/@ungap/structured-clone": "1.3.1"
|
|
21
23
|
},
|
|
22
24
|
"homepage": "https://github.com/dataform-co/dataform",
|
|
23
25
|
"license": "Apache-2.0",
|
|
@@ -34,7 +36,7 @@
|
|
|
34
36
|
"publishConfig": {
|
|
35
37
|
"access": "public"
|
|
36
38
|
},
|
|
37
|
-
"version": "3.0.
|
|
39
|
+
"version": "3.0.62",
|
|
38
40
|
"name": "@dataform/core",
|
|
39
41
|
"description": "Dataform core API.",
|
|
40
42
|
"main": "bundle.js"
|