@dataform/core 3.0.61-alpha.1 → 3.0.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.d.ts +181 -353
- package/bundle.js +1 -1
- package/configs.proto +25 -34
- package/core.proto +37 -14
- package/package.json +4 -2
package/configs.proto
CHANGED
|
@@ -82,26 +82,9 @@ message WorkflowSettings {
|
|
|
82
82
|
// Optional. If set to true, unit tests will be included in the compiled graph.
|
|
83
83
|
bool include_tests_in_compiled_graph = 16;
|
|
84
84
|
|
|
85
|
-
// Optional.
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
message DefaultIcebergConfig {
|
|
90
|
-
// Optional. Bucket name used to construct a storage URI when creating an
|
|
91
|
-
// Iceberg table.
|
|
92
|
-
string bucket_name = 1;
|
|
93
|
-
|
|
94
|
-
// Optional. Table folder root used to construct a storage URI when creating
|
|
95
|
-
// an Iceberg table.
|
|
96
|
-
string table_folder_root = 2;
|
|
97
|
-
|
|
98
|
-
// Optional. Table folder subpath used to construct a storage URI when
|
|
99
|
-
// creating an Iceberg table.
|
|
100
|
-
string table_folder_subpath = 3;
|
|
101
|
-
|
|
102
|
-
// Optional. The connection specifying the credentials to be used to read and
|
|
103
|
-
// write to external storage, such as Cloud Storage.
|
|
104
|
-
string connection = 4;
|
|
85
|
+
// Optional. If set to true, governance controls (e.g. Data Policies) will be
|
|
86
|
+
// preserved on BigQuery tables.
|
|
87
|
+
bool preserve_governance_controls = 18;
|
|
105
88
|
}
|
|
106
89
|
|
|
107
90
|
// Action configs defines the contents of `actions.yaml` configuration files.
|
|
@@ -297,6 +280,10 @@ message ActionConfig {
|
|
|
297
280
|
// If unset, the value from workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.
|
|
298
281
|
// Dataform CLI only (GCP Dataform support pending).
|
|
299
282
|
string reservation = 24;
|
|
283
|
+
|
|
284
|
+
// Optional. If true, preserves existing BigQuery column governance controls
|
|
285
|
+
// (e.g. data policy) when recreating a table.
|
|
286
|
+
optional bool preserve_governance_controls = 25;
|
|
300
287
|
}
|
|
301
288
|
|
|
302
289
|
message Metadata {
|
|
@@ -399,6 +386,18 @@ message ActionConfig {
|
|
|
399
386
|
SYNCHRONIZE = 3;
|
|
400
387
|
}
|
|
401
388
|
|
|
389
|
+
enum IncrementalStrategy {
|
|
390
|
+
// Strategy is unspecified. Defaults to MERGE if uniqueKey is configured,
|
|
391
|
+
// or APPEND otherwise.
|
|
392
|
+
INCREMENTAL_STRATEGY_UNSPECIFIED = 0;
|
|
393
|
+
// MERGE strategy. Matches rows based on uniqueKey to update existing rows
|
|
394
|
+
// or insert new ones.
|
|
395
|
+
INCREMENTAL_STRATEGY_MERGE = 1;
|
|
396
|
+
// INSERT_OVERWRITE strategy. Overwrites entire partitions in the target
|
|
397
|
+
// table that are present in the new staging data.
|
|
398
|
+
INCREMENTAL_STRATEGY_INSERT_OVERWRITE = 2;
|
|
399
|
+
}
|
|
400
|
+
|
|
402
401
|
message IncrementalTableConfig {
|
|
403
402
|
// The name of the incremental table.
|
|
404
403
|
string name = 1;
|
|
@@ -502,6 +501,12 @@ message ActionConfig {
|
|
|
502
501
|
// If unset, the value from workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.
|
|
503
502
|
// Dataform CLI only (GCP Dataform support pending).
|
|
504
503
|
string reservation = 27;
|
|
504
|
+
|
|
505
|
+
// Optional. The incremental strategy to use when updating the table.
|
|
506
|
+
IncrementalStrategy incremental_strategy = 28;
|
|
507
|
+
|
|
508
|
+
// Array of SQL-based predicates for when incremental updates are applied.
|
|
509
|
+
repeated string incremental_predicates = 29;
|
|
505
510
|
}
|
|
506
511
|
|
|
507
512
|
message AssertionConfig {
|
|
@@ -658,15 +663,6 @@ message ActionConfig {
|
|
|
658
663
|
|
|
659
664
|
// A notebook runtime field definition could be added here, to allow atomic
|
|
660
665
|
// runtime settings of notebooks.
|
|
661
|
-
|
|
662
|
-
enum ExecutionEngine {
|
|
663
|
-
EXECUTION_ENGINE_UNSPECIFIED = 0;
|
|
664
|
-
COLAB = 1;
|
|
665
|
-
MANAGED_SPARK = 2;
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
// Optional. The execution engine to use for this notebook.
|
|
669
|
-
ExecutionEngine execution_engine = 10;
|
|
670
666
|
}
|
|
671
667
|
|
|
672
668
|
message DataPreparationConfig {
|
|
@@ -773,8 +769,3 @@ message NotebookRuntimeOptionsConfig {
|
|
|
773
769
|
}
|
|
774
770
|
|
|
775
771
|
}
|
|
776
|
-
|
|
777
|
-
message ManagedSparkExecutionOptionsConfig {
|
|
778
|
-
// Required: A GCS bucket URI to stage the notebook file before execution.
|
|
779
|
-
string staging_bucket_uri = 1;
|
|
780
|
-
}
|
package/core.proto
CHANGED
|
@@ -16,8 +16,8 @@ syntax = "proto3";
|
|
|
16
16
|
|
|
17
17
|
package dataform;
|
|
18
18
|
|
|
19
|
-
import "configs.proto";
|
|
20
19
|
import "extension.proto";
|
|
20
|
+
|
|
21
21
|
import "google/protobuf/struct.proto";
|
|
22
22
|
|
|
23
23
|
option java_package = "com.dataform.protos";
|
|
@@ -52,11 +52,29 @@ message ProjectConfig {
|
|
|
52
52
|
|
|
53
53
|
bool include_tests_in_compiled_graph = 22;
|
|
54
54
|
|
|
55
|
-
|
|
55
|
+
bool preserve_governance_controls = 24;
|
|
56
56
|
|
|
57
57
|
reserved 3, 4, 6, 8, 10, 12, 13;
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
+
message DefaultIcebergConfig {
|
|
61
|
+
// Optional. Bucket name used to construct a storage URI when creating an
|
|
62
|
+
// Iceberg table.
|
|
63
|
+
string bucket_name = 1;
|
|
64
|
+
|
|
65
|
+
// Optional. Table folder root used to construct a storage URI when creating
|
|
66
|
+
// an Iceberg table.
|
|
67
|
+
string table_folder_root = 2;
|
|
68
|
+
|
|
69
|
+
// Optional. Table folder subpath used to construct a storage URI when
|
|
70
|
+
// creating an Iceberg table.
|
|
71
|
+
string table_folder_subpath = 3;
|
|
72
|
+
|
|
73
|
+
// Optional. The connection specifying the credentials to be used to read and
|
|
74
|
+
// write to external storage, such as Cloud Storage.
|
|
75
|
+
string connection = 4;
|
|
76
|
+
}
|
|
77
|
+
|
|
60
78
|
message CompileConfig {
|
|
61
79
|
// Required.
|
|
62
80
|
string project_dir = 1;
|
|
@@ -108,6 +126,8 @@ message BigQueryOptions {
|
|
|
108
126
|
TableFormat table_format = 9;
|
|
109
127
|
FileFormat file_format = 10;
|
|
110
128
|
string storage_uri = 11;
|
|
129
|
+
repeated string incremental_predicates = 12;
|
|
130
|
+
bool preserve_governance_controls = 13;
|
|
111
131
|
}
|
|
112
132
|
|
|
113
133
|
message GraphErrors {
|
|
@@ -192,6 +212,18 @@ enum OnSchemaChange {
|
|
|
192
212
|
SYNCHRONIZE = 3;
|
|
193
213
|
}
|
|
194
214
|
|
|
215
|
+
enum IncrementalStrategy {
|
|
216
|
+
// Strategy is unspecified. Defaults to MERGE if uniqueKey is configured,
|
|
217
|
+
// or APPEND otherwise.
|
|
218
|
+
INCREMENTAL_STRATEGY_UNSPECIFIED = 0;
|
|
219
|
+
// MERGE strategy. Matches rows based on uniqueKey to update existing row
|
|
220
|
+
// or insert new ones.
|
|
221
|
+
MERGE = 1;
|
|
222
|
+
// INSERT_OVERWRITE strategy. Overwrites entire partitions in the target
|
|
223
|
+
// table that are present in the new staging data.
|
|
224
|
+
INSERT_OVERWRITE = 2;
|
|
225
|
+
}
|
|
226
|
+
|
|
195
227
|
message Table {
|
|
196
228
|
// For legacy compatability reasons, both incremental tables and views are
|
|
197
229
|
// configured in compiled graphs via the Table proto.
|
|
@@ -228,6 +260,9 @@ message Table {
|
|
|
228
260
|
|
|
229
261
|
string jit_code = 38;
|
|
230
262
|
|
|
263
|
+
// The incremental strategy to use when updating the incremental table.
|
|
264
|
+
IncrementalStrategy incremental_strategy = 39;
|
|
265
|
+
|
|
231
266
|
// Warehouse specific features.
|
|
232
267
|
BigQueryOptions bigquery = 22;
|
|
233
268
|
|
|
@@ -338,14 +373,6 @@ message Notebook {
|
|
|
338
373
|
bool disabled = 6;
|
|
339
374
|
|
|
340
375
|
string notebook_contents = 7;
|
|
341
|
-
|
|
342
|
-
enum ExecutionEngine {
|
|
343
|
-
EXECUTION_ENGINE_UNSPECIFIED = 0;
|
|
344
|
-
COLAB = 1;
|
|
345
|
-
MANAGED_SPARK = 2;
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
ExecutionEngine execution_engine = 8;
|
|
349
376
|
}
|
|
350
377
|
|
|
351
378
|
message RepositorySnapshotDestination {
|
|
@@ -364,10 +391,6 @@ message NotebookRuntimeOptions {
|
|
|
364
391
|
}
|
|
365
392
|
}
|
|
366
393
|
|
|
367
|
-
message ManagedSparkExecutionOptions {
|
|
368
|
-
string staging_bucket_uri = 1;
|
|
369
|
-
}
|
|
370
|
-
|
|
371
394
|
// Data Preparation Related entries
|
|
372
395
|
message DataPreparation {
|
|
373
396
|
// Used ONLY as an identifiers. All outputs should be listed in the targets
|
package/package.json
CHANGED
|
@@ -17,7 +17,9 @@
|
|
|
17
17
|
"**/node-fetch": "^2.6.7",
|
|
18
18
|
"**/markdown-it": "^12.3.2",
|
|
19
19
|
"**/wrap-ansi": "7.0.0",
|
|
20
|
-
"**/string-width": "4.1.0"
|
|
20
|
+
"**/string-width": "4.1.0",
|
|
21
|
+
"**/@eslint-community/eslint-utils": "4.9.1",
|
|
22
|
+
"**/@ungap/structured-clone": "1.3.1"
|
|
21
23
|
},
|
|
22
24
|
"homepage": "https://github.com/dataform-co/dataform",
|
|
23
25
|
"license": "Apache-2.0",
|
|
@@ -34,7 +36,7 @@
|
|
|
34
36
|
"publishConfig": {
|
|
35
37
|
"access": "public"
|
|
36
38
|
},
|
|
37
|
-
"version": "3.0.
|
|
39
|
+
"version": "3.0.62",
|
|
38
40
|
"name": "@dataform/core",
|
|
39
41
|
"description": "Dataform core API.",
|
|
40
42
|
"main": "bundle.js"
|