@dataform/core 3.0.61 → 3.0.62

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/configs.proto CHANGED
@@ -81,24 +81,10 @@ message WorkflowSettings {
81
81
 
82
82
  // Optional. If set to true, unit tests will be included in the compiled graph.
83
83
  bool include_tests_in_compiled_graph = 16;
84
- }
85
-
86
- message DefaultIcebergConfig {
87
- // Optional. Bucket name used to construct a storage URI when creating an
88
- // Iceberg table.
89
- string bucket_name = 1;
90
-
91
- // Optional. Table folder root used to construct a storage URI when creating
92
- // an Iceberg table.
93
- string table_folder_root = 2;
94
84
 
95
- // Optional. Table folder subpath used to construct a storage URI when
96
- // creating an Iceberg table.
97
- string table_folder_subpath = 3;
98
-
99
- // Optional. The connection specifying the credentials to be used to read and
100
- // write to external storage, such as Cloud Storage.
101
- string connection = 4;
85
+ // Optional. If set to true, governance controls (e.g. Data Policies) will be
86
+ // preserved on BigQuery tables.
87
+ bool preserve_governance_controls = 18;
102
88
  }
103
89
 
104
90
  // Action configs defines the contents of `actions.yaml` configuration files.
@@ -294,6 +280,10 @@ message ActionConfig {
294
280
  // If unset, the value from workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.
295
281
  // Dataform CLI only (GCP Dataform support pending).
296
282
  string reservation = 24;
283
+
284
+ // Optional. If true, preserves existing BigQuery column governance controls
285
+ // (e.g. data policy) when recreating a table.
286
+ optional bool preserve_governance_controls = 25;
297
287
  }
298
288
 
299
289
  message Metadata {
@@ -396,6 +386,18 @@ message ActionConfig {
396
386
  SYNCHRONIZE = 3;
397
387
  }
398
388
 
389
+ enum IncrementalStrategy {
390
+ // Strategy is unspecified. Defaults to MERGE if uniqueKey is configured,
391
+ // or APPEND otherwise.
392
+ INCREMENTAL_STRATEGY_UNSPECIFIED = 0;
393
+ // MERGE strategy. Matches rows based on uniqueKey to update existing rows
394
+ // or insert new ones.
395
+ INCREMENTAL_STRATEGY_MERGE = 1;
396
+ // INSERT_OVERWRITE strategy. Overwrites entire partitions in the target
397
+ // table that are present in the new staging data.
398
+ INCREMENTAL_STRATEGY_INSERT_OVERWRITE = 2;
399
+ }
400
+
399
401
  message IncrementalTableConfig {
400
402
  // The name of the incremental table.
401
403
  string name = 1;
@@ -499,6 +501,12 @@ message ActionConfig {
499
501
  // If unset, the value from workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.
500
502
  // Dataform CLI only (GCP Dataform support pending).
501
503
  string reservation = 27;
504
+
505
+ // Optional. The incremental strategy to use when updating the table.
506
+ IncrementalStrategy incremental_strategy = 28;
507
+
508
+ // Array of SQL-based predicates for when incremental updates are applied.
509
+ repeated string incremental_predicates = 29;
502
510
  }
503
511
 
504
512
  message AssertionConfig {
package/core.proto CHANGED
@@ -16,8 +16,8 @@ syntax = "proto3";
16
16
 
17
17
  package dataform;
18
18
 
19
- import "configs.proto";
20
19
  import "extension.proto";
20
+
21
21
  import "google/protobuf/struct.proto";
22
22
 
23
23
  option java_package = "com.dataform.protos";
@@ -52,9 +52,29 @@ message ProjectConfig {
52
52
 
53
53
  bool include_tests_in_compiled_graph = 22;
54
54
 
55
+ bool preserve_governance_controls = 24;
56
+
55
57
  reserved 3, 4, 6, 8, 10, 12, 13;
56
58
  }
57
59
 
60
+ message DefaultIcebergConfig {
61
+ // Optional. Bucket name used to construct a storage URI when creating an
62
+ // Iceberg table.
63
+ string bucket_name = 1;
64
+
65
+ // Optional. Table folder root used to construct a storage URI when creating
66
+ // an Iceberg table.
67
+ string table_folder_root = 2;
68
+
69
+ // Optional. Table folder subpath used to construct a storage URI when
70
+ // creating an Iceberg table.
71
+ string table_folder_subpath = 3;
72
+
73
+ // Optional. The connection specifying the credentials to be used to read and
74
+ // write to external storage, such as Cloud Storage.
75
+ string connection = 4;
76
+ }
77
+
58
78
  message CompileConfig {
59
79
  // Required.
60
80
  string project_dir = 1;
@@ -65,7 +85,8 @@ message CompileConfig {
65
85
  // Project configuration overrides.
66
86
  ProjectConfig project_config_override = 3;
67
87
 
68
- // Override compilation timeout settings.
88
+ // Override top-level compilation timeout (the single compile worker that
89
+ // runs once per invocation).
69
90
  int32 timeout_millis = 6;
70
91
 
71
92
  Extension extension = 10;
@@ -105,6 +126,8 @@ message BigQueryOptions {
105
126
  TableFormat table_format = 9;
106
127
  FileFormat file_format = 10;
107
128
  string storage_uri = 11;
129
+ repeated string incremental_predicates = 12;
130
+ bool preserve_governance_controls = 13;
108
131
  }
109
132
 
110
133
  message GraphErrors {
@@ -189,6 +212,18 @@ enum OnSchemaChange {
189
212
  SYNCHRONIZE = 3;
190
213
  }
191
214
 
215
+ enum IncrementalStrategy {
216
+ // Strategy is unspecified. Defaults to MERGE if uniqueKey is configured,
217
+ // or APPEND otherwise.
218
+ INCREMENTAL_STRATEGY_UNSPECIFIED = 0;
219
+ // MERGE strategy. Matches rows based on uniqueKey to update existing row
220
+ // or insert new ones.
221
+ MERGE = 1;
222
+ // INSERT_OVERWRITE strategy. Overwrites entire partitions in the target
223
+ // table that are present in the new staging data.
224
+ INSERT_OVERWRITE = 2;
225
+ }
226
+
192
227
  message Table {
193
228
  // For legacy compatability reasons, both incremental tables and views are
194
229
  // configured in compiled graphs via the Table proto.
@@ -225,6 +260,9 @@ message Table {
225
260
 
226
261
  string jit_code = 38;
227
262
 
263
+ // The incremental strategy to use when updating the incremental table.
264
+ IncrementalStrategy incremental_strategy = 39;
265
+
228
266
  // Warehouse specific features.
229
267
  BigQueryOptions bigquery = 22;
230
268
 
package/package.json CHANGED
@@ -17,7 +17,9 @@
17
17
  "**/node-fetch": "^2.6.7",
18
18
  "**/markdown-it": "^12.3.2",
19
19
  "**/wrap-ansi": "7.0.0",
20
- "**/string-width": "4.1.0"
20
+ "**/string-width": "4.1.0",
21
+ "**/@eslint-community/eslint-utils": "4.9.1",
22
+ "**/@ungap/structured-clone": "1.3.1"
21
23
  },
22
24
  "homepage": "https://github.com/dataform-co/dataform",
23
25
  "license": "Apache-2.0",
@@ -34,7 +36,7 @@
34
36
  "publishConfig": {
35
37
  "access": "public"
36
38
  },
37
- "version": "3.0.61",
39
+ "version": "3.0.62",
38
40
  "name": "@dataform/core",
39
41
  "description": "Dataform core API.",
40
42
  "main": "bundle.js"