@dataform/core 3.0.61-alpha.1 → 3.0.62

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/configs.proto CHANGED
@@ -82,26 +82,9 @@ message WorkflowSettings {
82
82
  // Optional. If set to true, unit tests will be included in the compiled graph.
83
83
  bool include_tests_in_compiled_graph = 16;
84
84
 
85
- // Optional. Default Managed Spark execution options for Notebook actions.
86
- ManagedSparkExecutionOptionsConfig default_managed_spark_execution_options = 17;
87
- }
88
-
89
- message DefaultIcebergConfig {
90
- // Optional. Bucket name used to construct a storage URI when creating an
91
- // Iceberg table.
92
- string bucket_name = 1;
93
-
94
- // Optional. Table folder root used to construct a storage URI when creating
95
- // an Iceberg table.
96
- string table_folder_root = 2;
97
-
98
- // Optional. Table folder subpath used to construct a storage URI when
99
- // creating an Iceberg table.
100
- string table_folder_subpath = 3;
101
-
102
- // Optional. The connection specifying the credentials to be used to read and
103
- // write to external storage, such as Cloud Storage.
104
- string connection = 4;
85
+ // Optional. If set to true, governance controls (e.g. Data Policies) will be
86
+ // preserved on BigQuery tables.
87
+ bool preserve_governance_controls = 18;
105
88
  }
106
89
 
107
90
  // Action configs defines the contents of `actions.yaml` configuration files.
@@ -297,6 +280,10 @@ message ActionConfig {
297
280
  // If unset, the value from workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.
298
281
  // Dataform CLI only (GCP Dataform support pending).
299
282
  string reservation = 24;
283
+
284
+ // Optional. If true, preserves existing BigQuery column governance controls
285
+ // (e.g. data policy) when recreating a table.
286
+ optional bool preserve_governance_controls = 25;
300
287
  }
301
288
 
302
289
  message Metadata {
@@ -399,6 +386,18 @@ message ActionConfig {
399
386
  SYNCHRONIZE = 3;
400
387
  }
401
388
 
389
+ enum IncrementalStrategy {
390
+ // Strategy is unspecified. Defaults to MERGE if uniqueKey is configured,
391
+ // or APPEND otherwise.
392
+ INCREMENTAL_STRATEGY_UNSPECIFIED = 0;
393
+ // MERGE strategy. Matches rows based on uniqueKey to update existing rows
394
+ // or insert new ones.
395
+ INCREMENTAL_STRATEGY_MERGE = 1;
396
+ // INSERT_OVERWRITE strategy. Overwrites entire partitions in the target
397
+ // table that are present in the new staging data.
398
+ INCREMENTAL_STRATEGY_INSERT_OVERWRITE = 2;
399
+ }
400
+
402
401
  message IncrementalTableConfig {
403
402
  // The name of the incremental table.
404
403
  string name = 1;
@@ -502,6 +501,12 @@ message ActionConfig {
502
501
  // If unset, the value from workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.
503
502
  // Dataform CLI only (GCP Dataform support pending).
504
503
  string reservation = 27;
504
+
505
+ // Optional. The incremental strategy to use when updating the table.
506
+ IncrementalStrategy incremental_strategy = 28;
507
+
508
+ // Array of SQL-based predicates for when incremental updates are applied.
509
+ repeated string incremental_predicates = 29;
505
510
  }
506
511
 
507
512
  message AssertionConfig {
@@ -658,15 +663,6 @@ message ActionConfig {
658
663
 
659
664
  // A notebook runtime field definition could be added here, to allow atomic
660
665
  // runtime settings of notebooks.
661
-
662
- enum ExecutionEngine {
663
- EXECUTION_ENGINE_UNSPECIFIED = 0;
664
- COLAB = 1;
665
- MANAGED_SPARK = 2;
666
- }
667
-
668
- // Optional. The execution engine to use for this notebook.
669
- ExecutionEngine execution_engine = 10;
670
666
  }
671
667
 
672
668
  message DataPreparationConfig {
@@ -773,8 +769,3 @@ message NotebookRuntimeOptionsConfig {
773
769
  }
774
770
 
775
771
  }
776
-
777
- message ManagedSparkExecutionOptionsConfig {
778
- // Required: A GCS bucket URI to stage the notebook file before execution.
779
- string staging_bucket_uri = 1;
780
- }
package/core.proto CHANGED
@@ -16,8 +16,8 @@ syntax = "proto3";
16
16
 
17
17
  package dataform;
18
18
 
19
- import "configs.proto";
20
19
  import "extension.proto";
20
+
21
21
  import "google/protobuf/struct.proto";
22
22
 
23
23
  option java_package = "com.dataform.protos";
@@ -52,11 +52,29 @@ message ProjectConfig {
52
52
 
53
53
  bool include_tests_in_compiled_graph = 22;
54
54
 
55
- ManagedSparkExecutionOptions default_managed_spark_execution_options = 23;
55
+ bool preserve_governance_controls = 24;
56
56
 
57
57
  reserved 3, 4, 6, 8, 10, 12, 13;
58
58
  }
59
59
 
60
+ message DefaultIcebergConfig {
61
+ // Optional. Bucket name used to construct a storage URI when creating an
62
+ // Iceberg table.
63
+ string bucket_name = 1;
64
+
65
+ // Optional. Table folder root used to construct a storage URI when creating
66
+ // an Iceberg table.
67
+ string table_folder_root = 2;
68
+
69
+ // Optional. Table folder subpath used to construct a storage URI when
70
+ // creating an Iceberg table.
71
+ string table_folder_subpath = 3;
72
+
73
+ // Optional. The connection specifying the credentials to be used to read and
74
+ // write to external storage, such as Cloud Storage.
75
+ string connection = 4;
76
+ }
77
+
60
78
  message CompileConfig {
61
79
  // Required.
62
80
  string project_dir = 1;
@@ -108,6 +126,8 @@ message BigQueryOptions {
108
126
  TableFormat table_format = 9;
109
127
  FileFormat file_format = 10;
110
128
  string storage_uri = 11;
129
+ repeated string incremental_predicates = 12;
130
+ bool preserve_governance_controls = 13;
111
131
  }
112
132
 
113
133
  message GraphErrors {
@@ -192,6 +212,18 @@ enum OnSchemaChange {
192
212
  SYNCHRONIZE = 3;
193
213
  }
194
214
 
215
+ enum IncrementalStrategy {
216
+ // Strategy is unspecified. Defaults to MERGE if uniqueKey is configured,
217
+ // or APPEND otherwise.
218
+ INCREMENTAL_STRATEGY_UNSPECIFIED = 0;
219
+ // MERGE strategy. Matches rows based on uniqueKey to update existing row
220
+ // or insert new ones.
221
+ MERGE = 1;
222
+ // INSERT_OVERWRITE strategy. Overwrites entire partitions in the target
223
+ // table that are present in the new staging data.
224
+ INSERT_OVERWRITE = 2;
225
+ }
226
+
195
227
  message Table {
196
228
  // For legacy compatability reasons, both incremental tables and views are
197
229
  // configured in compiled graphs via the Table proto.
@@ -228,6 +260,9 @@ message Table {
228
260
 
229
261
  string jit_code = 38;
230
262
 
263
+ // The incremental strategy to use when updating the incremental table.
264
+ IncrementalStrategy incremental_strategy = 39;
265
+
231
266
  // Warehouse specific features.
232
267
  BigQueryOptions bigquery = 22;
233
268
 
@@ -338,14 +373,6 @@ message Notebook {
338
373
  bool disabled = 6;
339
374
 
340
375
  string notebook_contents = 7;
341
-
342
- enum ExecutionEngine {
343
- EXECUTION_ENGINE_UNSPECIFIED = 0;
344
- COLAB = 1;
345
- MANAGED_SPARK = 2;
346
- }
347
-
348
- ExecutionEngine execution_engine = 8;
349
376
  }
350
377
 
351
378
  message RepositorySnapshotDestination {
@@ -364,10 +391,6 @@ message NotebookRuntimeOptions {
364
391
  }
365
392
  }
366
393
 
367
- message ManagedSparkExecutionOptions {
368
- string staging_bucket_uri = 1;
369
- }
370
-
371
394
  // Data Preparation Related entries
372
395
  message DataPreparation {
373
396
  // Used ONLY as an identifiers. All outputs should be listed in the targets
package/package.json CHANGED
@@ -17,7 +17,9 @@
17
17
  "**/node-fetch": "^2.6.7",
18
18
  "**/markdown-it": "^12.3.2",
19
19
  "**/wrap-ansi": "7.0.0",
20
- "**/string-width": "4.1.0"
20
+ "**/string-width": "4.1.0",
21
+ "**/@eslint-community/eslint-utils": "4.9.1",
22
+ "**/@ungap/structured-clone": "1.3.1"
21
23
  },
22
24
  "homepage": "https://github.com/dataform-co/dataform",
23
25
  "license": "Apache-2.0",
@@ -34,7 +36,7 @@
34
36
  "publishConfig": {
35
37
  "access": "public"
36
38
  },
37
- "version": "3.0.61-alpha.1",
39
+ "version": "3.0.62",
38
40
  "name": "@dataform/core",
39
41
  "description": "Dataform core API.",
40
42
  "main": "bundle.js"