@dataform/core 3.0.13 → 3.0.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/configs.proto CHANGED
@@ -58,8 +58,6 @@ message WorkflowSettings {
58
58
  NotebookRuntimeOptionsConfig default_notebook_runtime_options = 10;
59
59
  }
60
60
 
61
- // TODO(ekrekr): consolidate ActionsConfigs proto types in the JS API.
62
-
63
61
  // Action configs defines the contents of `actions.yaml` configuration files.
64
62
  message ActionConfigs {
65
63
  repeated ActionConfig actions = 1;
@@ -315,9 +313,6 @@ message ActionConfig {
315
313
  // If true, prevents the dataset from being rebuilt from scratch.
316
314
  bool protected = 10;
317
315
 
318
- // is specified, if a row arrives whose key matches an existing row's key,
319
- // then the existing row is overwritten with the new data.
320
-
321
316
  // If set, unique key represents a set of names of columns that will act as
322
317
  // a the unique key. To enforce this, when updating the incremental
323
318
  // table, Dataform merges rows with `uniqueKey` instead of appending them.
@@ -511,7 +506,8 @@ message ActionConfig {
511
506
  // be add as dedpendency to this action
512
507
  bool depend_on_dependency_assertions = 9;
513
508
 
514
- // TODO(ekrekr): add a notebook runtime field definition.
509
+ // A notebook runtime field definition could be added here, to allow atomic
510
+ // runtime settings of notebooks.
515
511
  }
516
512
 
517
513
  message DataPreparationConfig {
@@ -537,6 +533,8 @@ message ActionConfig {
537
533
 
538
534
  ErrorTableConfig error_table = 11;
539
535
 
536
+ LoadModeConfig load_mode = 12;
537
+
540
538
  message ErrorTableConfig {
541
539
  // The name of the error table.
542
540
  string name = 1;
@@ -561,6 +559,25 @@ message ActionConfig {
561
559
  NotebookConfig notebook = 7;
562
560
  DataPreparationConfig data_preparation = 8;
563
561
  }
562
+
563
+ message LoadModeConfig {
564
+ LoadMode mode = 1;
565
+ // Required when mode is MAXIMUM or UNIQUE
566
+ string incremental_column = 2;
567
+ }
568
+
569
+ enum LoadMode {
570
+ // Replace existing table (default).
571
+ REPLACE_TABLE = 0;
572
+ // Insert into destination table.
573
+ APPEND = 1;
574
+ // Insert only records where the specified column value exceeds the existing
575
+ // maximum value in the destination table.
576
+ MAXIMUM = 2;
577
+ // Insert only records where the specified column value is not already
578
+ // present in the destination column values.
579
+ UNIQUE = 3;
580
+ }
564
581
  }
565
582
 
566
583
  message NotebookRuntimeOptionsConfig {
@@ -569,6 +586,7 @@ message NotebookRuntimeOptionsConfig {
569
586
  string output_bucket = 1;
570
587
  }
571
588
 
572
- // Colab runtime template (https://cloud.google.com/colab/docs/runtimes), from which a runtime is created for notebook executions.
589
+ // Colab runtime template (https://cloud.google.com/colab/docs/runtimes), from
590
+ // which a runtime is created for notebook executions.
573
591
  string runtime_template_name = 2;
574
592
  }
package/core.proto CHANGED
@@ -297,7 +297,8 @@ message DataPreparation {
297
297
  bool disabled = 6;
298
298
 
299
299
  oneof definition {
300
- // YAML file containing the data preparation YAML with resolved dependencies.
300
+ // YAML file containing the data preparation YAML with resolved
301
+ // dependencies, as well as error table and load configuration information.
301
302
  string data_preparation_yaml = 11;
302
303
 
303
304
  // SQL representing a compiled SQL string
@@ -308,9 +309,27 @@ message DataPreparation {
308
309
  Target error_table = 12;
309
310
  int32 error_table_retention_days = 13;
310
311
 
312
+ LoadConfiguration load = 16;
313
+
311
314
  reserved 7, 10;
312
315
  }
313
316
 
317
+ message LoadConfiguration {
318
+ oneof mode {
319
+ SimpleLoadMode replace = 1;
320
+ SimpleLoadMode append = 2;
321
+ IncrementalLoadMode maximum = 3;
322
+ IncrementalLoadMode unique = 4;
323
+ IncrementalLoadMode automatic = 5;
324
+ }
325
+ }
326
+
327
+ message SimpleLoadMode {}
328
+
329
+ message IncrementalLoadMode {
330
+ string column_name = 1;
331
+ }
332
+
314
333
  message CompiledGraph {
315
334
  ProjectConfig project_config = 4;
316
335
 
@@ -318,8 +337,8 @@ message CompiledGraph {
318
337
  repeated Operation operations = 2;
319
338
  repeated Assertion assertions = 3;
320
339
  repeated Declaration declarations = 9;
321
- // TODO(ekrekr): move the Test action outside of the compiled graph, because
322
- // it is not used at runtime.
340
+ // Note that the Test action doesn't really belong in the compiled graph
341
+ // because it is not used at runtime. Consider moving it if useful.
323
342
  repeated Test tests = 8;
324
343
  repeated Notebook notebooks = 12;
325
344
  repeated DataPreparation data_preparations = 13;
package/package.json CHANGED
@@ -32,7 +32,7 @@
32
32
  "publishConfig": {
33
33
  "access": "public"
34
34
  },
35
- "version": "3.0.13",
35
+ "version": "3.0.14",
36
36
  "name": "@dataform/core",
37
37
  "description": "Dataform core API.",
38
38
  "main": "bundle.js"