@dataform/core 3.0.12 → 3.0.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/configs.proto CHANGED
@@ -58,8 +58,6 @@ message WorkflowSettings {
58
58
  NotebookRuntimeOptionsConfig default_notebook_runtime_options = 10;
59
59
  }
60
60
 
61
- // TODO(ekrekr): consolidate ActionsConfigs proto types in the JS API.
62
-
63
61
  // Action configs defines the contents of `actions.yaml` configuration files.
64
62
  message ActionConfigs {
65
63
  repeated ActionConfig actions = 1;
@@ -315,9 +313,6 @@ message ActionConfig {
315
313
  // If true, prevents the dataset from being rebuilt from scratch.
316
314
  bool protected = 10;
317
315
 
318
- // is specified, if a row arrives whose key matches an existing row's key,
319
- // then the existing row is overwritten with the new data.
320
-
321
316
  // If set, unique key represents a set of names of columns that will act as
322
317
  // a the unique key. To enforce this, when updating the incremental
323
318
  // table, Dataform merges rows with `uniqueKey` instead of appending them.
@@ -511,7 +506,8 @@ message ActionConfig {
511
506
  // be add as dedpendency to this action
512
507
  bool depend_on_dependency_assertions = 9;
513
508
 
514
- // TODO(ekrekr): add a notebook runtime field definition.
509
+ // A notebook runtime field definition could be added here, to allow atomic
510
+ // runtime settings of notebooks.
515
511
  }
516
512
 
517
513
  message DataPreparationConfig {
@@ -534,6 +530,23 @@ message ActionConfig {
534
530
 
535
531
  // Description of the data preparation.
536
532
  string description = 8;
533
+
534
+ ErrorTableConfig error_table = 11;
535
+
536
+ LoadModeConfig load_mode = 12;
537
+
538
+ message ErrorTableConfig {
539
+ // The name of the error table.
540
+ string name = 1;
541
+
542
+ // The dataset (schema) of the error table.
543
+ string dataset = 2;
544
+
545
+ // The Google Cloud project (database) of the error table.
546
+ string project = 3;
547
+
548
+ int32 retention_days = 4;
549
+ }
537
550
  }
538
551
 
539
552
  oneof action {
@@ -546,6 +559,25 @@ message ActionConfig {
546
559
  NotebookConfig notebook = 7;
547
560
  DataPreparationConfig data_preparation = 8;
548
561
  }
562
+
563
+ message LoadModeConfig {
564
+ LoadMode mode = 1;
565
+ // Required when mode is MAXIMUM or UNIQUE
566
+ string incremental_column = 2;
567
+ }
568
+
569
+ enum LoadMode {
570
+ // Replace existing table (default).
571
+ REPLACE_TABLE = 0;
572
+ // Insert into destination table.
573
+ APPEND = 1;
574
+ // Insert only records where the specified column value exceeds the existing
575
+ // maximum value in the destination table.
576
+ MAXIMUM = 2;
577
+ // Insert only records where the specified column value is not already
578
+ // present in the destination column values.
579
+ UNIQUE = 3;
580
+ }
549
581
  }
550
582
 
551
583
  message NotebookRuntimeOptionsConfig {
@@ -554,6 +586,7 @@ message NotebookRuntimeOptionsConfig {
554
586
  string output_bucket = 1;
555
587
  }
556
588
 
557
- // Colab runtime template (https://cloud.google.com/colab/docs/runtimes), from which a runtime is created for notebook executions.
589
+ // Colab runtime template (https://cloud.google.com/colab/docs/runtimes), from
590
+ // which a runtime is created for notebook executions.
558
591
  string runtime_template_name = 2;
559
592
  }
package/core.proto CHANGED
@@ -296,12 +296,40 @@ message DataPreparation {
296
296
 
297
297
  bool disabled = 6;
298
298
 
299
- // YAML file containing the data preparation YAML with resolved dependencies.
300
- string data_preparation_yaml = 11;
299
+ oneof definition {
300
+ // YAML file containing the data preparation YAML with resolved
301
+ // dependencies, as well as error table and load configuration information.
302
+ string data_preparation_yaml = 11;
303
+
304
+ // SQL representing a compiled SQL string
305
+ string query = 15;
306
+ }
307
+
308
+ // Error table configuration
309
+ Target error_table = 12;
310
+ int32 error_table_retention_days = 13;
311
+
312
+ LoadConfiguration load = 16;
301
313
 
302
314
  reserved 7, 10;
303
315
  }
304
316
 
317
+ message LoadConfiguration {
318
+ oneof mode {
319
+ SimpleLoadMode replace = 1;
320
+ SimpleLoadMode append = 2;
321
+ IncrementalLoadMode maximum = 3;
322
+ IncrementalLoadMode unique = 4;
323
+ IncrementalLoadMode automatic = 5;
324
+ }
325
+ }
326
+
327
+ message SimpleLoadMode {}
328
+
329
+ message IncrementalLoadMode {
330
+ string column_name = 1;
331
+ }
332
+
305
333
  message CompiledGraph {
306
334
  ProjectConfig project_config = 4;
307
335
 
@@ -309,8 +337,8 @@ message CompiledGraph {
309
337
  repeated Operation operations = 2;
310
338
  repeated Assertion assertions = 3;
311
339
  repeated Declaration declarations = 9;
312
- // TODO(ekrekr): move the Test action outside of the compiled graph, because
313
- // it is not used at runtime.
340
+ // Note that the Test action doesn't really belong in the compiled graph
341
+ // because it is not used at runtime. Consider moving it if useful.
314
342
  repeated Test tests = 8;
315
343
  repeated Notebook notebooks = 12;
316
344
  repeated DataPreparation data_preparations = 13;
package/package.json CHANGED
@@ -32,7 +32,7 @@
32
32
  "publishConfig": {
33
33
  "access": "public"
34
34
  },
35
- "version": "3.0.12",
35
+ "version": "3.0.14",
36
36
  "name": "@dataform/core",
37
37
  "description": "Dataform core API.",
38
38
  "main": "bundle.js"