@dataform/core 3.0.13 → 3.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.d.ts +535 -117
- package/bundle.js +2 -2
- package/configs.proto +25 -7
- package/core.proto +22 -3
- package/package.json +1 -1
package/configs.proto
CHANGED
|
@@ -58,8 +58,6 @@ message WorkflowSettings {
|
|
|
58
58
|
NotebookRuntimeOptionsConfig default_notebook_runtime_options = 10;
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
-
// TODO(ekrekr): consolidate ActionsConfigs proto types in the JS API.
|
|
62
|
-
|
|
63
61
|
// Action configs defines the contents of `actions.yaml` configuration files.
|
|
64
62
|
message ActionConfigs {
|
|
65
63
|
repeated ActionConfig actions = 1;
|
|
@@ -315,9 +313,6 @@ message ActionConfig {
|
|
|
315
313
|
// If true, prevents the dataset from being rebuilt from scratch.
|
|
316
314
|
bool protected = 10;
|
|
317
315
|
|
|
318
|
-
// is specified, if a row arrives whose key matches an existing row's key,
|
|
319
|
-
// then the existing row is overwritten with the new data.
|
|
320
|
-
|
|
321
316
|
// If set, unique key represents a set of names of columns that will act as
|
|
322
317
|
// a the unique key. To enforce this, when updating the incremental
|
|
323
318
|
// table, Dataform merges rows with `uniqueKey` instead of appending them.
|
|
@@ -511,7 +506,8 @@ message ActionConfig {
|
|
|
511
506
|
// be add as dedpendency to this action
|
|
512
507
|
bool depend_on_dependency_assertions = 9;
|
|
513
508
|
|
|
514
|
-
//
|
|
509
|
+
// A notebook runtime field definition could be added here, to allow atomic
|
|
510
|
+
// runtime settings of notebooks.
|
|
515
511
|
}
|
|
516
512
|
|
|
517
513
|
message DataPreparationConfig {
|
|
@@ -537,6 +533,8 @@ message ActionConfig {
|
|
|
537
533
|
|
|
538
534
|
ErrorTableConfig error_table = 11;
|
|
539
535
|
|
|
536
|
+
LoadModeConfig load_mode = 12;
|
|
537
|
+
|
|
540
538
|
message ErrorTableConfig {
|
|
541
539
|
// The name of the error table.
|
|
542
540
|
string name = 1;
|
|
@@ -561,6 +559,25 @@ message ActionConfig {
|
|
|
561
559
|
NotebookConfig notebook = 7;
|
|
562
560
|
DataPreparationConfig data_preparation = 8;
|
|
563
561
|
}
|
|
562
|
+
|
|
563
|
+
message LoadModeConfig {
|
|
564
|
+
LoadMode mode = 1;
|
|
565
|
+
// Required when mode is MAXIMUM or UNIQUE
|
|
566
|
+
string incremental_column = 2;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
enum LoadMode {
|
|
570
|
+
// Replace existing table (default).
|
|
571
|
+
REPLACE_TABLE = 0;
|
|
572
|
+
// Insert into destination table.
|
|
573
|
+
APPEND = 1;
|
|
574
|
+
// Insert only records where the specified column value exceeds the existing
|
|
575
|
+
// maximum value in the destination table.
|
|
576
|
+
MAXIMUM = 2;
|
|
577
|
+
// Insert only records where the specified column value is not already
|
|
578
|
+
// present in the destination column values.
|
|
579
|
+
UNIQUE = 3;
|
|
580
|
+
}
|
|
564
581
|
}
|
|
565
582
|
|
|
566
583
|
message NotebookRuntimeOptionsConfig {
|
|
@@ -569,6 +586,7 @@ message NotebookRuntimeOptionsConfig {
|
|
|
569
586
|
string output_bucket = 1;
|
|
570
587
|
}
|
|
571
588
|
|
|
572
|
-
// Colab runtime template (https://cloud.google.com/colab/docs/runtimes), from
|
|
589
|
+
// Colab runtime template (https://cloud.google.com/colab/docs/runtimes), from
|
|
590
|
+
// which a runtime is created for notebook executions.
|
|
573
591
|
string runtime_template_name = 2;
|
|
574
592
|
}
|
package/core.proto
CHANGED
|
@@ -297,7 +297,8 @@ message DataPreparation {
|
|
|
297
297
|
bool disabled = 6;
|
|
298
298
|
|
|
299
299
|
oneof definition {
|
|
300
|
-
// YAML file containing the data preparation YAML with resolved
|
|
300
|
+
// YAML file containing the data preparation YAML with resolved
|
|
301
|
+
// dependencies, as well as error table and load configuration information.
|
|
301
302
|
string data_preparation_yaml = 11;
|
|
302
303
|
|
|
303
304
|
// SQL representing a compiled SQL string
|
|
@@ -308,9 +309,27 @@ message DataPreparation {
|
|
|
308
309
|
Target error_table = 12;
|
|
309
310
|
int32 error_table_retention_days = 13;
|
|
310
311
|
|
|
312
|
+
LoadConfiguration load = 16;
|
|
313
|
+
|
|
311
314
|
reserved 7, 10;
|
|
312
315
|
}
|
|
313
316
|
|
|
317
|
+
message LoadConfiguration {
|
|
318
|
+
oneof mode {
|
|
319
|
+
SimpleLoadMode replace = 1;
|
|
320
|
+
SimpleLoadMode append = 2;
|
|
321
|
+
IncrementalLoadMode maximum = 3;
|
|
322
|
+
IncrementalLoadMode unique = 4;
|
|
323
|
+
IncrementalLoadMode automatic = 5;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
message SimpleLoadMode {}
|
|
328
|
+
|
|
329
|
+
message IncrementalLoadMode {
|
|
330
|
+
string column_name = 1;
|
|
331
|
+
}
|
|
332
|
+
|
|
314
333
|
message CompiledGraph {
|
|
315
334
|
ProjectConfig project_config = 4;
|
|
316
335
|
|
|
@@ -318,8 +337,8 @@ message CompiledGraph {
|
|
|
318
337
|
repeated Operation operations = 2;
|
|
319
338
|
repeated Assertion assertions = 3;
|
|
320
339
|
repeated Declaration declarations = 9;
|
|
321
|
-
//
|
|
322
|
-
// it is not used at runtime.
|
|
340
|
+
// Note that the Test action doesn't really belong in the compiled graph
|
|
341
|
+
// because it is not used at runtime. Consider moving it if useful.
|
|
323
342
|
repeated Test tests = 8;
|
|
324
343
|
repeated Notebook notebooks = 12;
|
|
325
344
|
repeated DataPreparation data_preparations = 13;
|