@dataform/core 3.0.12 → 3.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.d.ts +752 -148
- package/bundle.js +2 -2
- package/configs.proto +40 -7
- package/core.proto +32 -4
- package/package.json +1 -1
package/configs.proto
CHANGED
|
@@ -58,8 +58,6 @@ message WorkflowSettings {
|
|
|
58
58
|
NotebookRuntimeOptionsConfig default_notebook_runtime_options = 10;
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
-
// TODO(ekrekr): consolidate ActionsConfigs proto types in the JS API.
|
|
62
|
-
|
|
63
61
|
// Action configs defines the contents of `actions.yaml` configuration files.
|
|
64
62
|
message ActionConfigs {
|
|
65
63
|
repeated ActionConfig actions = 1;
|
|
@@ -315,9 +313,6 @@ message ActionConfig {
|
|
|
315
313
|
// If true, prevents the dataset from being rebuilt from scratch.
|
|
316
314
|
bool protected = 10;
|
|
317
315
|
|
|
318
|
-
// is specified, if a row arrives whose key matches an existing row's key,
|
|
319
|
-
// then the existing row is overwritten with the new data.
|
|
320
|
-
|
|
321
316
|
// If set, unique key represents a set of names of columns that will act as
|
|
322
317
|
// a the unique key. To enforce this, when updating the incremental
|
|
323
318
|
// table, Dataform merges rows with `uniqueKey` instead of appending them.
|
|
@@ -511,7 +506,8 @@ message ActionConfig {
|
|
|
511
506
|
// be add as dedpendency to this action
|
|
512
507
|
bool depend_on_dependency_assertions = 9;
|
|
513
508
|
|
|
514
|
-
//
|
|
509
|
+
// A notebook runtime field definition could be added here, to allow atomic
|
|
510
|
+
// runtime settings of notebooks.
|
|
515
511
|
}
|
|
516
512
|
|
|
517
513
|
message DataPreparationConfig {
|
|
@@ -534,6 +530,23 @@ message ActionConfig {
|
|
|
534
530
|
|
|
535
531
|
// Description of the data preparation.
|
|
536
532
|
string description = 8;
|
|
533
|
+
|
|
534
|
+
ErrorTableConfig error_table = 11;
|
|
535
|
+
|
|
536
|
+
LoadModeConfig load_mode = 12;
|
|
537
|
+
|
|
538
|
+
message ErrorTableConfig {
|
|
539
|
+
// The name of the error table.
|
|
540
|
+
string name = 1;
|
|
541
|
+
|
|
542
|
+
// The dataset (schema) of the error table.
|
|
543
|
+
string dataset = 2;
|
|
544
|
+
|
|
545
|
+
// The Google Cloud project (database) of the error table.
|
|
546
|
+
string project = 3;
|
|
547
|
+
|
|
548
|
+
int32 retention_days = 4;
|
|
549
|
+
}
|
|
537
550
|
}
|
|
538
551
|
|
|
539
552
|
oneof action {
|
|
@@ -546,6 +559,25 @@ message ActionConfig {
|
|
|
546
559
|
NotebookConfig notebook = 7;
|
|
547
560
|
DataPreparationConfig data_preparation = 8;
|
|
548
561
|
}
|
|
562
|
+
|
|
563
|
+
message LoadModeConfig {
|
|
564
|
+
LoadMode mode = 1;
|
|
565
|
+
// Required when mode is MAXIMUM or UNIQUE
|
|
566
|
+
string incremental_column = 2;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
enum LoadMode {
|
|
570
|
+
// Replace existing table (default).
|
|
571
|
+
REPLACE_TABLE = 0;
|
|
572
|
+
// Insert into destination table.
|
|
573
|
+
APPEND = 1;
|
|
574
|
+
// Insert only records where the specified column value exceeds the existing
|
|
575
|
+
// maximum value in the destination table.
|
|
576
|
+
MAXIMUM = 2;
|
|
577
|
+
// Insert only records where the specified column value is not already
|
|
578
|
+
// present in the destination column values.
|
|
579
|
+
UNIQUE = 3;
|
|
580
|
+
}
|
|
549
581
|
}
|
|
550
582
|
|
|
551
583
|
message NotebookRuntimeOptionsConfig {
|
|
@@ -554,6 +586,7 @@ message NotebookRuntimeOptionsConfig {
|
|
|
554
586
|
string output_bucket = 1;
|
|
555
587
|
}
|
|
556
588
|
|
|
557
|
-
// Colab runtime template (https://cloud.google.com/colab/docs/runtimes), from
|
|
589
|
+
// Colab runtime template (https://cloud.google.com/colab/docs/runtimes), from
|
|
590
|
+
// which a runtime is created for notebook executions.
|
|
558
591
|
string runtime_template_name = 2;
|
|
559
592
|
}
|
package/core.proto
CHANGED
|
@@ -296,12 +296,40 @@ message DataPreparation {
|
|
|
296
296
|
|
|
297
297
|
bool disabled = 6;
|
|
298
298
|
|
|
299
|
-
|
|
300
|
-
|
|
299
|
+
oneof definition {
|
|
300
|
+
// YAML file containing the data preparation YAML with resolved
|
|
301
|
+
// dependencies, as well as error table and load configuration information.
|
|
302
|
+
string data_preparation_yaml = 11;
|
|
303
|
+
|
|
304
|
+
// SQL representing a compiled SQL string
|
|
305
|
+
string query = 15;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
// Error table configuration
|
|
309
|
+
Target error_table = 12;
|
|
310
|
+
int32 error_table_retention_days = 13;
|
|
311
|
+
|
|
312
|
+
LoadConfiguration load = 16;
|
|
301
313
|
|
|
302
314
|
reserved 7, 10;
|
|
303
315
|
}
|
|
304
316
|
|
|
317
|
+
message LoadConfiguration {
|
|
318
|
+
oneof mode {
|
|
319
|
+
SimpleLoadMode replace = 1;
|
|
320
|
+
SimpleLoadMode append = 2;
|
|
321
|
+
IncrementalLoadMode maximum = 3;
|
|
322
|
+
IncrementalLoadMode unique = 4;
|
|
323
|
+
IncrementalLoadMode automatic = 5;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
message SimpleLoadMode {}
|
|
328
|
+
|
|
329
|
+
message IncrementalLoadMode {
|
|
330
|
+
string column_name = 1;
|
|
331
|
+
}
|
|
332
|
+
|
|
305
333
|
message CompiledGraph {
|
|
306
334
|
ProjectConfig project_config = 4;
|
|
307
335
|
|
|
@@ -309,8 +337,8 @@ message CompiledGraph {
|
|
|
309
337
|
repeated Operation operations = 2;
|
|
310
338
|
repeated Assertion assertions = 3;
|
|
311
339
|
repeated Declaration declarations = 9;
|
|
312
|
-
//
|
|
313
|
-
// it is not used at runtime.
|
|
340
|
+
// Note that the Test action doesn't really belong in the compiled graph
|
|
341
|
+
// because it is not used at runtime. Consider moving it if useful.
|
|
314
342
|
repeated Test tests = 8;
|
|
315
343
|
repeated Notebook notebooks = 12;
|
|
316
344
|
repeated DataPreparation data_preparations = 13;
|