@dataform/core 3.0.59 → 3.0.61-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.d.ts +261 -1
- package/bundle.js +2 -1
- package/configs.proto +20 -0
- package/core.proto +16 -1
- package/db_adapter.proto +2 -0
- package/jit.proto +2 -0
- package/package.json +1 -1
package/configs.proto
CHANGED
|
@@ -81,6 +81,9 @@ message WorkflowSettings {
|
|
|
81
81
|
|
|
82
82
|
// Optional. If set to true, unit tests will be included in the compiled graph.
|
|
83
83
|
bool include_tests_in_compiled_graph = 16;
|
|
84
|
+
|
|
85
|
+
// Optional. Default Managed Spark execution options for Notebook actions.
|
|
86
|
+
ManagedSparkExecutionOptionsConfig default_managed_spark_execution_options = 17;
|
|
84
87
|
}
|
|
85
88
|
|
|
86
89
|
message DefaultIcebergConfig {
|
|
@@ -542,6 +545,9 @@ message ActionConfig {
|
|
|
542
545
|
// If unset, the value from workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.
|
|
543
546
|
// Dataform CLI only (GCP Dataform support pending).
|
|
544
547
|
string reservation = 11;
|
|
548
|
+
|
|
549
|
+
// Metadata for this assertion.
|
|
550
|
+
Metadata metadata = 12;
|
|
545
551
|
}
|
|
546
552
|
|
|
547
553
|
message OperationConfig {
|
|
@@ -652,6 +658,15 @@ message ActionConfig {
|
|
|
652
658
|
|
|
653
659
|
// A notebook runtime field definition could be added here, to allow atomic
|
|
654
660
|
// runtime settings of notebooks.
|
|
661
|
+
|
|
662
|
+
enum ExecutionEngine {
|
|
663
|
+
EXECUTION_ENGINE_UNSPECIFIED = 0;
|
|
664
|
+
COLAB = 1;
|
|
665
|
+
MANAGED_SPARK = 2;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
// Optional. The execution engine to use for this notebook.
|
|
669
|
+
ExecutionEngine execution_engine = 10;
|
|
655
670
|
}
|
|
656
671
|
|
|
657
672
|
message DataPreparationConfig {
|
|
@@ -758,3 +773,8 @@ message NotebookRuntimeOptionsConfig {
|
|
|
758
773
|
}
|
|
759
774
|
|
|
760
775
|
}
|
|
776
|
+
|
|
777
|
+
message ManagedSparkExecutionOptionsConfig {
|
|
778
|
+
// Required: A GCS bucket URI to stage the notebook file before execution.
|
|
779
|
+
string staging_bucket_uri = 1;
|
|
780
|
+
}
|
package/core.proto
CHANGED
|
@@ -52,6 +52,8 @@ message ProjectConfig {
|
|
|
52
52
|
|
|
53
53
|
bool include_tests_in_compiled_graph = 22;
|
|
54
54
|
|
|
55
|
+
ManagedSparkExecutionOptions default_managed_spark_execution_options = 23;
|
|
56
|
+
|
|
55
57
|
reserved 3, 4, 6, 8, 10, 12, 13;
|
|
56
58
|
}
|
|
57
59
|
|
|
@@ -65,7 +67,8 @@ message CompileConfig {
|
|
|
65
67
|
// Project configuration overrides.
|
|
66
68
|
ProjectConfig project_config_override = 3;
|
|
67
69
|
|
|
68
|
-
// Override compilation timeout
|
|
70
|
+
// Override top-level compilation timeout (the single compile worker that
|
|
71
|
+
// runs once per invocation).
|
|
69
72
|
int32 timeout_millis = 6;
|
|
70
73
|
|
|
71
74
|
Extension extension = 10;
|
|
@@ -335,6 +338,14 @@ message Notebook {
|
|
|
335
338
|
bool disabled = 6;
|
|
336
339
|
|
|
337
340
|
string notebook_contents = 7;
|
|
341
|
+
|
|
342
|
+
enum ExecutionEngine {
|
|
343
|
+
EXECUTION_ENGINE_UNSPECIFIED = 0;
|
|
344
|
+
COLAB = 1;
|
|
345
|
+
MANAGED_SPARK = 2;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
ExecutionEngine execution_engine = 8;
|
|
338
349
|
}
|
|
339
350
|
|
|
340
351
|
message RepositorySnapshotDestination {
|
|
@@ -353,6 +364,10 @@ message NotebookRuntimeOptions {
|
|
|
353
364
|
}
|
|
354
365
|
}
|
|
355
366
|
|
|
367
|
+
message ManagedSparkExecutionOptions {
|
|
368
|
+
string staging_bucket_uri = 1;
|
|
369
|
+
}
|
|
370
|
+
|
|
356
371
|
// Data Preparation Related entries
|
|
357
372
|
message DataPreparation {
|
|
358
373
|
// Used ONLY as an identifiers. All outputs should be listed in the targets
|
package/db_adapter.proto
CHANGED
package/jit.proto
CHANGED