@dataform/core 3.0.60 → 3.0.61-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/configs.proto CHANGED
@@ -81,6 +81,9 @@ message WorkflowSettings {
81
81
 
82
82
  // Optional. If set to true, unit tests will be included in the compiled graph.
83
83
  bool include_tests_in_compiled_graph = 16;
84
+
85
+ // Optional. Default Managed Spark execution options for Notebook actions.
86
+ ManagedSparkExecutionOptionsConfig default_managed_spark_execution_options = 17;
84
87
  }
85
88
 
86
89
  message DefaultIcebergConfig {
@@ -655,6 +658,15 @@ message ActionConfig {
655
658
 
656
659
  // A notebook runtime field definition could be added here, to allow atomic
657
660
  // runtime settings of notebooks.
661
+
662
+ enum ExecutionEngine {
663
+ EXECUTION_ENGINE_UNSPECIFIED = 0;
664
+ COLAB = 1;
665
+ MANAGED_SPARK = 2;
666
+ }
667
+
668
+ // Optional. The execution engine to use for this notebook.
669
+ ExecutionEngine execution_engine = 10;
658
670
  }
659
671
 
660
672
  message DataPreparationConfig {
@@ -761,3 +773,8 @@ message NotebookRuntimeOptionsConfig {
761
773
  }
762
774
 
763
775
  }
776
+
777
+ message ManagedSparkExecutionOptionsConfig {
778
+ // Required: A GCS bucket URI to stage the notebook file before execution.
779
+ string staging_bucket_uri = 1;
780
+ }
package/core.proto CHANGED
@@ -52,6 +52,8 @@ message ProjectConfig {
52
52
 
53
53
  bool include_tests_in_compiled_graph = 22;
54
54
 
55
+ ManagedSparkExecutionOptions default_managed_spark_execution_options = 23;
56
+
55
57
  reserved 3, 4, 6, 8, 10, 12, 13;
56
58
  }
57
59
 
@@ -65,7 +67,8 @@ message CompileConfig {
65
67
  // Project configuration overrides.
66
68
  ProjectConfig project_config_override = 3;
67
69
 
68
- // Override compilation timeout settings.
70
+ // Override top-level compilation timeout (the single compile worker that
71
+ // runs once per invocation).
69
72
  int32 timeout_millis = 6;
70
73
 
71
74
  Extension extension = 10;
@@ -335,6 +338,14 @@ message Notebook {
335
338
  bool disabled = 6;
336
339
 
337
340
  string notebook_contents = 7;
341
+
342
+ enum ExecutionEngine {
343
+ EXECUTION_ENGINE_UNSPECIFIED = 0;
344
+ COLAB = 1;
345
+ MANAGED_SPARK = 2;
346
+ }
347
+
348
+ ExecutionEngine execution_engine = 8;
338
349
  }
339
350
 
340
351
  message RepositorySnapshotDestination {
@@ -353,6 +364,10 @@ message NotebookRuntimeOptions {
353
364
  }
354
365
  }
355
366
 
367
+ message ManagedSparkExecutionOptions {
368
+ string staging_bucket_uri = 1;
369
+ }
370
+
356
371
  // Data Preparation Related entries
357
372
  message DataPreparation {
358
373
  // Used ONLY as an identifiers. All outputs should be listed in the targets
package/package.json CHANGED
@@ -34,7 +34,7 @@
34
34
  "publishConfig": {
35
35
  "access": "public"
36
36
  },
37
- "version": "3.0.60",
37
+ "version": "3.0.61-alpha.1",
38
38
  "name": "@dataform/core",
39
39
  "description": "Dataform core API.",
40
40
  "main": "bundle.js"