@dataform/core 3.0.58 → 3.0.59-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/configs.proto CHANGED
@@ -81,6 +81,9 @@ message WorkflowSettings {
81
81
 
82
82
  // Optional. If set to true, unit tests will be included in the compiled graph.
83
83
  bool include_tests_in_compiled_graph = 16;
84
+
85
+ // Optional. Default Managed Spark execution options for Notebook actions.
86
+ ManagedSparkExecutionOptionsConfig default_managed_spark_execution_options = 17;
84
87
  }
85
88
 
86
89
  message DefaultIcebergConfig {
@@ -542,6 +545,9 @@ message ActionConfig {
542
545
  // If unset, the value from workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.
543
546
  // Dataform CLI only (GCP Dataform support pending).
544
547
  string reservation = 11;
548
+
549
+ // Metadata for this assertion.
550
+ Metadata metadata = 12;
545
551
  }
546
552
 
547
553
  message OperationConfig {
@@ -652,6 +658,15 @@ message ActionConfig {
652
658
 
653
659
  // A notebook runtime field definition could be added here, to allow atomic
654
660
  // runtime settings of notebooks.
661
+
662
+ enum ExecutionEngine {
663
+ EXECUTION_ENGINE_UNSPECIFIED = 0;
664
+ COLAB = 1;
665
+ MANAGED_SPARK = 2;
666
+ }
667
+
668
+ // Optional. The execution engine to use for this notebook.
669
+ ExecutionEngine execution_engine = 10;
655
670
  }
656
671
 
657
672
  message DataPreparationConfig {
@@ -758,3 +773,8 @@ message NotebookRuntimeOptionsConfig {
758
773
  }
759
774
 
760
775
  }
776
+
777
+ message ManagedSparkExecutionOptionsConfig {
778
+ // Required: A GCS bucket URI to stage the notebook file before execution.
779
+ string staging_bucket_uri = 1;
780
+ }
package/core.proto CHANGED
@@ -52,6 +52,8 @@ message ProjectConfig {
52
52
 
53
53
  bool include_tests_in_compiled_graph = 22;
54
54
 
55
+ ManagedSparkExecutionOptions default_managed_spark_execution_options = 23;
56
+
55
57
  reserved 3, 4, 6, 8, 10, 12, 13;
56
58
  }
57
59
 
@@ -335,6 +337,14 @@ message Notebook {
335
337
  bool disabled = 6;
336
338
 
337
339
  string notebook_contents = 7;
340
+
341
+ enum ExecutionEngine {
342
+ EXECUTION_ENGINE_UNSPECIFIED = 0;
343
+ COLAB = 1;
344
+ MANAGED_SPARK = 2;
345
+ }
346
+
347
+ ExecutionEngine execution_engine = 8;
338
348
  }
339
349
 
340
350
  message RepositorySnapshotDestination {
@@ -353,6 +363,10 @@ message NotebookRuntimeOptions {
353
363
  }
354
364
  }
355
365
 
366
+ message ManagedSparkExecutionOptions {
367
+ string staging_bucket_uri = 1;
368
+ }
369
+
356
370
  // Data Preparation Related entries
357
371
  message DataPreparation {
358
372
  // Used ONLY as an identifiers. All outputs should be listed in the targets
package/db_adapter.proto CHANGED
@@ -16,6 +16,8 @@ syntax = "proto3";
16
16
 
17
17
  package dataform;
18
18
 
19
+ option go_package = "github.com/dataform-co/dataform/protos/dataform";
20
+
19
21
  import "core.proto";
20
22
 
21
23
  // Common structures for Evaluation, execution and compilation.
package/jit.proto CHANGED
@@ -16,6 +16,8 @@ syntax = "proto3";
16
16
 
17
17
  package dataform;
18
18
 
19
+ option go_package = "github.com/dataform-co/dataform/protos/dataform";
20
+
19
21
  import "google/protobuf/empty.proto";
20
22
  import "google/protobuf/struct.proto";
21
23
  import "google/protobuf/timestamp.proto";
@@ -122,6 +124,8 @@ enum JitCompilationTargetType {
122
124
  JIT_COMPILATION_TARGET_TYPE_OPERATION = 2;
123
125
  // Incremental table target.
124
126
  JIT_COMPILATION_TARGET_TYPE_INCREMENTAL_TABLE = 3;
127
+ // Assertion target.
128
+ JIT_COMPILATION_TARGET_TYPE_ASSERTION = 4;
125
129
  }
126
130
 
127
131
  // JiT compilation request.
@@ -148,6 +152,7 @@ message JitCompilationResponse {
148
152
  JitTableResult table = 1;
149
153
  JitOperationResult operation = 2;
150
154
  JitIncrementalTableResult incremental_table = 3;
155
+ JitAssertionResult assertion = 4;
151
156
  }
152
157
  }
153
158
 
@@ -175,3 +180,9 @@ message JitOperationResult {
175
180
  // Sequence of SQL operations.
176
181
  repeated string queries = 1;
177
182
  }
183
+
184
+ // JiT compilation result for assertion actions.
185
+ message JitAssertionResult {
186
+ // SQL Select query that returns rows iff the assertion fails.
187
+ string query = 1;
188
+ }
package/package.json CHANGED
@@ -34,7 +34,7 @@
34
34
  "publishConfig": {
35
35
  "access": "public"
36
36
  },
37
- "version": "3.0.58",
37
+ "version": "3.0.59-alpha.1",
38
38
  "name": "@dataform/core",
39
39
  "description": "Dataform core API.",
40
40
  "main": "bundle.js"