@dataform/core 3.0.58 → 3.0.59-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.d.ts +353 -3
- package/bundle.js +1 -1
- package/configs.proto +20 -0
- package/core.proto +14 -0
- package/db_adapter.proto +2 -0
- package/jit.proto +11 -0
- package/package.json +1 -1
package/configs.proto
CHANGED
|
@@ -81,6 +81,9 @@ message WorkflowSettings {
|
|
|
81
81
|
|
|
82
82
|
// Optional. If set to true, unit tests will be included in the compiled graph.
|
|
83
83
|
bool include_tests_in_compiled_graph = 16;
|
|
84
|
+
|
|
85
|
+
// Optional. Default Managed Spark execution options for Notebook actions.
|
|
86
|
+
ManagedSparkExecutionOptionsConfig default_managed_spark_execution_options = 17;
|
|
84
87
|
}
|
|
85
88
|
|
|
86
89
|
message DefaultIcebergConfig {
|
|
@@ -542,6 +545,9 @@ message ActionConfig {
|
|
|
542
545
|
// If unset, the value from workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.
|
|
543
546
|
// Dataform CLI only (GCP Dataform support pending).
|
|
544
547
|
string reservation = 11;
|
|
548
|
+
|
|
549
|
+
// Metadata for this assertion.
|
|
550
|
+
Metadata metadata = 12;
|
|
545
551
|
}
|
|
546
552
|
|
|
547
553
|
message OperationConfig {
|
|
@@ -652,6 +658,15 @@ message ActionConfig {
|
|
|
652
658
|
|
|
653
659
|
// A notebook runtime field definition could be added here, to allow atomic
|
|
654
660
|
// runtime settings of notebooks.
|
|
661
|
+
|
|
662
|
+
enum ExecutionEngine {
|
|
663
|
+
EXECUTION_ENGINE_UNSPECIFIED = 0;
|
|
664
|
+
COLAB = 1;
|
|
665
|
+
MANAGED_SPARK = 2;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
// Optional. The execution engine to use for this notebook.
|
|
669
|
+
ExecutionEngine execution_engine = 10;
|
|
655
670
|
}
|
|
656
671
|
|
|
657
672
|
message DataPreparationConfig {
|
|
@@ -758,3 +773,8 @@ message NotebookRuntimeOptionsConfig {
|
|
|
758
773
|
}
|
|
759
774
|
|
|
760
775
|
}
|
|
776
|
+
|
|
777
|
+
message ManagedSparkExecutionOptionsConfig {
|
|
778
|
+
// Required: A GCS bucket URI to stage the notebook file before execution.
|
|
779
|
+
string staging_bucket_uri = 1;
|
|
780
|
+
}
|
package/core.proto
CHANGED
|
@@ -52,6 +52,8 @@ message ProjectConfig {
|
|
|
52
52
|
|
|
53
53
|
bool include_tests_in_compiled_graph = 22;
|
|
54
54
|
|
|
55
|
+
ManagedSparkExecutionOptions default_managed_spark_execution_options = 23;
|
|
56
|
+
|
|
55
57
|
reserved 3, 4, 6, 8, 10, 12, 13;
|
|
56
58
|
}
|
|
57
59
|
|
|
@@ -335,6 +337,14 @@ message Notebook {
|
|
|
335
337
|
bool disabled = 6;
|
|
336
338
|
|
|
337
339
|
string notebook_contents = 7;
|
|
340
|
+
|
|
341
|
+
enum ExecutionEngine {
|
|
342
|
+
EXECUTION_ENGINE_UNSPECIFIED = 0;
|
|
343
|
+
COLAB = 1;
|
|
344
|
+
MANAGED_SPARK = 2;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
ExecutionEngine execution_engine = 8;
|
|
338
348
|
}
|
|
339
349
|
|
|
340
350
|
message RepositorySnapshotDestination {
|
|
@@ -353,6 +363,10 @@ message NotebookRuntimeOptions {
|
|
|
353
363
|
}
|
|
354
364
|
}
|
|
355
365
|
|
|
366
|
+
message ManagedSparkExecutionOptions {
|
|
367
|
+
string staging_bucket_uri = 1;
|
|
368
|
+
}
|
|
369
|
+
|
|
356
370
|
// Data Preparation Related entries
|
|
357
371
|
message DataPreparation {
|
|
358
372
|
// Used ONLY as an identifiers. All outputs should be listed in the targets
|
package/db_adapter.proto
CHANGED
package/jit.proto
CHANGED
|
@@ -16,6 +16,8 @@ syntax = "proto3";
|
|
|
16
16
|
|
|
17
17
|
package dataform;
|
|
18
18
|
|
|
19
|
+
option go_package = "github.com/dataform-co/dataform/protos/dataform";
|
|
20
|
+
|
|
19
21
|
import "google/protobuf/empty.proto";
|
|
20
22
|
import "google/protobuf/struct.proto";
|
|
21
23
|
import "google/protobuf/timestamp.proto";
|
|
@@ -122,6 +124,8 @@ enum JitCompilationTargetType {
|
|
|
122
124
|
JIT_COMPILATION_TARGET_TYPE_OPERATION = 2;
|
|
123
125
|
// Incremental table target.
|
|
124
126
|
JIT_COMPILATION_TARGET_TYPE_INCREMENTAL_TABLE = 3;
|
|
127
|
+
// Assertion target.
|
|
128
|
+
JIT_COMPILATION_TARGET_TYPE_ASSERTION = 4;
|
|
125
129
|
}
|
|
126
130
|
|
|
127
131
|
// JiT compilation request.
|
|
@@ -148,6 +152,7 @@ message JitCompilationResponse {
|
|
|
148
152
|
JitTableResult table = 1;
|
|
149
153
|
JitOperationResult operation = 2;
|
|
150
154
|
JitIncrementalTableResult incremental_table = 3;
|
|
155
|
+
JitAssertionResult assertion = 4;
|
|
151
156
|
}
|
|
152
157
|
}
|
|
153
158
|
|
|
@@ -175,3 +180,9 @@ message JitOperationResult {
|
|
|
175
180
|
// Sequence of SQL operations.
|
|
176
181
|
repeated string queries = 1;
|
|
177
182
|
}
|
|
183
|
+
|
|
184
|
+
// JiT compilation result for assertion actions.
|
|
185
|
+
message JitAssertionResult {
|
|
186
|
+
// SQL Select query that returns rows iff the assertion fails.
|
|
187
|
+
string query = 1;
|
|
188
|
+
}
|