@dataform/core 3.0.57 → 3.0.59-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.d.ts +362 -3
- package/bundle.js +1 -1
- package/configs.proto +20 -0
- package/core.proto +16 -0
- package/db_adapter.proto +2 -0
- package/jit.proto +11 -0
- package/package.json +1 -1
package/configs.proto
CHANGED
|
@@ -81,6 +81,9 @@ message WorkflowSettings {
|
|
|
81
81
|
|
|
82
82
|
// Optional. If set to true, unit tests will be included in the compiled graph.
|
|
83
83
|
bool include_tests_in_compiled_graph = 16;
|
|
84
|
+
|
|
85
|
+
// Optional. Default Managed Spark execution options for Notebook actions.
|
|
86
|
+
ManagedSparkExecutionOptionsConfig default_managed_spark_execution_options = 17;
|
|
84
87
|
}
|
|
85
88
|
|
|
86
89
|
message DefaultIcebergConfig {
|
|
@@ -542,6 +545,9 @@ message ActionConfig {
|
|
|
542
545
|
// If unset, the value from workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.
|
|
543
546
|
// Dataform CLI only (GCP Dataform support pending).
|
|
544
547
|
string reservation = 11;
|
|
548
|
+
|
|
549
|
+
// Metadata for this assertion.
|
|
550
|
+
Metadata metadata = 12;
|
|
545
551
|
}
|
|
546
552
|
|
|
547
553
|
message OperationConfig {
|
|
@@ -652,6 +658,15 @@ message ActionConfig {
|
|
|
652
658
|
|
|
653
659
|
// A notebook runtime field definition could be added here, to allow atomic
|
|
654
660
|
// runtime settings of notebooks.
|
|
661
|
+
|
|
662
|
+
enum ExecutionEngine {
|
|
663
|
+
EXECUTION_ENGINE_UNSPECIFIED = 0;
|
|
664
|
+
COLAB = 1;
|
|
665
|
+
MANAGED_SPARK = 2;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
// Optional. The execution engine to use for this notebook.
|
|
669
|
+
ExecutionEngine execution_engine = 10;
|
|
655
670
|
}
|
|
656
671
|
|
|
657
672
|
message DataPreparationConfig {
|
|
@@ -758,3 +773,8 @@ message NotebookRuntimeOptionsConfig {
|
|
|
758
773
|
}
|
|
759
774
|
|
|
760
775
|
}
|
|
776
|
+
|
|
777
|
+
message ManagedSparkExecutionOptionsConfig {
|
|
778
|
+
// Required: A GCS bucket URI to stage the notebook file before execution.
|
|
779
|
+
string staging_bucket_uri = 1;
|
|
780
|
+
}
|
package/core.proto
CHANGED
|
@@ -52,6 +52,8 @@ message ProjectConfig {
|
|
|
52
52
|
|
|
53
53
|
bool include_tests_in_compiled_graph = 22;
|
|
54
54
|
|
|
55
|
+
ManagedSparkExecutionOptions default_managed_spark_execution_options = 23;
|
|
56
|
+
|
|
55
57
|
reserved 3, 4, 6, 8, 10, 12, 13;
|
|
56
58
|
}
|
|
57
59
|
|
|
@@ -274,6 +276,8 @@ message Assertion {
|
|
|
274
276
|
// Only present for auto assertions.
|
|
275
277
|
Target parent_action = 15;
|
|
276
278
|
|
|
279
|
+
string jit_code = 16;
|
|
280
|
+
|
|
277
281
|
// Generated.
|
|
278
282
|
string file_name = 7;
|
|
279
283
|
|
|
@@ -333,6 +337,14 @@ message Notebook {
|
|
|
333
337
|
bool disabled = 6;
|
|
334
338
|
|
|
335
339
|
string notebook_contents = 7;
|
|
340
|
+
|
|
341
|
+
enum ExecutionEngine {
|
|
342
|
+
EXECUTION_ENGINE_UNSPECIFIED = 0;
|
|
343
|
+
COLAB = 1;
|
|
344
|
+
MANAGED_SPARK = 2;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
ExecutionEngine execution_engine = 8;
|
|
336
348
|
}
|
|
337
349
|
|
|
338
350
|
message RepositorySnapshotDestination {
|
|
@@ -351,6 +363,10 @@ message NotebookRuntimeOptions {
|
|
|
351
363
|
}
|
|
352
364
|
}
|
|
353
365
|
|
|
366
|
+
message ManagedSparkExecutionOptions {
|
|
367
|
+
string staging_bucket_uri = 1;
|
|
368
|
+
}
|
|
369
|
+
|
|
354
370
|
// Data Preparation Related entries
|
|
355
371
|
message DataPreparation {
|
|
356
372
|
// Used ONLY as an identifiers. All outputs should be listed in the targets
|
package/db_adapter.proto
CHANGED
package/jit.proto
CHANGED
|
@@ -16,6 +16,8 @@ syntax = "proto3";
|
|
|
16
16
|
|
|
17
17
|
package dataform;
|
|
18
18
|
|
|
19
|
+
option go_package = "github.com/dataform-co/dataform/protos/dataform";
|
|
20
|
+
|
|
19
21
|
import "google/protobuf/empty.proto";
|
|
20
22
|
import "google/protobuf/struct.proto";
|
|
21
23
|
import "google/protobuf/timestamp.proto";
|
|
@@ -122,6 +124,8 @@ enum JitCompilationTargetType {
|
|
|
122
124
|
JIT_COMPILATION_TARGET_TYPE_OPERATION = 2;
|
|
123
125
|
// Incremental table target.
|
|
124
126
|
JIT_COMPILATION_TARGET_TYPE_INCREMENTAL_TABLE = 3;
|
|
127
|
+
// Assertion target.
|
|
128
|
+
JIT_COMPILATION_TARGET_TYPE_ASSERTION = 4;
|
|
125
129
|
}
|
|
126
130
|
|
|
127
131
|
// JiT compilation request.
|
|
@@ -148,6 +152,7 @@ message JitCompilationResponse {
|
|
|
148
152
|
JitTableResult table = 1;
|
|
149
153
|
JitOperationResult operation = 2;
|
|
150
154
|
JitIncrementalTableResult incremental_table = 3;
|
|
155
|
+
JitAssertionResult assertion = 4;
|
|
151
156
|
}
|
|
152
157
|
}
|
|
153
158
|
|
|
@@ -175,3 +180,9 @@ message JitOperationResult {
|
|
|
175
180
|
// Sequence of SQL operations.
|
|
176
181
|
repeated string queries = 1;
|
|
177
182
|
}
|
|
183
|
+
|
|
184
|
+
// JiT compilation result for assertion actions.
|
|
185
|
+
message JitAssertionResult {
|
|
186
|
+
// SQL Select query that returns rows iff the assertion fails.
|
|
187
|
+
string query = 1;
|
|
188
|
+
}
|