@dataform/core 3.0.43 → 3.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.d.ts +2644 -1134
- package/bundle.js +1 -1
- package/configs.proto +14 -0
- package/core.proto +27 -0
- package/db_adapter.proto +92 -0
- package/jit.proto +122 -0
- package/package.json +1 -1
package/configs.proto
CHANGED
|
@@ -694,6 +694,11 @@ message ActionConfig {
|
|
|
694
694
|
}
|
|
695
695
|
}
|
|
696
696
|
|
|
697
|
+
message RepositorySnapshotDestinationConfig {
|
|
698
|
+
// Storage URI to upload the repository snapshot to.
|
|
699
|
+
string repository_snapshot_uri = 1;
|
|
700
|
+
}
|
|
701
|
+
|
|
697
702
|
message NotebookRuntimeOptionsConfig {
|
|
698
703
|
oneof output_sink {
|
|
699
704
|
// Storage bucket to output notebooks to after their execution.
|
|
@@ -703,4 +708,13 @@ message NotebookRuntimeOptionsConfig {
|
|
|
703
708
|
// Colab runtime template (https://cloud.google.com/colab/docs/runtimes), from
|
|
704
709
|
// which a runtime is created for notebook executions.
|
|
705
710
|
string runtime_template_name = 2;
|
|
711
|
+
|
|
712
|
+
// The destination for the snapshot of repository files to be available for
|
|
713
|
+
// read-only access inside a notebook runtime.
|
|
714
|
+
oneof repository_snapshot_storage {
|
|
715
|
+
// Storage URI to upload the snapshot to.
|
|
716
|
+
// For empty URI it defaults to the provided output_bucket.
|
|
717
|
+
RepositorySnapshotDestinationConfig repository_snapshot_destination = 3;
|
|
718
|
+
}
|
|
719
|
+
|
|
706
720
|
}
|
package/core.proto
CHANGED
|
@@ -114,6 +114,17 @@ message CompilationError {
|
|
|
114
114
|
string stack = 3;
|
|
115
115
|
}
|
|
116
116
|
|
|
117
|
+
// Compilation mode, unspecified is interpreted to AoT.
|
|
118
|
+
enum ActionCompilationMode {
|
|
119
|
+
ACTION_COMPILATION_MODE_UNSPECIFIED = 0;
|
|
120
|
+
// Ahead-of-time compilation (regular Dataform compilation).
|
|
121
|
+
ACTION_COMPILATION_MODE_AOT = 1;
|
|
122
|
+
// Just-in-time compilation.
|
|
123
|
+
// Will only populate jit_code fields in compiled graph,
|
|
124
|
+
// to be re-compiled at execution time.
|
|
125
|
+
ACTION_COMPILATION_MODE_JIT = 2;
|
|
126
|
+
}
|
|
127
|
+
|
|
117
128
|
message ActionDescriptor {
|
|
118
129
|
string description = 1;
|
|
119
130
|
// For Operations, 'columns' may be set iff has_output == true.
|
|
@@ -121,6 +132,7 @@ message ActionDescriptor {
|
|
|
121
132
|
repeated ColumnDescriptor columns = 2;
|
|
122
133
|
map<string, string> bigquery_labels = 3;
|
|
123
134
|
Metadata metadata = 4;
|
|
135
|
+
ActionCompilationMode compilation_mode = 5;
|
|
124
136
|
}
|
|
125
137
|
|
|
126
138
|
message Metadata {
|
|
@@ -202,6 +214,8 @@ message Table {
|
|
|
202
214
|
repeated string incremental_pre_ops = 28;
|
|
203
215
|
repeated string incremental_post_ops = 29;
|
|
204
216
|
|
|
217
|
+
string jit_code = 38;
|
|
218
|
+
|
|
205
219
|
// Warehouse specific features.
|
|
206
220
|
BigQueryOptions bigquery = 22;
|
|
207
221
|
|
|
@@ -221,6 +235,7 @@ message Operation {
|
|
|
221
235
|
bool disabled = 14;
|
|
222
236
|
|
|
223
237
|
repeated string queries = 6;
|
|
238
|
+
string jit_code = 15;
|
|
224
239
|
bool has_output = 8;
|
|
225
240
|
repeated string tags = 9;
|
|
226
241
|
|
|
@@ -278,6 +293,8 @@ message Test {
|
|
|
278
293
|
string test_query = 2;
|
|
279
294
|
string expected_output_query = 3;
|
|
280
295
|
|
|
296
|
+
string jit_code = 10;
|
|
297
|
+
|
|
281
298
|
// Generated.
|
|
282
299
|
string file_name = 4;
|
|
283
300
|
|
|
@@ -307,12 +324,20 @@ message Notebook {
|
|
|
307
324
|
string notebook_contents = 7;
|
|
308
325
|
}
|
|
309
326
|
|
|
327
|
+
message RepositorySnapshotDestination {
|
|
328
|
+
string repository_snapshot_uri = 1;
|
|
329
|
+
}
|
|
330
|
+
|
|
310
331
|
message NotebookRuntimeOptions {
|
|
311
332
|
oneof output_sink {
|
|
312
333
|
string output_bucket = 1;
|
|
313
334
|
}
|
|
314
335
|
|
|
315
336
|
string runtime_template_name = 2;
|
|
337
|
+
|
|
338
|
+
oneof repository_snapshot_storage {
|
|
339
|
+
RepositorySnapshotDestination repository_snapshot_destination = 3;
|
|
340
|
+
}
|
|
316
341
|
}
|
|
317
342
|
|
|
318
343
|
// Data Preparation Related entries
|
|
@@ -393,6 +418,8 @@ message CompiledGraph {
|
|
|
393
418
|
|
|
394
419
|
repeated Target targets = 11;
|
|
395
420
|
|
|
421
|
+
google.protobuf.Struct jit_data = 15;
|
|
422
|
+
|
|
396
423
|
reserved 5, 6;
|
|
397
424
|
}
|
|
398
425
|
|
package/db_adapter.proto
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// Copyright 2023 Google LLC
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
|
|
15
|
+
syntax = "proto3";
|
|
16
|
+
|
|
17
|
+
package dataform;
|
|
18
|
+
|
|
19
|
+
import "protos/core.proto";
|
|
20
|
+
|
|
21
|
+
// Common structures for Evaluation, execution and compilation.
|
|
22
|
+
|
|
23
|
+
message Field {
|
|
24
|
+
enum Primitive {
|
|
25
|
+
UNKNOWN = 0;
|
|
26
|
+
INTEGER = 1;
|
|
27
|
+
FLOAT = 2;
|
|
28
|
+
NUMERIC = 5;
|
|
29
|
+
BOOLEAN = 3;
|
|
30
|
+
STRING = 4;
|
|
31
|
+
DATE = 6;
|
|
32
|
+
DATETIME = 7;
|
|
33
|
+
TIMESTAMP = 8;
|
|
34
|
+
TIME = 9;
|
|
35
|
+
BYTES = 10;
|
|
36
|
+
ANY = 11;
|
|
37
|
+
GEOGRAPHY = 12;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
enum Flag {
|
|
41
|
+
UNKNOWN_FLAG = 0;
|
|
42
|
+
REPEATED = 1;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
string name = 1;
|
|
46
|
+
|
|
47
|
+
repeated Flag flags = 6;
|
|
48
|
+
|
|
49
|
+
oneof type {
|
|
50
|
+
Primitive primitive = 7;
|
|
51
|
+
Fields struct = 3;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
string description = 5;
|
|
55
|
+
|
|
56
|
+
reserved 2, 4;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
message Fields {
|
|
60
|
+
repeated Field fields = 1;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
message TableMetadata {
|
|
64
|
+
Target target = 1;
|
|
65
|
+
enum Type {
|
|
66
|
+
UNKNOWN = 0;
|
|
67
|
+
TABLE = 1;
|
|
68
|
+
VIEW = 2;
|
|
69
|
+
}
|
|
70
|
+
Type type = 6;
|
|
71
|
+
|
|
72
|
+
repeated Field fields = 3;
|
|
73
|
+
string description = 5;
|
|
74
|
+
map<string, string> labels = 7;
|
|
75
|
+
int64 last_updated_millis = 4;
|
|
76
|
+
|
|
77
|
+
message BigQuery {
|
|
78
|
+
bool has_streaming_buffer = 1;
|
|
79
|
+
}
|
|
80
|
+
BigQuery bigquery = 8;
|
|
81
|
+
|
|
82
|
+
reserved 2;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
message ExecutionMetadata {
|
|
86
|
+
message BigqueryMetadata {
|
|
87
|
+
string job_id = 1;
|
|
88
|
+
int64 total_bytes_processed = 2;
|
|
89
|
+
int64 total_bytes_billed = 3;
|
|
90
|
+
}
|
|
91
|
+
BigqueryMetadata bigquery = 1;
|
|
92
|
+
}
|
package/jit.proto
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
// Copyright 2023 Google LLC
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
|
|
15
|
+
syntax = "proto3";
|
|
16
|
+
|
|
17
|
+
package dataform;
|
|
18
|
+
|
|
19
|
+
import "google/protobuf/empty.proto";
|
|
20
|
+
import "google/protobuf/struct.proto";
|
|
21
|
+
import "protos/core.proto";
|
|
22
|
+
import "protos/db_adapter.proto";
|
|
23
|
+
|
|
24
|
+
// Database adapter, available in JiT compilation context.
|
|
25
|
+
service DbAdapter {
|
|
26
|
+
// Executes a query.
|
|
27
|
+
rpc Execute(ExecuteRequest) returns (ExecuteResponse) {}
|
|
28
|
+
|
|
29
|
+
// Lists table in schema.
|
|
30
|
+
rpc ListTables(ListTablesRequest) returns (ListTablesResponse) {}
|
|
31
|
+
|
|
32
|
+
// Gets table for target.
|
|
33
|
+
rpc GetTable(GetTableRequest) returns (TableMetadata) {}
|
|
34
|
+
|
|
35
|
+
// Drops table of specified target.
|
|
36
|
+
rpc DeleteTable(DeleteTableRequest) returns (google.protobuf.Empty) {}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Request to execute synchronous SQL query.
|
|
40
|
+
message ExecuteRequest {
|
|
41
|
+
// SQL statement query.
|
|
42
|
+
string statement = 1;
|
|
43
|
+
// Max rows to return in the job.
|
|
44
|
+
int64 row_limit = 2;
|
|
45
|
+
// Max bytes to process.
|
|
46
|
+
int64 byte_limit = 3;
|
|
47
|
+
// Whether or not fetch result rows.
|
|
48
|
+
bool fetch_results = 4;
|
|
49
|
+
|
|
50
|
+
BigQueryExecuteOptions big_query_options = 5;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
message BigQueryExecuteOptions {
|
|
54
|
+
// Priority - interactive if true or batch if false (default).
|
|
55
|
+
bool interactive = 1;
|
|
56
|
+
// See https://docs.cloud.google.com/bigquery/docs/reference/legacy-sql.
|
|
57
|
+
bool use_legacy_sql = 2;
|
|
58
|
+
// Labels to add to the job.
|
|
59
|
+
map<string, string> labels = 3;
|
|
60
|
+
// Location to execute the job at.
|
|
61
|
+
string location = 4;
|
|
62
|
+
// Job prefix to add.
|
|
63
|
+
string job_prefix = 5;
|
|
64
|
+
// Is dry run job.
|
|
65
|
+
bool dry_run = 6;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Synchronous execution response result.
|
|
69
|
+
message ExecuteResponse {
|
|
70
|
+
// Job metadata.
|
|
71
|
+
ExecutionMetadata execution_metadata = 1;
|
|
72
|
+
// Rows. For BigQuery, see
|
|
73
|
+
// https://docs.cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults.
|
|
74
|
+
repeated google.protobuf.Struct rows = 2;
|
|
75
|
+
repeated string errors = 3;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Request to list tables in schema.
|
|
79
|
+
message ListTablesRequest {
|
|
80
|
+
string database = 1;
|
|
81
|
+
string schema = 2;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Tables metadata in schema.
|
|
85
|
+
message ListTablesResponse {
|
|
86
|
+
repeated TableMetadata tables = 1;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Request to get table metadata for target.
|
|
90
|
+
message GetTableRequest {
|
|
91
|
+
Target target = 1;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Request to drop the table for target.
|
|
95
|
+
message DeleteTableRequest {
|
|
96
|
+
Target target = 1;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// JiT compilation result for table actions (including views).
|
|
100
|
+
// Fields match the Table message.
|
|
101
|
+
message JitTableResult {
|
|
102
|
+
// SQL Select query of the table.
|
|
103
|
+
string query = 1;
|
|
104
|
+
// Optional pre-operation statements.
|
|
105
|
+
repeated string pre_ops = 3;
|
|
106
|
+
// Optional post-operation statements.
|
|
107
|
+
repeated string post_ops = 4;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// JiT compilation result for incremental table actions.
|
|
111
|
+
message JitIncrementalTableResult {
|
|
112
|
+
// Fields match the Table message.
|
|
113
|
+
JitTableResult regular = 1;
|
|
114
|
+
// Fields match incremental_ fields in the Table message.
|
|
115
|
+
JitTableResult incremental = 2;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// JiT compilation result for operation actions.
|
|
119
|
+
message JitOperationResult {
|
|
120
|
+
// Sequence of SQL operations.
|
|
121
|
+
repeated string queries = 1;
|
|
122
|
+
}
|