@dataform/core 3.0.44 → 3.0.46

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/configs.proto CHANGED
@@ -694,6 +694,11 @@ message ActionConfig {
694
694
  }
695
695
  }
696
696
 
697
+ message RepositorySnapshotDestinationConfig {
698
+ // Storage URI to upload the repository snapshot to.
699
+ string repository_snapshot_uri = 1;
700
+ }
701
+
697
702
  message NotebookRuntimeOptionsConfig {
698
703
  oneof output_sink {
699
704
  // Storage bucket to output notebooks to after their execution.
@@ -703,4 +708,13 @@ message NotebookRuntimeOptionsConfig {
703
708
  // Colab runtime template (https://cloud.google.com/colab/docs/runtimes), from
704
709
  // which a runtime is created for notebook executions.
705
710
  string runtime_template_name = 2;
711
+
712
+ // The destination for the snapshot of repository files to be available for
713
+ // read-only access inside a notebook runtime.
714
+ oneof repository_snapshot_storage {
715
+ // Storage URI to upload the snapshot to.
716
+ // For empty URI it defaults to the provided output_bucket.
717
+ RepositorySnapshotDestinationConfig repository_snapshot_destination = 3;
718
+ }
719
+
706
720
  }
package/core.proto CHANGED
@@ -114,6 +114,17 @@ message CompilationError {
114
114
  string stack = 3;
115
115
  }
116
116
 
117
+ // Compilation mode, unspecified is interpreted to AoT.
118
+ enum ActionCompilationMode {
119
+ ACTION_COMPILATION_MODE_UNSPECIFIED = 0;
120
+ // Ahead-of-time compilation (regular Dataform compilation).
121
+ ACTION_COMPILATION_MODE_AOT = 1;
122
+ // Just-in-time compilation.
123
+ // Will only populate jit_code fields in compiled graph,
124
+ // to be re-compiled at execution time.
125
+ ACTION_COMPILATION_MODE_JIT = 2;
126
+ }
127
+
117
128
  message ActionDescriptor {
118
129
  string description = 1;
119
130
  // For Operations, 'columns' may be set iff has_output == true.
@@ -121,6 +132,7 @@ message ActionDescriptor {
121
132
  repeated ColumnDescriptor columns = 2;
122
133
  map<string, string> bigquery_labels = 3;
123
134
  Metadata metadata = 4;
135
+ ActionCompilationMode compilation_mode = 5;
124
136
  }
125
137
 
126
138
  message Metadata {
@@ -202,6 +214,8 @@ message Table {
202
214
  repeated string incremental_pre_ops = 28;
203
215
  repeated string incremental_post_ops = 29;
204
216
 
217
+ string jit_code = 38;
218
+
205
219
  // Warehouse specific features.
206
220
  BigQueryOptions bigquery = 22;
207
221
 
@@ -221,6 +235,7 @@ message Operation {
221
235
  bool disabled = 14;
222
236
 
223
237
  repeated string queries = 6;
238
+ string jit_code = 15;
224
239
  bool has_output = 8;
225
240
  repeated string tags = 9;
226
241
 
@@ -278,6 +293,8 @@ message Test {
278
293
  string test_query = 2;
279
294
  string expected_output_query = 3;
280
295
 
296
+ string jit_code = 10;
297
+
281
298
  // Generated.
282
299
  string file_name = 4;
283
300
 
@@ -307,12 +324,20 @@ message Notebook {
307
324
  string notebook_contents = 7;
308
325
  }
309
326
 
327
+ message RepositorySnapshotDestination {
328
+ string repository_snapshot_uri = 1;
329
+ }
330
+
310
331
  message NotebookRuntimeOptions {
311
332
  oneof output_sink {
312
333
  string output_bucket = 1;
313
334
  }
314
335
 
315
336
  string runtime_template_name = 2;
337
+
338
+ oneof repository_snapshot_storage {
339
+ RepositorySnapshotDestination repository_snapshot_destination = 3;
340
+ }
316
341
  }
317
342
 
318
343
  // Data Preparation Related entries
@@ -393,6 +418,8 @@ message CompiledGraph {
393
418
 
394
419
  repeated Target targets = 11;
395
420
 
421
+ google.protobuf.Struct jit_data = 15;
422
+
396
423
  reserved 5, 6;
397
424
  }
398
425
 
@@ -0,0 +1,92 @@
1
+ // Copyright 2023 Google LLC
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+
15
+ syntax = "proto3";
16
+
17
+ package dataform;
18
+
19
+ import "protos/core.proto";
20
+
21
+ // Common structures for Evaluation, execution and compilation.
22
+
23
+ message Field {
24
+ enum Primitive {
25
+ UNKNOWN = 0;
26
+ INTEGER = 1;
27
+ FLOAT = 2;
28
+ NUMERIC = 5;
29
+ BOOLEAN = 3;
30
+ STRING = 4;
31
+ DATE = 6;
32
+ DATETIME = 7;
33
+ TIMESTAMP = 8;
34
+ TIME = 9;
35
+ BYTES = 10;
36
+ ANY = 11;
37
+ GEOGRAPHY = 12;
38
+ }
39
+
40
+ enum Flag {
41
+ UNKNOWN_FLAG = 0;
42
+ REPEATED = 1;
43
+ }
44
+
45
+ string name = 1;
46
+
47
+ repeated Flag flags = 6;
48
+
49
+ oneof type {
50
+ Primitive primitive = 7;
51
+ Fields struct = 3;
52
+ }
53
+
54
+ string description = 5;
55
+
56
+ reserved 2, 4;
57
+ }
58
+
59
+ message Fields {
60
+ repeated Field fields = 1;
61
+ }
62
+
63
+ message TableMetadata {
64
+ Target target = 1;
65
+ enum Type {
66
+ UNKNOWN = 0;
67
+ TABLE = 1;
68
+ VIEW = 2;
69
+ }
70
+ Type type = 6;
71
+
72
+ repeated Field fields = 3;
73
+ string description = 5;
74
+ map<string, string> labels = 7;
75
+ int64 last_updated_millis = 4;
76
+
77
+ message BigQuery {
78
+ bool has_streaming_buffer = 1;
79
+ }
80
+ BigQuery bigquery = 8;
81
+
82
+ reserved 2;
83
+ }
84
+
85
+ message ExecutionMetadata {
86
+ message BigqueryMetadata {
87
+ string job_id = 1;
88
+ int64 total_bytes_processed = 2;
89
+ int64 total_bytes_billed = 3;
90
+ }
91
+ BigqueryMetadata bigquery = 1;
92
+ }
package/jit.proto ADDED
@@ -0,0 +1,157 @@
1
+ // Copyright 2023 Google LLC
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+
15
+ syntax = "proto3";
16
+
17
+ package dataform;
18
+
19
+ import "google/protobuf/empty.proto";
20
+ import "google/protobuf/struct.proto";
21
+ import "protos/core.proto";
22
+ import "protos/db_adapter.proto";
23
+
24
+ // Database adapter, available in JiT compilation context.
25
+ service DbAdapter {
26
+ // Executes a query.
27
+ rpc Execute(ExecuteRequest) returns (ExecuteResponse) {}
28
+
29
+ // Lists table in schema.
30
+ rpc ListTables(ListTablesRequest) returns (ListTablesResponse) {}
31
+
32
+ // Gets table for target.
33
+ rpc GetTable(GetTableRequest) returns (TableMetadata) {}
34
+
35
+ // Drops table of specified target.
36
+ rpc DeleteTable(DeleteTableRequest) returns (google.protobuf.Empty) {}
37
+ }
38
+
39
+ // Request to execute synchronous SQL query.
40
+ message ExecuteRequest {
41
+ // SQL statement query.
42
+ string statement = 1;
43
+ // Max rows to return in the job.
44
+ int64 row_limit = 2;
45
+ // Max bytes to process.
46
+ int64 byte_limit = 3;
47
+ // Whether or not fetch result rows.
48
+ bool fetch_results = 4;
49
+
50
+ BigQueryExecuteOptions big_query_options = 5;
51
+ }
52
+
53
+ message BigQueryExecuteOptions {
54
+ // Priority - interactive if true or batch if false (default).
55
+ bool interactive = 1;
56
+ // See https://docs.cloud.google.com/bigquery/docs/reference/legacy-sql.
57
+ bool use_legacy_sql = 2;
58
+ // Labels to add to the job.
59
+ map<string, string> labels = 3;
60
+ // Location to execute the job at.
61
+ string location = 4;
62
+ // Job prefix to add.
63
+ string job_prefix = 5;
64
+ // Is dry run job.
65
+ bool dry_run = 6;
66
+ }
67
+
68
+ // Synchronous execution response result.
69
+ message ExecuteResponse {
70
+ // Job metadata.
71
+ ExecutionMetadata execution_metadata = 1;
72
+ // Rows. For BigQuery, see
73
+ // https://docs.cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults.
74
+ repeated google.protobuf.Struct rows = 2;
75
+ repeated string errors = 3;
76
+ }
77
+
78
+ // Request to list tables in schema.
79
+ message ListTablesRequest {
80
+ string database = 1;
81
+ string schema = 2;
82
+ }
83
+
84
+ // Tables metadata in schema.
85
+ message ListTablesResponse {
86
+ repeated TableMetadata tables = 1;
87
+ }
88
+
89
+ // Request to get table metadata for target.
90
+ message GetTableRequest {
91
+ Target target = 1;
92
+ }
93
+
94
+ // Request to drop the table for target.
95
+ message DeleteTableRequest {
96
+ Target target = 1;
97
+ }
98
+
99
+ // JiT compilation target type.
100
+ enum JitCompilationTargetType {
101
+ // Unspecified - will result in error.
102
+ JIT_COMPILATION_TARGET_TYPE_UNSPECIFIED = 0;
103
+ // Table/view target.
104
+ JIT_COMPILATION_TARGET_TYPE_TABLE = 1;
105
+ // Operation target.
106
+ JIT_COMPILATION_TARGET_TYPE_OPERATION = 2;
107
+ // Incremental table target.
108
+ JIT_COMPILATION_TARGET_TYPE_INCREMENTAL_TABLE = 3;
109
+ }
110
+
111
+ // JiT compilation request.
112
+ message JitCompilationRequest {
113
+ // Canonical target being compiled.
114
+ Target target = 1;
115
+ // Canonical target dependencies.
116
+ repeated Target dependencies = 2;
117
+ // JiT code to compile.
118
+ string jit_code = 3;
119
+ // JiT data, exposed to code in context.
120
+ google.protobuf.Struct jit_data = 4;
121
+ // JiT compilation target type.
122
+ JitCompilationTargetType compilation_target_type = 5;
123
+ }
124
+
125
+ // JiT compilation response.
126
+ message JitCompilationResponse {
127
+ oneof response {
128
+ JitTableResult table = 1;
129
+ JitOperationResult operation = 2;
130
+ JitIncrementalTableResult incremental_table = 3;
131
+ }
132
+ }
133
+
134
+ // JiT compilation result for table actions (including views).
135
+ // Fields match the Table message.
136
+ message JitTableResult {
137
+ // SQL Select query of the table.
138
+ string query = 1;
139
+ // Optional pre-operation statements.
140
+ repeated string pre_ops = 3;
141
+ // Optional post-operation statements.
142
+ repeated string post_ops = 4;
143
+ }
144
+
145
+ // JiT compilation result for incremental table actions.
146
+ message JitIncrementalTableResult {
147
+ // Fields match the Table message.
148
+ JitTableResult regular = 1;
149
+ // Fields match incremental_ fields in the Table message.
150
+ JitTableResult incremental = 2;
151
+ }
152
+
153
+ // JiT compilation result for operation actions.
154
+ message JitOperationResult {
155
+ // Sequence of SQL operations.
156
+ repeated string queries = 1;
157
+ }
package/package.json CHANGED
@@ -34,7 +34,7 @@
34
34
  "publishConfig": {
35
35
  "access": "public"
36
36
  },
37
- "version": "3.0.44",
37
+ "version": "3.0.46",
38
38
  "name": "@dataform/core",
39
39
  "description": "Dataform core API.",
40
40
  "main": "bundle.js"