@dataform/core 2.8.0 → 3.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.js +17 -1
- package/core.proto +121 -56
- package/package.json +2 -4
package/core.proto
CHANGED
|
@@ -1,4 +1,18 @@
|
|
|
1
|
-
|
|
1
|
+
// Copyright 2023 Google LLC
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
|
|
15
|
+
syntax = "proto3";
|
|
2
16
|
|
|
3
17
|
package dataform;
|
|
4
18
|
|
|
@@ -19,29 +33,11 @@ message ProjectConfig {
|
|
|
19
33
|
|
|
20
34
|
map<string, string> vars = 14;
|
|
21
35
|
|
|
22
|
-
int32 concurrent_query_limit = 13;
|
|
23
|
-
int32 idempotent_action_retries = 8;
|
|
24
|
-
|
|
25
36
|
string database_suffix = 15;
|
|
26
37
|
string schema_suffix = 7;
|
|
27
38
|
string table_prefix = 11;
|
|
28
39
|
|
|
29
|
-
|
|
30
|
-
// Deprecated.
|
|
31
|
-
// Please use 'default_database' instead.
|
|
32
|
-
string gcloud_project_id = 6;
|
|
33
|
-
// This setting no longer has any effect.
|
|
34
|
-
bool use_run_cache = 10;
|
|
35
|
-
|
|
36
|
-
reserved 3, 4, 12;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
message GenerateIndexConfig {
|
|
40
|
-
CompileConfig compile_config = 1;
|
|
41
|
-
repeated string include_paths = 2;
|
|
42
|
-
repeated string definition_paths = 3;
|
|
43
|
-
|
|
44
|
-
reserved 4;
|
|
40
|
+
reserved 3, 4, 6, 8, 10, 12, 13;
|
|
45
41
|
}
|
|
46
42
|
|
|
47
43
|
message CompileConfig {
|
|
@@ -57,10 +53,7 @@ message CompileConfig {
|
|
|
57
53
|
// Override compilation timeout settings.
|
|
58
54
|
int32 timeout_millis = 6;
|
|
59
55
|
|
|
60
|
-
|
|
61
|
-
bool use_main = 9;
|
|
62
|
-
|
|
63
|
-
reserved 2, 4, 5, 7;
|
|
56
|
+
reserved 2, 4, 5, 7, 9;
|
|
64
57
|
}
|
|
65
58
|
|
|
66
59
|
message Target {
|
|
@@ -79,29 +72,6 @@ message BigQueryOptions {
|
|
|
79
72
|
map<string, string> additional_options = 7;
|
|
80
73
|
}
|
|
81
74
|
|
|
82
|
-
message RedshiftOptions {
|
|
83
|
-
string dist_key = 1;
|
|
84
|
-
string dist_style = 2;
|
|
85
|
-
repeated string sort_keys = 3;
|
|
86
|
-
string sort_style = 4;
|
|
87
|
-
|
|
88
|
-
bool bind = 5 [deprecated = true];
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
message SnowflakeOptions {
|
|
92
|
-
bool secure = 1;
|
|
93
|
-
bool transient = 2;
|
|
94
|
-
repeated string cluster_by = 3;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
message SQLDataWarehouseOptions {
|
|
98
|
-
string distribution = 1;
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
message PrestoOptions {
|
|
102
|
-
repeated string partition_by = 3;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
75
|
message GraphErrors {
|
|
106
76
|
repeated CompilationError compilation_errors = 1;
|
|
107
77
|
|
|
@@ -129,7 +99,8 @@ message ColumnDescriptor {
|
|
|
129
99
|
string description = 1;
|
|
130
100
|
|
|
131
101
|
// For nested records, the path might look like e.g. ["record", "column"].
|
|
132
|
-
// For simple columns, this will always contain a single entry e.g.
|
|
102
|
+
// For simple columns, this will always contain a single entry e.g.
|
|
103
|
+
// ["column"].
|
|
133
104
|
repeated string path = 2;
|
|
134
105
|
|
|
135
106
|
string display_name = 3;
|
|
@@ -198,16 +169,12 @@ message Table {
|
|
|
198
169
|
repeated string incremental_post_ops = 29;
|
|
199
170
|
|
|
200
171
|
// Warehouse specific features.
|
|
201
|
-
RedshiftOptions redshift = 21;
|
|
202
172
|
BigQueryOptions bigquery = 22;
|
|
203
|
-
SnowflakeOptions snowflake = 33;
|
|
204
|
-
SQLDataWarehouseOptions sql_data_warehouse = 25;
|
|
205
|
-
PrestoOptions presto = 34;
|
|
206
173
|
|
|
207
174
|
// Generated.
|
|
208
175
|
string file_name = 18;
|
|
209
176
|
|
|
210
|
-
reserved 1,2, 7, 12, 16;
|
|
177
|
+
reserved 1, 2, 7, 12, 16;
|
|
211
178
|
}
|
|
212
179
|
|
|
213
180
|
message Operation {
|
|
@@ -262,7 +229,6 @@ enum ActionHermeticity {
|
|
|
262
229
|
}
|
|
263
230
|
|
|
264
231
|
message Declaration {
|
|
265
|
-
|
|
266
232
|
Target target = 2;
|
|
267
233
|
Target canonical_target = 5;
|
|
268
234
|
|
|
@@ -282,6 +248,14 @@ message Test {
|
|
|
282
248
|
string file_name = 4;
|
|
283
249
|
}
|
|
284
250
|
|
|
251
|
+
message Notebook {
|
|
252
|
+
ActionConfig config = 1;
|
|
253
|
+
|
|
254
|
+
Target target = 2;
|
|
255
|
+
|
|
256
|
+
string notebook_contents = 3;
|
|
257
|
+
}
|
|
258
|
+
|
|
285
259
|
message CompiledGraph {
|
|
286
260
|
ProjectConfig project_config = 4;
|
|
287
261
|
|
|
@@ -289,7 +263,10 @@ message CompiledGraph {
|
|
|
289
263
|
repeated Operation operations = 2;
|
|
290
264
|
repeated Assertion assertions = 3;
|
|
291
265
|
repeated Declaration declarations = 9;
|
|
266
|
+
// TODO(ekrekr): move the Test action outside of the compiled graph, because
|
|
267
|
+
// it is not used at runtime.
|
|
292
268
|
repeated Test tests = 8;
|
|
269
|
+
repeated Notebook notebooks = 12;
|
|
293
270
|
|
|
294
271
|
GraphErrors graph_errors = 7;
|
|
295
272
|
|
|
@@ -320,9 +297,97 @@ message CompileExecutionResponse {
|
|
|
320
297
|
CompiledGraph compiled_graph = 1;
|
|
321
298
|
}
|
|
322
299
|
|
|
323
|
-
// This feature list is added to when making potentilly backwards breaking
|
|
324
|
-
// caller of Dataform Core know whether it supports the
|
|
300
|
+
// This feature list is added to when making potentilly backwards breaking
|
|
301
|
+
// changes. It lets the caller of Dataform Core know whether it supports the
|
|
302
|
+
// change.
|
|
325
303
|
enum SupportedFeatures {
|
|
326
304
|
UNKNOWN_FEATURE = 0;
|
|
327
305
|
ARRAY_BUFFER_IPC = 1;
|
|
328
306
|
}
|
|
307
|
+
|
|
308
|
+
message ActionConfigs {
|
|
309
|
+
repeated ActionConfig actions = 1;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
message ActionConfig {
|
|
313
|
+
// Target is a unique action identifier.
|
|
314
|
+
Target target = 1;
|
|
315
|
+
|
|
316
|
+
// A list of user-defined tags with which the action should be labeled.
|
|
317
|
+
repeated string tags = 2;
|
|
318
|
+
|
|
319
|
+
// Targets of actions that this action is dependent on.
|
|
320
|
+
repeated Target dependency_targets = 3;
|
|
321
|
+
|
|
322
|
+
// Filename to load the action contents from.
|
|
323
|
+
string file_name = 4;
|
|
324
|
+
|
|
325
|
+
// Type specific configs. Must correspond to the declared type.
|
|
326
|
+
oneof action_config {
|
|
327
|
+
TableConfig table = 5;
|
|
328
|
+
ViewConfig view = 6;
|
|
329
|
+
IncrementalTableConfig incremental_table = 7;
|
|
330
|
+
AssertionConfig assertion = 8;
|
|
331
|
+
OperationConfig operation = 9;
|
|
332
|
+
DeclarationConfig declaration = 10;
|
|
333
|
+
NotebookConfig notebook = 11;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
// Configuration options for BigQuery tables.
|
|
338
|
+
message TableConfig {
|
|
339
|
+
// If set to true, this action will not be executed. However, the action may
|
|
340
|
+
// still be depended upon.
|
|
341
|
+
bool disabled = 1;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// Configuration options for BigQuery views.
|
|
345
|
+
message ViewConfig {
|
|
346
|
+
// If set to true, this action will not be executed. However, the action may
|
|
347
|
+
// still be depended upon.
|
|
348
|
+
bool disabled = 1;
|
|
349
|
+
|
|
350
|
+
// If set to true, will make the view materialized.
|
|
351
|
+
// For more information, read the [BigQuery materialized view
|
|
352
|
+
// docs](https://cloud.google.com/bigquery/docs/materialized-views-intro).
|
|
353
|
+
bool materialized = 2;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// Configuration options for BigQuery incremental tables.
|
|
357
|
+
message IncrementalTableConfig {
|
|
358
|
+
// If set to true, this action will not be executed. However, the action may
|
|
359
|
+
// still be depended upon.
|
|
360
|
+
bool disabled = 1;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// Configuration options for BigQuery table assertions.
|
|
364
|
+
message AssertionConfig {
|
|
365
|
+
// If set to true, this action will not be executed. However, the action may
|
|
366
|
+
// still be depended upon.
|
|
367
|
+
bool disabled = 1;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// Configuration options for BigQuery table opertions.
|
|
371
|
+
message OperationConfig {
|
|
372
|
+
// If set to true, this action will not be executed. However, the action may
|
|
373
|
+
// still be depended upon.
|
|
374
|
+
bool disabled = 1;
|
|
375
|
+
|
|
376
|
+
// Declares that this `operations` action creates a dataset which should be
|
|
377
|
+
// referenceable using the `ref` function.
|
|
378
|
+
bool has_output = 2;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
// Configuration options for BigQuery table declarations.
|
|
382
|
+
// TODO(ekrekr): remove this empty message, if explicit action type without a
|
|
383
|
+
// `type` field proves to be impossible / ugly.
|
|
384
|
+
message DeclarationConfig {}
|
|
385
|
+
|
|
386
|
+
// Configuration options for Vertex AI notebooks.
|
|
387
|
+
message NotebookConfig {
|
|
388
|
+
// If set to true, this action will not be executed. However, the action may
|
|
389
|
+
// still be depended upon.
|
|
390
|
+
bool disabled = 1;
|
|
391
|
+
|
|
392
|
+
// TODO(ekrekr): add a notebook runtime field definition.
|
|
393
|
+
}
|
package/package.json
CHANGED
|
@@ -4,7 +4,6 @@
|
|
|
4
4
|
"**/object-path": "^0.11.5",
|
|
5
5
|
"**/http-proxy": "^1.18.1",
|
|
6
6
|
"**/serialize-javascript": "^3.1.0",
|
|
7
|
-
"**/lodash": "^4.17.21",
|
|
8
7
|
"**/ssri": "^7.1.1",
|
|
9
8
|
"**/postcss": "^8.2.10",
|
|
10
9
|
"**/browserslist": "^4.16.5",
|
|
@@ -28,7 +27,7 @@
|
|
|
28
27
|
"**/markdown-it": "^12.3.2"
|
|
29
28
|
},
|
|
30
29
|
"homepage": "https://github.com/dataform-co/dataform",
|
|
31
|
-
"license": "
|
|
30
|
+
"license": "Apache-2.0",
|
|
32
31
|
"keywords": [
|
|
33
32
|
"dataform",
|
|
34
33
|
"etl",
|
|
@@ -37,13 +36,12 @@
|
|
|
37
36
|
"data-modelling",
|
|
38
37
|
"sql",
|
|
39
38
|
"bigquery",
|
|
40
|
-
"redshift",
|
|
41
39
|
"dataops"
|
|
42
40
|
],
|
|
43
41
|
"publishConfig": {
|
|
44
42
|
"access": "public"
|
|
45
43
|
},
|
|
46
|
-
"version": "
|
|
44
|
+
"version": "3.0.0-alpha.0",
|
|
47
45
|
"name": "@dataform/core",
|
|
48
46
|
"description": "Dataform core API.",
|
|
49
47
|
"main": "bundle.js"
|