@dataform/core 2.8.0-beta.1 → 3.0.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/bundle.js +17 -1
  2. package/core.proto +121 -56
  3. package/package.json +2 -4
package/core.proto CHANGED
@@ -1,4 +1,18 @@
1
- syntax="proto3";
1
+ // Copyright 2023 Google LLC
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+
15
+ syntax = "proto3";
2
16
 
3
17
  package dataform;
4
18
 
@@ -19,29 +33,11 @@ message ProjectConfig {
19
33
 
20
34
  map<string, string> vars = 14;
21
35
 
22
- int32 concurrent_query_limit = 13;
23
- int32 idempotent_action_retries = 8;
24
-
25
36
  string database_suffix = 15;
26
37
  string schema_suffix = 7;
27
38
  string table_prefix = 11;
28
39
 
29
-
30
- // Deprecated.
31
- // Please use 'default_database' instead.
32
- string gcloud_project_id = 6;
33
- // This setting no longer has any effect.
34
- bool use_run_cache = 10;
35
-
36
- reserved 3, 4, 12;
37
- }
38
-
39
- message GenerateIndexConfig {
40
- CompileConfig compile_config = 1;
41
- repeated string include_paths = 2;
42
- repeated string definition_paths = 3;
43
-
44
- reserved 4;
40
+ reserved 3, 4, 6, 8, 10, 12, 13;
45
41
  }
46
42
 
47
43
  message CompileConfig {
@@ -57,10 +53,7 @@ message CompileConfig {
57
53
  // Override compilation timeout settings.
58
54
  int32 timeout_millis = 6;
59
55
 
60
- // Whether to use the new main function instead of gen index.
61
- bool use_main = 9;
62
-
63
- reserved 2, 4, 5, 7;
56
+ reserved 2, 4, 5, 7, 9;
64
57
  }
65
58
 
66
59
  message Target {
@@ -79,29 +72,6 @@ message BigQueryOptions {
79
72
  map<string, string> additional_options = 7;
80
73
  }
81
74
 
82
- message RedshiftOptions {
83
- string dist_key = 1;
84
- string dist_style = 2;
85
- repeated string sort_keys = 3;
86
- string sort_style = 4;
87
-
88
- bool bind = 5 [deprecated = true];
89
- }
90
-
91
- message SnowflakeOptions {
92
- bool secure = 1;
93
- bool transient = 2;
94
- repeated string cluster_by = 3;
95
- }
96
-
97
- message SQLDataWarehouseOptions {
98
- string distribution = 1;
99
- }
100
-
101
- message PrestoOptions {
102
- repeated string partition_by = 3;
103
- }
104
-
105
75
  message GraphErrors {
106
76
  repeated CompilationError compilation_errors = 1;
107
77
 
@@ -129,7 +99,8 @@ message ColumnDescriptor {
129
99
  string description = 1;
130
100
 
131
101
  // For nested records, the path might look like e.g. ["record", "column"].
132
- // For simple columns, this will always contain a single entry e.g. ["column"].
102
+ // For simple columns, this will always contain a single entry e.g.
103
+ // ["column"].
133
104
  repeated string path = 2;
134
105
 
135
106
  string display_name = 3;
@@ -198,16 +169,12 @@ message Table {
198
169
  repeated string incremental_post_ops = 29;
199
170
 
200
171
  // Warehouse specific features.
201
- RedshiftOptions redshift = 21;
202
172
  BigQueryOptions bigquery = 22;
203
- SnowflakeOptions snowflake = 33;
204
- SQLDataWarehouseOptions sql_data_warehouse = 25;
205
- PrestoOptions presto = 34;
206
173
 
207
174
  // Generated.
208
175
  string file_name = 18;
209
176
 
210
- reserved 1,2, 7, 12, 16;
177
+ reserved 1, 2, 7, 12, 16;
211
178
  }
212
179
 
213
180
  message Operation {
@@ -262,7 +229,6 @@ enum ActionHermeticity {
262
229
  }
263
230
 
264
231
  message Declaration {
265
-
266
232
  Target target = 2;
267
233
  Target canonical_target = 5;
268
234
 
@@ -282,6 +248,14 @@ message Test {
282
248
  string file_name = 4;
283
249
  }
284
250
 
251
+ message Notebook {
252
+ ActionConfig config = 1;
253
+
254
+ Target target = 2;
255
+
256
+ string notebook_contents = 3;
257
+ }
258
+
285
259
  message CompiledGraph {
286
260
  ProjectConfig project_config = 4;
287
261
 
@@ -289,7 +263,10 @@ message CompiledGraph {
289
263
  repeated Operation operations = 2;
290
264
  repeated Assertion assertions = 3;
291
265
  repeated Declaration declarations = 9;
266
+ // TODO(ekrekr): move the Test action outside of the compiled graph, because
267
+ // it is not used at runtime.
292
268
  repeated Test tests = 8;
269
+ repeated Notebook notebooks = 12;
293
270
 
294
271
  GraphErrors graph_errors = 7;
295
272
 
@@ -320,9 +297,97 @@ message CompileExecutionResponse {
320
297
  CompiledGraph compiled_graph = 1;
321
298
  }
322
299
 
323
- // This feature list is added to when making potentilly backwards breaking changes. It lets the
324
- // caller of Dataform Core know whether it supports the change.
300
+ // This feature list is added to when making potentilly backwards breaking
301
+ // changes. It lets the caller of Dataform Core know whether it supports the
302
+ // change.
325
303
  enum SupportedFeatures {
326
304
  UNKNOWN_FEATURE = 0;
327
305
  ARRAY_BUFFER_IPC = 1;
328
306
  }
307
+
308
+ message ActionConfigs {
309
+ repeated ActionConfig actions = 1;
310
+ }
311
+
312
+ message ActionConfig {
313
+ // Target is a unique action identifier.
314
+ Target target = 1;
315
+
316
+ // A list of user-defined tags with which the action should be labeled.
317
+ repeated string tags = 2;
318
+
319
+ // Targets of actions that this action is dependent on.
320
+ repeated Target dependency_targets = 3;
321
+
322
+ // Filename to load the action contents from.
323
+ string file_name = 4;
324
+
325
+ // Type specific configs. Must correspond to the declared type.
326
+ oneof action_config {
327
+ TableConfig table = 5;
328
+ ViewConfig view = 6;
329
+ IncrementalTableConfig incremental_table = 7;
330
+ AssertionConfig assertion = 8;
331
+ OperationConfig operation = 9;
332
+ DeclarationConfig declaration = 10;
333
+ NotebookConfig notebook = 11;
334
+ }
335
+ }
336
+
337
+ // Configuration options for BigQuery tables.
338
+ message TableConfig {
339
+ // If set to true, this action will not be executed. However, the action may
340
+ // still be depended upon.
341
+ bool disabled = 1;
342
+ }
343
+
344
+ // Configuration options for BigQuery views.
345
+ message ViewConfig {
346
+ // If set to true, this action will not be executed. However, the action may
347
+ // still be depended upon.
348
+ bool disabled = 1;
349
+
350
+ // If set to true, will make the view materialized.
351
+ // For more information, read the [BigQuery materialized view
352
+ // docs](https://cloud.google.com/bigquery/docs/materialized-views-intro).
353
+ bool materialized = 2;
354
+ }
355
+
356
+ // Configuration options for BigQuery incremental tables.
357
+ message IncrementalTableConfig {
358
+ // If set to true, this action will not be executed. However, the action may
359
+ // still be depended upon.
360
+ bool disabled = 1;
361
+ }
362
+
363
+ // Configuration options for BigQuery table assertions.
364
+ message AssertionConfig {
365
+ // If set to true, this action will not be executed. However, the action may
366
+ // still be depended upon.
367
+ bool disabled = 1;
368
+ }
369
+
370
+ // Configuration options for BigQuery table opertions.
371
+ message OperationConfig {
372
+ // If set to true, this action will not be executed. However, the action may
373
+ // still be depended upon.
374
+ bool disabled = 1;
375
+
376
+ // Declares that this `operations` action creates a dataset which should be
377
+ // referenceable using the `ref` function.
378
+ bool has_output = 2;
379
+ }
380
+
381
+ // Configuration options for BigQuery table declarations.
382
+ // TODO(ekrekr): remove this empty message, if explicit action type without a
383
+ // `type` field proves to be impossible / ugly.
384
+ message DeclarationConfig {}
385
+
386
+ // Configuration options for Vertex AI notebooks.
387
+ message NotebookConfig {
388
+ // If set to true, this action will not be executed. However, the action may
389
+ // still be depended upon.
390
+ bool disabled = 1;
391
+
392
+ // TODO(ekrekr): add a notebook runtime field definition.
393
+ }
package/package.json CHANGED
@@ -4,7 +4,6 @@
4
4
  "**/object-path": "^0.11.5",
5
5
  "**/http-proxy": "^1.18.1",
6
6
  "**/serialize-javascript": "^3.1.0",
7
- "**/lodash": "^4.17.21",
8
7
  "**/ssri": "^7.1.1",
9
8
  "**/postcss": "^8.2.10",
10
9
  "**/browserslist": "^4.16.5",
@@ -28,7 +27,7 @@
28
27
  "**/markdown-it": "^12.3.2"
29
28
  },
30
29
  "homepage": "https://github.com/dataform-co/dataform",
31
- "license": "MIT",
30
+ "license": "Apache-2.0",
32
31
  "keywords": [
33
32
  "dataform",
34
33
  "etl",
@@ -37,13 +36,12 @@
37
36
  "data-modelling",
38
37
  "sql",
39
38
  "bigquery",
40
- "redshift",
41
39
  "dataops"
42
40
  ],
43
41
  "publishConfig": {
44
42
  "access": "public"
45
43
  },
46
- "version": "2.8.0-beta.1",
44
+ "version": "3.0.0-alpha.0",
47
45
  "name": "@dataform/core",
48
46
  "description": "Dataform core API.",
49
47
  "main": "bundle.js"