@dataform/core 2.8.4 → 3.0.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/bundle.js +17 -1
  2. package/core.proto +143 -55
  3. package/package.json +2 -4
package/core.proto CHANGED
@@ -1,4 +1,18 @@
1
- syntax="proto3";
1
+ // Copyright 2023 Google LLC
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+
15
+ syntax = "proto3";
2
16
 
3
17
  package dataform;
4
18
 
@@ -19,29 +33,13 @@ message ProjectConfig {
19
33
 
20
34
  map<string, string> vars = 14;
21
35
 
22
- int32 concurrent_query_limit = 13;
23
- int32 idempotent_action_retries = 8;
24
-
25
36
  string database_suffix = 15;
26
37
  string schema_suffix = 7;
27
38
  string table_prefix = 11;
28
39
 
40
+ string dataform_core_version = 17;
29
41
 
30
- // Deprecated.
31
- // Please use 'default_database' instead.
32
- string gcloud_project_id = 6;
33
- // This setting no longer has any effect.
34
- bool use_run_cache = 10;
35
-
36
- reserved 3, 4, 12;
37
- }
38
-
39
- message GenerateIndexConfig {
40
- CompileConfig compile_config = 1;
41
- repeated string include_paths = 2;
42
- repeated string definition_paths = 3;
43
-
44
- reserved 4;
42
+ reserved 3, 4, 6, 8, 10, 12, 13;
45
43
  }
46
44
 
47
45
  message CompileConfig {
@@ -57,10 +55,7 @@ message CompileConfig {
57
55
  // Override compilation timeout settings.
58
56
  int32 timeout_millis = 6;
59
57
 
60
- // Whether to use the new main function instead of gen index.
61
- bool use_main = 9;
62
-
63
- reserved 2, 4, 5, 7;
58
+ reserved 2, 4, 5, 7, 9;
64
59
  }
65
60
 
66
61
  message Target {
@@ -79,29 +74,6 @@ message BigQueryOptions {
79
74
  map<string, string> additional_options = 7;
80
75
  }
81
76
 
82
- message RedshiftOptions {
83
- string dist_key = 1;
84
- string dist_style = 2;
85
- repeated string sort_keys = 3;
86
- string sort_style = 4;
87
-
88
- bool bind = 5 [deprecated = true];
89
- }
90
-
91
- message SnowflakeOptions {
92
- bool secure = 1;
93
- bool transient = 2;
94
- repeated string cluster_by = 3;
95
- }
96
-
97
- message SQLDataWarehouseOptions {
98
- string distribution = 1;
99
- }
100
-
101
- message PrestoOptions {
102
- repeated string partition_by = 3;
103
- }
104
-
105
77
  message GraphErrors {
106
78
  repeated CompilationError compilation_errors = 1;
107
79
 
@@ -129,7 +101,8 @@ message ColumnDescriptor {
129
101
  string description = 1;
130
102
 
131
103
  // For nested records, the path might look like e.g. ["record", "column"].
132
- // For simple columns, this will always contain a single entry e.g. ["column"].
104
+ // For simple columns, this will always contain a single entry e.g.
105
+ // ["column"].
133
106
  repeated string path = 2;
134
107
 
135
108
  string display_name = 3;
@@ -168,6 +141,12 @@ enum TableType {
168
141
  }
169
142
 
170
143
  message Table {
144
+ // For legacy compatability reasons, both incremental tables and views are
145
+ // configured in compiled graphs via the Table proto.
146
+ TableType enum_type = 36;
147
+
148
+ ActionConfig config = 37;
149
+
171
150
  Target target = 4;
172
151
  Target canonical_target = 32;
173
152
 
@@ -180,7 +159,6 @@ message Table {
180
159
  string query = 5;
181
160
  bool protected = 9;
182
161
  bool materialized = 35;
183
- TableType enum_type = 36;
184
162
 
185
163
  ActionDescriptor action_descriptor = 24;
186
164
 
@@ -198,19 +176,17 @@ message Table {
198
176
  repeated string incremental_post_ops = 29;
199
177
 
200
178
  // Warehouse specific features.
201
- RedshiftOptions redshift = 21;
202
179
  BigQueryOptions bigquery = 22;
203
- SnowflakeOptions snowflake = 33;
204
- SQLDataWarehouseOptions sql_data_warehouse = 25;
205
- PrestoOptions presto = 34;
206
180
 
207
181
  // Generated.
208
182
  string file_name = 18;
209
183
 
210
- reserved 1,2, 7, 12, 16;
184
+ reserved 1, 2, 7, 12, 16;
211
185
  }
212
186
 
213
187
  message Operation {
188
+ ActionConfig config = 15;
189
+
214
190
  Target target = 3;
215
191
  Target canonical_target = 13;
216
192
  repeated Target dependency_targets = 11;
@@ -232,6 +208,8 @@ message Operation {
232
208
  }
233
209
 
234
210
  message Assertion {
211
+ ActionConfig config = 16;
212
+
235
213
  Target target = 8;
236
214
  Target canonical_target = 13;
237
215
 
@@ -262,6 +240,7 @@ enum ActionHermeticity {
262
240
  }
263
241
 
264
242
  message Declaration {
243
+ ActionConfig config = 16;
265
244
 
266
245
  Target target = 2;
267
246
  Target canonical_target = 5;
@@ -282,6 +261,14 @@ message Test {
282
261
  string file_name = 4;
283
262
  }
284
263
 
264
+ message Notebook {
265
+ ActionConfig config = 1;
266
+
267
+ Target target = 2;
268
+
269
+ string notebook_contents = 3;
270
+ }
271
+
285
272
  message CompiledGraph {
286
273
  ProjectConfig project_config = 4;
287
274
 
@@ -289,7 +276,10 @@ message CompiledGraph {
289
276
  repeated Operation operations = 2;
290
277
  repeated Assertion assertions = 3;
291
278
  repeated Declaration declarations = 9;
279
+ // TODO(ekrekr): move the Test action outside of the compiled graph, because
280
+ // it is not used at runtime.
292
281
  repeated Test tests = 8;
282
+ repeated Notebook notebooks = 12;
293
283
 
294
284
  GraphErrors graph_errors = 7;
295
285
 
@@ -320,9 +310,107 @@ message CompileExecutionResponse {
320
310
  CompiledGraph compiled_graph = 1;
321
311
  }
322
312
 
323
- // This feature list is added to when making potentilly backwards breaking changes. It lets the
324
- // caller of Dataform Core know whether it supports the change.
313
+ // This feature list is added to when making potentilly backwards breaking
314
+ // changes. It lets the caller of Dataform Core know whether it supports the
315
+ // change.
325
316
  enum SupportedFeatures {
326
317
  UNKNOWN_FEATURE = 0;
327
318
  ARRAY_BUFFER_IPC = 1;
328
319
  }
320
+
321
+ message ActionConfigs {
322
+ repeated ActionConfig actions = 1;
323
+ }
324
+
325
+ message ActionConfig {
326
+ // Target is a unique action identifier.
327
+ Target target = 1;
328
+
329
+ // A list of user-defined tags with which the action should be labeled.
330
+ repeated string tags = 2;
331
+
332
+ // Targets of actions that this action is dependent on.
333
+ repeated Target dependency_targets = 3;
334
+
335
+ // Filename to load the action contents from.
336
+ string file_name = 4;
337
+
338
+ // Type specific configs. Must correspond to the declared type.
339
+ oneof action_config {
340
+ TableConfig table = 5;
341
+ ViewConfig view = 6;
342
+ IncrementalTableConfig incremental_table = 7;
343
+ AssertionConfig assertion = 8;
344
+ OperationConfig operation = 9;
345
+ DeclarationConfig declaration = 10;
346
+ NotebookConfig notebook = 11;
347
+ }
348
+ }
349
+
350
+ // Configuration options for BigQuery tables.
351
+ message TableConfig {
352
+ // If set to true, this action will not be executed. However, the action may
353
+ // still be depended upon.
354
+ bool disabled = 1;
355
+ }
356
+
357
+ // Configuration options for BigQuery views.
358
+ message ViewConfig {
359
+ // If set to true, this action will not be executed. However, the action may
360
+ // still be depended upon.
361
+ bool disabled = 1;
362
+
363
+ // If set to true, will make the view materialized.
364
+ // For more information, read the [BigQuery materialized view
365
+ // docs](https://cloud.google.com/bigquery/docs/materialized-views-intro).
366
+ bool materialized = 2;
367
+ }
368
+
369
+ // Configuration options for BigQuery incremental tables.
370
+ message IncrementalTableConfig {
371
+ // If set to true, this action will not be executed. However, the action may
372
+ // still be depended upon.
373
+ bool disabled = 1;
374
+
375
+ // If set to true, running this action will ignore the full-refresh option.
376
+ // This is useful for tables which are built from transient data, to ensure
377
+ // that historical data is never lost.
378
+ bool protected = 2;
379
+
380
+ // Unique keys for merge criteria for incremental tables.
381
+ // If configured, records with matching unique key(s) will be updated, rather
382
+ // than new rows being inserted.
383
+ repeated string unique_key = 3;
384
+ }
385
+
386
+ // Configuration options for BigQuery table assertions.
387
+ message AssertionConfig {
388
+ // If set to true, this action will not be executed. However, the action may
389
+ // still be depended upon.
390
+ bool disabled = 1;
391
+ }
392
+
393
+ // Configuration options for BigQuery table opertions.
394
+ message OperationConfig {
395
+ // If set to true, this action will not be executed. However, the action may
396
+ // still be depended upon.
397
+ bool disabled = 1;
398
+
399
+ // Declares that this `operations` action creates a dataset which should be
400
+ // referenceable using the `ref` function.
401
+ bool has_output = 2;
402
+ }
403
+
404
+ // Configuration options for BigQuery table declarations.
405
+ // TODO(ekrekr): remove this empty message, if explicit action type without a
406
+ // `type` field proves to be impossible / ugly.
407
+ message DeclarationConfig {}
408
+
409
+ // Configuration options for Vertex AI notebooks.
410
+ message NotebookConfig {
411
+ // If set to true, this action will not be executed. However, the action may
412
+ // still be depended upon.
413
+ bool disabled = 1;
414
+
415
+ // TODO(ekrekr): add a notebook runtime field definition.
416
+ }
package/package.json CHANGED
@@ -4,7 +4,6 @@
4
4
  "**/object-path": "^0.11.5",
5
5
  "**/http-proxy": "^1.18.1",
6
6
  "**/serialize-javascript": "^3.1.0",
7
- "**/lodash": "^4.17.21",
8
7
  "**/ssri": "^7.1.1",
9
8
  "**/postcss": "^8.2.10",
10
9
  "**/browserslist": "^4.16.5",
@@ -28,7 +27,7 @@
28
27
  "**/markdown-it": "^12.3.2"
29
28
  },
30
29
  "homepage": "https://github.com/dataform-co/dataform",
31
- "license": "MIT",
30
+ "license": "Apache-2.0",
32
31
  "keywords": [
33
32
  "dataform",
34
33
  "etl",
@@ -37,13 +36,12 @@
37
36
  "data-modelling",
38
37
  "sql",
39
38
  "bigquery",
40
- "redshift",
41
39
  "dataops"
42
40
  ],
43
41
  "publishConfig": {
44
42
  "access": "public"
45
43
  },
46
- "version": "2.8.4",
44
+ "version": "3.0.0-alpha.1",
47
45
  "name": "@dataform/core",
48
46
  "description": "Dataform core API.",
49
47
  "main": "bundle.js"