@dataform/core 3.1.2 → 3.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/configs.proto CHANGED
@@ -29,7 +29,11 @@ import "property_graph_config.proto";
29
29
  // Workflow Settings defines the contents of the `workflow_settings.yaml`
30
30
  // configuration file.
31
31
  message WorkflowSettings {
32
- // The desired dataform core version to compile against.
32
+ // The desired dataform core version to compile against. When set, the CLI
33
+ // installs this version into a copy of the project in a temporary directory
34
+ // and compiles the copy, which skips paths matched by `.gitignore` or
35
+ // `.dataformignore` outside `definitions/` and `includes/`. For details, see
36
+ // https://dataform-co.github.io/dataform/docs/stateless-install.
33
37
  string dataform_core_version = 1;
34
38
 
35
39
  // Required. The default Google Cloud project (database).
@@ -91,8 +95,21 @@ message WorkflowSettings {
91
95
  LineageSettings lineage = 19;
92
96
 
93
97
  // Optional. Default key-value pairs for BigQuery job labels applied to
94
- // BigQuery jobs executed for actions in this project. Can be overridden by
95
- // action-level `job_labels`.
98
+ // BigQuery jobs executed for actions in this project. Merged key-by-key when
99
+ // overridden via `projectConfigOverride`. Action-level `jobLabels` override
100
+ // matching keys; an action can't remove a default key. Built-in assertions
101
+ // use the job labels of their table. When an action has any job labels,
102
+ // including these defaults, the Dataform CLI doesn't also apply the table's
103
+ // `labels` to its jobs. `dataform run --job-labels` overrides matching keys.
104
+ // Keys and values can contain only lowercase ASCII letters, numbers,
105
+ // underscores and hyphens, and can be at most 63 characters long. Keys must
106
+ // start with a lowercase letter; values can be empty. An action can have at
107
+ // most 32 job labels, including these defaults. Keys that start with
108
+ // `dataform_`, `dataform-`, `dataprep-`, `gcp-` or `goog-` are reserved, as
109
+ // are `bigquery-workflow`, `single-file-asset-type`, `origin`, `bq-dea` and
110
+ // `data-preparation-file-loader`. Numbers and booleans are converted to
111
+ // strings, so quote values that must be kept exactly as written, for example
112
+ // `"0042"`.
96
113
  map<string, string> default_job_labels = 20;
97
114
 
98
115
  // Optional. Default config options for Lakehouse (4-part) targets.
@@ -316,8 +333,11 @@ message ActionConfig {
316
333
  optional bool preserve_governance_controls = 25;
317
334
 
318
335
  // Optional. Key-value pairs for BigQuery job labels applied to the query
319
- // job(s) executed for this table. Overrides matching keys from
320
- // `default_job_labels` in `workflow_settings.yaml`.
336
+ // job(s) executed for this table and its built-in assertions. Overrides
337
+ // matching keys from `defaultJobLabels` in `workflow_settings.yaml`. When
338
+ // the table has any job labels, including defaults, the Dataform CLI
339
+ // doesn't also apply its `labels` to its jobs. See `defaultJobLabels` for
340
+ // the allowed keys and values.
321
341
  map<string, string> job_labels = 26;
322
342
 
323
343
  // Configuration options for a Lakehouse (4-part) table.
@@ -412,8 +432,11 @@ message ActionConfig {
412
432
  string reservation = 21;
413
433
 
414
434
  // Optional. Key-value pairs for BigQuery job labels applied to the query
415
- // job(s) executed for this view. Overrides matching keys from
416
- // `default_job_labels` in `workflow_settings.yaml`.
435
+ // job(s) executed for this view and its built-in assertions. Overrides
436
+ // matching keys from `defaultJobLabels` in `workflow_settings.yaml`. When
437
+ // the view has any job labels, including defaults, the Dataform CLI
438
+ // doesn't also apply its `labels` to its jobs. See `defaultJobLabels` for
439
+ // the allowed keys and values.
417
440
  map<string, string> job_labels = 22;
418
441
  }
419
442
 
@@ -556,8 +579,11 @@ message ActionConfig {
556
579
  optional bool preserve_governance_controls = 30;
557
580
 
558
581
  // Optional. Key-value pairs for BigQuery job labels applied to the query
559
- // job(s) executed for this incremental table. Overrides matching keys from
560
- // `default_job_labels` in `workflow_settings.yaml`.
582
+ // job(s) executed for this incremental table and its built-in assertions.
583
+ // Overrides matching keys from `defaultJobLabels` in
584
+ // `workflow_settings.yaml`. When the table has any job labels, including
585
+ // defaults, the Dataform CLI doesn't also apply its `labels` to its jobs.
586
+ // See `defaultJobLabels` for the allowed keys and values.
561
587
  map<string, string> job_labels = 31;
562
588
 
563
589
  // Configuration options for a Lakehouse (4-part) incremental table.
@@ -611,7 +637,8 @@ message ActionConfig {
611
637
 
612
638
  // Optional. Key-value pairs for BigQuery job labels applied to the query
613
639
  // job(s) executed for this assertion. Overrides matching keys from
614
- // `default_job_labels` in `workflow_settings.yaml`.
640
+ // `defaultJobLabels` in `workflow_settings.yaml`. See `defaultJobLabels`
641
+ // for the allowed keys and values.
615
642
  map<string, string> job_labels = 13;
616
643
  }
617
644
 
@@ -668,7 +695,8 @@ message ActionConfig {
668
695
 
669
696
  // Optional. Key-value pairs for BigQuery job labels applied to the query
670
697
  // job(s) executed for this operation. Overrides matching keys from
671
- // `default_job_labels` in `workflow_settings.yaml`.
698
+ // `defaultJobLabels` in `workflow_settings.yaml`. See `defaultJobLabels`
699
+ // for the allowed keys and values.
672
700
  map<string, string> job_labels = 14;
673
701
 
674
702
  // Configuration options for a Lakehouse (4-part) operation target.
@@ -769,7 +797,8 @@ message ActionConfig {
769
797
 
770
798
  // Optional. Key-value pairs for BigQuery job labels applied to the BigQuery
771
799
  // job(s) executed for this data preparation. Overrides matching keys from
772
- // `default_job_labels` in `workflow_settings.yaml`.
800
+ // `defaultJobLabels` in `workflow_settings.yaml`. See `defaultJobLabels`
801
+ // for the allowed keys and values.
773
802
  map<string, string> job_labels = 15;
774
803
 
775
804
  message ErrorTableConfig {
@@ -795,6 +824,7 @@ message ActionConfig {
795
824
  DeclarationConfig declaration = 6;
796
825
  NotebookConfig notebook = 7;
797
826
  DataPreparationConfig data_preparation = 8;
827
+ // Not supported in `actions.yaml`; use `definitions/**/graph.yaml`.
798
828
  PropertyGraphConfig property_graph = 9;
799
829
  }
800
830
 
package/package.json CHANGED
@@ -14,7 +14,6 @@
14
14
  "protobufjs": "^7.6.5",
15
15
  "ansi-regex": "^3.0.1",
16
16
  "node-fetch": "^2.6.7",
17
- "markdown-it": "14.3.1",
18
17
  "mdurl": "2.0.0",
19
18
  "wrap-ansi": "7.0.0",
20
19
  "string-width": "4.1.0",
@@ -43,7 +42,7 @@
43
42
  "publishConfig": {
44
43
  "access": "public"
45
44
  },
46
- "version": "3.1.2",
45
+ "version": "3.1.4",
47
46
  "name": "@dataform/core",
48
47
  "description": "Dataform core API.",
49
48
  "main": "bundle.js"
@@ -0,0 +1,276 @@
1
+ // Copyright 2023 Google LLC
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+
15
+ // Defines the configuration schema for BigQuery Property Graph YAML files
16
+ // (`definitions/**/graph.yaml`), aligned with the Google Semantic Ontology V1
17
+ // specification (`version: 0.2.0.dev0/google`). At most one `graph.yaml` file
18
+ // is allowed per project in V1. Both `snake_case` (recommended in `graph.yaml`,
19
+ // e.g. `semantic_model`, `primary_key`, `from_columns`, `to_columns`,
20
+ // `ai_context`, `job_labels`, `depend_on_dependency_assertions`,
21
+ // `include_dependent_assertions`) and `camelCase` keys are accepted.
22
+ syntax = "proto3";
23
+
24
+ package dataform;
25
+
26
+ option java_package = "com.dataform.protos";
27
+ option java_outer_classname = "PropertyGraphConfigMeta";
28
+ option java_multiple_files = true;
29
+
30
+ option go_package = "github.com/dataform-co/dataform/protos/dataform";
31
+
32
+ // Top-level configuration for a Dataform BigQuery Property Graph YAML file
33
+ // (`definitions/**/graph.yaml`), aligned with the Google Semantic Ontology V1
34
+ // specification (`version: 0.2.0.dev0/google`).
35
+ message PropertyGraphConfig {
36
+ // Required. Semantic Ontology specification version (`"0.2.0.dev0/google"`).
37
+ string version = 1;
38
+ // Required (`semantic_model` in YAML). Semantic models declared in the file
39
+ // (must contain 1 model in V1).
40
+ repeated SemanticModelConfig semantic_model = 2;
41
+ // Optional. If set to true, this property graph action will not be executed,
42
+ // though it can still be depended upon.
43
+ bool disabled = 3;
44
+ // Optional (`depend_on_dependency_assertions` in YAML). If true, assertions
45
+ // dependent upon any of the graph's referenced dependencies are added as
46
+ // dependencies of this property graph action.
47
+ bool depend_on_dependency_assertions = 4;
48
+ // Optional. A list of user-defined tags with which the property graph action
49
+ // should be labeled.
50
+ repeated string tags = 5;
51
+ // Optional (`job_labels` in YAML). Key-value pairs for BigQuery job labels
52
+ // applied to the query job(s) executed for this property graph. Overrides
53
+ // matching keys from `defaultJobLabels` in `workflow_settings.yaml`. See
54
+ // `defaultJobLabels` for the allowed keys and values.
55
+ map<string, string> job_labels = 6;
56
+ }
57
+
58
+ // A single semantic model definition within `semantic_model` in
59
+ // `definitions/**/graph.yaml`.
60
+ message SemanticModelConfig {
61
+ // Required. Name of the semantic model / BigQuery property graph (must be a
62
+ // valid identifier).
63
+ string name = 1;
64
+ // Optional. Human-readable description of the semantic model. Populates
65
+ // Dataform's action metadata (`PropertyGraph.description`) only and has no
66
+ // effect on the generated BigQuery DDL (BigQuery `CREATE PROPERTY GRAPH` does
67
+ // not support graph-level `OPTIONS`).
68
+ string description = 2;
69
+ // Optional (`ai_context` in YAML). AI context metadata for the model.
70
+ // `instructions` and `examples` are folded into Dataform's action metadata
71
+ // (`PropertyGraph.description`) only; model-level `ai_context` has no effect
72
+ // on the generated BigQuery DDL.
73
+ GraphAiContextConfig ai_context = 3;
74
+ // Optional. Deployment target(s) for the property graph (at most 1 in V1).
75
+ repeated GraphDeploymentConfig deployments = 4;
76
+ // Required (`entities` or `datasets` in YAML). Node entities in the semantic
77
+ // model (must contain at least 1 entity).
78
+ repeated GraphEntityConfig entities = 5;
79
+ // Optional. Edge relationships connecting entities in the semantic model.
80
+ repeated GraphRelationshipConfig relationships = 6;
81
+ // Optional. Top-level aggregate metrics compiled as `MEASURE(...)` properties
82
+ // on target entities.
83
+ repeated GraphMetricConfig metrics = 7;
84
+ }
85
+
86
+ // Deployment target mapping for a semantic model (`deployments` in
87
+ // `graph.yaml`). At most one deployment is supported in V1; when omitted, the
88
+ // property graph is deployed to the project's default database and default
89
+ // schema with the `semantic_model.name`.
90
+ message GraphDeploymentConfig {
91
+ // Logical deployment profile name (for example, `default`).
92
+ string name = 1;
93
+ // Required. Target BigQuery property graph resource URI
94
+ // (`bigquery.googleapis.com/projects/<project>/datasets/<dataset>/propertyGraphs/<graph>`).
95
+ // Leading `//` is rejected. Sets the base project, dataset, and graph name on
96
+ // the Dataform action target; Dataform session overrides (`databaseSuffix`,
97
+ // `schemaSuffix`, `namePrefix`) still apply when resolving the final target.
98
+ string target = 2;
99
+ }
100
+
101
+ // A node entity definition in the semantic model (`entities` in `graph.yaml`).
102
+ // Compiles to a BigQuery `NODE TABLES` entry.
103
+ message GraphEntityConfig {
104
+ // Required. Unique entity name within the semantic model (must be a valid
105
+ // identifier).
106
+ string name = 1;
107
+ // Backing BigQuery table source (Dataform `ref`, `bigquery:p.d.t`, or GCP URI).
108
+ oneof source {
109
+ // Backing table reference when `source` in `graph.yaml` is a string
110
+ // (`bigquery:<project>.<dataset>.<table>`, `<project>.<dataset>.<table>`,
111
+ // or `bigquery.googleapis.com/projects/<project>/datasets/<dataset>/tables/<table>`;
112
+ // leading `//` is rejected). Mutually exclusive with `sourceRef`.
113
+ string source_string = 2;
114
+ // Backing Dataform action reference when `source` in `graph.yaml` is
115
+ // `{ ref: "<name>" }` or
116
+ // `{ ref: { name, schema, database, include_dependent_assertions } }`.
117
+ // Mutually exclusive with `sourceString`.
118
+ DataSourceRef source_ref = 3;
119
+ }
120
+ // Required (`primary_key` in YAML; accepts a single string or list of
121
+ // strings). Physical primary key column name(s) identifying rows of this
122
+ // entity, emitted in `KEY (...)`.
123
+ repeated string primary_key = 4;
124
+ // Secondary unique key column lists (`unique_keys` in YAML). Not supported in
125
+ // V1 because BigQuery `CREATE PROPERTY GRAPH` DDL supports only a single
126
+ // `KEY(...)` clause.
127
+ repeated GraphUniqueKeyList unique_keys = 5;
128
+ // Optional. Human-readable description of the entity, emitted in
129
+ // `DEFAULT LABEL OPTIONS(description=...)`.
130
+ string description = 6;
131
+ // Optional (`ai_context` in YAML). AI context metadata (`instructions`,
132
+ // `synonyms`, and `examples`) for the entity.
133
+ GraphAiContextConfig ai_context = 7;
134
+ // Optional. Properties exposed on the entity node table (`PROPERTIES (...)`).
135
+ // In `graph.yaml`, each item may be a bare column name string `"col"` or a
136
+ // `GraphFieldConfig` object. When omitted (and no `metrics` target the
137
+ // entity), `PROPERTIES ALL COLUMNS` is emitted. Required when one or more
138
+ // `metrics` target the entity.
139
+ repeated GraphFieldConfig fields = 8;
140
+ }
141
+
142
+ // Wrapper for a single composite unique key column list on an entity
143
+ // (`unique_keys` in `graph.yaml`). Accepted by the parser so V1 validation can
144
+ // report an explicit unsupported error.
145
+ message GraphUniqueKeyList {
146
+ // Column names comprising the unique key list.
147
+ repeated string columns = 1;
148
+ }
149
+
150
+ // An edge relationship definition connecting two entities in the semantic model
151
+ // (`relationships` in `graph.yaml`). Each relationship is a 1:N foreign-key
152
+ // edge from `from` to `to`, backed by the `from` entity's table and
153
+ // `primary_key`, and compiles to a BigQuery `EDGE TABLES` entry with
154
+ // `NO PROPERTIES` in V1.
155
+ message GraphRelationshipConfig {
156
+ // Required. Unique relationship name within the semantic model (must be a
157
+ // valid identifier).
158
+ string name = 1;
159
+ // Required. Name of the source entity (`SOURCE KEY ... REFERENCES <from>`).
160
+ string from = 5;
161
+ // Required. Name of the destination entity
162
+ // (`DESTINATION KEY ... REFERENCES <to>`).
163
+ string to = 6;
164
+ // Required (`from_columns` in YAML; accepts a single string or list of
165
+ // strings). Physical foreign-key column name(s) on the `from` entity table
166
+ // emitted in `DESTINATION KEY (<from_columns>) REFERENCES <to> (<to_columns>)`.
167
+ repeated string from_columns = 7;
168
+ // Required (`to_columns` in YAML; accepts a single string or list of
169
+ // strings). Referenced key column(s) on the `to` entity table (must match
170
+ // `<to>.primary_key`).
171
+ repeated string to_columns = 8;
172
+ // Optional. Human-readable description of the relationship, emitted in
173
+ // `DEFAULT LABEL OPTIONS(description=...)`.
174
+ string description = 9;
175
+ // Optional (`ai_context` in YAML). AI context metadata (`instructions`,
176
+ // `synonyms`, and `examples`) for the relationship.
177
+ GraphAiContextConfig ai_context = 10;
178
+ }
179
+
180
+ // Dimension metadata for a property field on an entity (`dimension` in
181
+ // `graph.yaml`). In `graph.yaml`, a boolean shorthand `dimension: true` is also
182
+ // accepted and normalized to `{ is_time: true }`.
183
+ message GraphDimensionConfig {
184
+ // Optional (`is_time` in YAML). If true, marks the field as a temporal
185
+ // dimension and folds `"Time dimension."` into `OPTIONS(description=...)`.
186
+ bool is_time = 1;
187
+ }
188
+
189
+ // A property field definition on an entity (`fields` in `graph.yaml`). In
190
+ // `graph.yaml`, a bare string `"col"` is shorthand for
191
+ // `{ name: "col", expression: "col" }`.
192
+ message GraphFieldConfig {
193
+ // Required. Property name exposed on the graph element (must be a valid
194
+ // identifier, unique case-insensitively on the entity).
195
+ string name = 1;
196
+ // Optional. SQL expression computing the property value; defaults to `name`.
197
+ // Passed through as-is into BigQuery DDL (`<expression> AS <name>` when
198
+ // distinct from `name`).
199
+ string expression = 2;
200
+ // Optional. Human-readable description of the field, folded into
201
+ // `OPTIONS(description=...)`.
202
+ string description = 3;
203
+ // Optional. Informational data type metadata. Accepted by the parser but has
204
+ // no effect on the generated BigQuery DDL.
205
+ string datatype = 4;
206
+ // Optional. Dimension metadata (e.g. `dimension: { is_time: true }` or
207
+ // `dimension: true`), folding `"Time dimension."` into
208
+ // `OPTIONS(description=...)` when `is_time` is true.
209
+ GraphDimensionConfig dimension = 5;
210
+ // Optional. Human-readable display label folded into
211
+ // `OPTIONS(description=...)`.
212
+ string label = 6;
213
+ // Optional (`ai_context` in YAML). AI context metadata (`instructions`,
214
+ // `synonyms`, and `examples`) for the field.
215
+ GraphAiContextConfig ai_context = 7;
216
+ }
217
+
218
+ // A top-level aggregate metric definition (`metrics` in `graph.yaml`), compiled
219
+ // as a `MEASURE(<expression>) AS <name>` property inside the target entity's
220
+ // `PROPERTIES (...)` list.
221
+ message GraphMetricConfig {
222
+ // Required. Unique metric name within the semantic model (must be a valid
223
+ // identifier and must not collide case-insensitively with any declared field
224
+ // on the target `entity`).
225
+ string name = 1;
226
+ // Required. Target entity name on whose node table the `MEASURE(...)`
227
+ // property is emitted.
228
+ string entity = 2;
229
+ // Required. Single supported SQL aggregate (`SUM`, `AVG`, `COUNT`, `MIN`, or
230
+ // `MAX`), or `COUNT(*)`. Passed through as-is into
231
+ // `MEASURE(<expression>) AS <name>`, except `COUNT(*)` which is lowered to
232
+ // `COUNT(<primary_key[0]>)`.
233
+ string expression = 3;
234
+ // Optional. Human-readable description of the metric, folded into
235
+ // `OPTIONS(description=...)`.
236
+ string description = 4;
237
+ // Optional. Human-readable display label folded into
238
+ // `OPTIONS(description=...)`.
239
+ string label = 5;
240
+ // Optional (`ai_context` in YAML). AI context metadata (`instructions`,
241
+ // `synonyms`, and `examples`) for the metric.
242
+ GraphAiContextConfig ai_context = 6;
243
+ }
244
+
245
+ // AI context metadata attached to a semantic model, entity, relationship, field,
246
+ // or metric (`ai_context` in `graph.yaml`). On entities and relationships,
247
+ // `DEFAULT LABEL OPTIONS(...)` is emitted before `PROPERTIES (...)`,
248
+ // `PROPERTIES ALL COLUMNS`, or `NO PROPERTIES`. Description sections (`label`,
249
+ // `"Time dimension."`, `description`, `ai_context.instructions`, and
250
+ // `"Examples: a; b"`) are joined with `"\n\n"`.
251
+ message GraphAiContextConfig {
252
+ // Optional. Alternative names emitted as `OPTIONS(synonyms=[...])`.
253
+ repeated string synonyms = 1;
254
+ // Optional. Example values or questions folded into the element's description
255
+ // (`"...\n\nExamples: ex1; ex2"`).
256
+ repeated string examples = 2;
257
+ // Optional. Free-form AI instructions folded into the element's description
258
+ // (`OPTIONS(description=...)`).
259
+ string instructions = 3;
260
+ }
261
+
262
+ // Reference to another Dataform action as a backing table source
263
+ // (`source: { ref: ... }`). In `graph.yaml`, a string shorthand
264
+ // `source: { ref: "<action_name>" }` is also accepted and normalized to
265
+ // `{ name: "<action_name>" }`.
266
+ message DataSourceRef {
267
+ // Required. Target Dataform action name.
268
+ string name = 1;
269
+ // Optional. Target dataset (schema) name.
270
+ string schema = 2;
271
+ // Optional. Target Google Cloud project (database) ID.
272
+ string database = 3;
273
+ // Optional (`include_dependent_assertions` in YAML). If true, depends on
274
+ // assertions of the target action.
275
+ bool include_dependent_assertions = 4;
276
+ }