@dataform/core 3.1.3 → 3.1.5-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/configs.proto CHANGED
@@ -29,7 +29,11 @@ import "property_graph_config.proto";
29
29
  // Workflow Settings defines the contents of the `workflow_settings.yaml`
30
30
  // configuration file.
31
31
  message WorkflowSettings {
32
- // The desired dataform core version to compile against.
32
+ // The desired dataform core version to compile against. When set, the CLI
33
+ // installs this version into a copy of the project in a temporary directory
34
+ // and compiles the copy, which skips paths matched by `.gitignore` or
35
+ // `.dataformignore` outside `definitions/` and `includes/`. For details, see
36
+ // https://dataform-co.github.io/dataform/docs/stateless-install.
33
37
  string dataform_core_version = 1;
34
38
 
35
39
  // Required. The default Google Cloud project (database).
@@ -820,6 +824,7 @@ message ActionConfig {
820
824
  DeclarationConfig declaration = 6;
821
825
  NotebookConfig notebook = 7;
822
826
  DataPreparationConfig data_preparation = 8;
827
+ // Not supported in `actions.yaml`; use `definitions/**/graph.yaml`.
823
828
  PropertyGraphConfig property_graph = 9;
824
829
  }
825
830
 
package/package.json CHANGED
@@ -42,7 +42,7 @@
42
42
  "publishConfig": {
43
43
  "access": "public"
44
44
  },
45
- "version": "3.1.3",
45
+ "version": "3.1.5-alpha.2",
46
46
  "name": "@dataform/core",
47
47
  "description": "Dataform core API.",
48
48
  "main": "bundle.js"
@@ -0,0 +1,276 @@
1
+ // Copyright 2023 Google LLC
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+
15
+ // Defines the configuration schema for BigQuery Property Graph YAML files
16
+ // (`definitions/**/graph.yaml`), aligned with the Google Semantic Ontology V1
17
+ // specification (`version: 0.2.0.dev0/google`). At most one `graph.yaml` file
18
+ // is allowed per project in V1. Both `snake_case` (recommended in `graph.yaml`,
19
+ // e.g. `semantic_model`, `primary_key`, `from_columns`, `to_columns`,
20
+ // `ai_context`, `job_labels`, `depend_on_dependency_assertions`,
21
+ // `include_dependent_assertions`) and `camelCase` keys are accepted.
22
+ syntax = "proto3";
23
+
24
+ package dataform;
25
+
26
+ option java_package = "com.dataform.protos";
27
+ option java_outer_classname = "PropertyGraphConfigMeta";
28
+ option java_multiple_files = true;
29
+
30
+ option go_package = "github.com/dataform-co/dataform/protos/dataform";
31
+
32
+ // Top-level configuration for a Dataform BigQuery Property Graph YAML file
33
+ // (`definitions/**/graph.yaml`), aligned with the Google Semantic Ontology V1
34
+ // specification (`version: 0.2.0.dev0/google`).
35
+ message PropertyGraphConfig {
36
+ // Required. Semantic Ontology specification version (`"0.2.0.dev0/google"`).
37
+ string version = 1;
38
+ // Required (`semantic_model` in YAML). Semantic models declared in the file
39
+ // (must contain 1 model in V1).
40
+ repeated SemanticModelConfig semantic_model = 2;
41
+ // Optional. If set to true, this property graph action will not be executed,
42
+ // though it can still be depended upon.
43
+ bool disabled = 3;
44
+ // Optional (`depend_on_dependency_assertions` in YAML). If true, assertions
45
+ // dependent upon any of the graph's referenced dependencies are added as
46
+ // dependencies of this property graph action.
47
+ bool depend_on_dependency_assertions = 4;
48
+ // Optional. A list of user-defined tags with which the property graph action
49
+ // should be labeled.
50
+ repeated string tags = 5;
51
+ // Optional (`job_labels` in YAML). Key-value pairs for BigQuery job labels
52
+ // applied to the query job(s) executed for this property graph. Overrides
53
+ // matching keys from `defaultJobLabels` in `workflow_settings.yaml`. See
54
+ // `defaultJobLabels` for the allowed keys and values.
55
+ map<string, string> job_labels = 6;
56
+ }
57
+
58
+ // A single semantic model definition within `semantic_model` in
59
+ // `definitions/**/graph.yaml`.
60
+ message SemanticModelConfig {
61
+ // Required. Name of the semantic model / BigQuery property graph (must be a
62
+ // valid identifier).
63
+ string name = 1;
64
+ // Optional. Human-readable description of the semantic model. Populates
65
+ // Dataform's action metadata (`PropertyGraph.description`) only and has no
66
+ // effect on the generated BigQuery DDL (BigQuery `CREATE PROPERTY GRAPH` does
67
+ // not support graph-level `OPTIONS`).
68
+ string description = 2;
69
+ // Optional (`ai_context` in YAML). AI context metadata for the model.
70
+ // `instructions` and `examples` are folded into Dataform's action metadata
71
+ // (`PropertyGraph.description`) only; model-level `ai_context` has no effect
72
+ // on the generated BigQuery DDL.
73
+ GraphAiContextConfig ai_context = 3;
74
+ // Optional. Deployment target(s) for the property graph (at most 1 in V1).
75
+ repeated GraphDeploymentConfig deployments = 4;
76
+ // Required (`entities` or `datasets` in YAML). Node entities in the semantic
77
+ // model (must contain at least 1 entity).
78
+ repeated GraphEntityConfig entities = 5;
79
+ // Optional. Edge relationships connecting entities in the semantic model.
80
+ repeated GraphRelationshipConfig relationships = 6;
81
+ // Optional. Top-level aggregate metrics compiled as `MEASURE(...)` properties
82
+ // on target entities.
83
+ repeated GraphMetricConfig metrics = 7;
84
+ }
85
+
86
+ // Deployment target mapping for a semantic model (`deployments` in
87
+ // `graph.yaml`). At most one deployment is supported in V1; when omitted, the
88
+ // property graph is deployed to the project's default database and default
89
+ // schema with the `semantic_model.name`.
90
+ message GraphDeploymentConfig {
91
+ // Logical deployment profile name (for example, `default`).
92
+ string name = 1;
93
+ // Required. Target BigQuery property graph resource URI
94
+ // (`bigquery.googleapis.com/projects/<project>/datasets/<dataset>/propertyGraphs/<graph>`).
95
+ // Leading `//` is rejected. Sets the base project, dataset, and graph name on
96
+ // the Dataform action target; Dataform session overrides (`databaseSuffix`,
97
+ // `schemaSuffix`, `namePrefix`) still apply when resolving the final target.
98
+ string target = 2;
99
+ }
100
+
101
+ // A node entity definition in the semantic model (`entities` in `graph.yaml`).
102
+ // Compiles to a BigQuery `NODE TABLES` entry.
103
+ message GraphEntityConfig {
104
+ // Required. Unique entity name within the semantic model (must be a valid
105
+ // identifier).
106
+ string name = 1;
107
+ // Backing BigQuery table source (Dataform `ref`, `bigquery:p.d.t`, or GCP URI).
108
+ oneof source {
109
+ // Backing table reference when `source` in `graph.yaml` is a string
110
+ // (`bigquery:<project>.<dataset>.<table>`, `<project>.<dataset>.<table>`,
111
+ // or `bigquery.googleapis.com/projects/<project>/datasets/<dataset>/tables/<table>`;
112
+ // leading `//` is rejected). Mutually exclusive with `sourceRef`.
113
+ string source_string = 2;
114
+ // Backing Dataform action reference when `source` in `graph.yaml` is
115
+ // `{ ref: "<name>" }` or
116
+ // `{ ref: { name, schema, database, include_dependent_assertions } }`.
117
+ // Mutually exclusive with `sourceString`.
118
+ DataSourceRef source_ref = 3;
119
+ }
120
+ // Required (`primary_key` in YAML; accepts a single string or list of
121
+ // strings). Physical primary key column name(s) identifying rows of this
122
+ // entity, emitted in `KEY (...)`.
123
+ repeated string primary_key = 4;
124
+ // Secondary unique key column lists (`unique_keys` in YAML). Not supported in
125
+ // V1 because BigQuery `CREATE PROPERTY GRAPH` DDL supports only a single
126
+ // `KEY(...)` clause.
127
+ repeated GraphUniqueKeyList unique_keys = 5;
128
+ // Optional. Human-readable description of the entity, emitted in
129
+ // `DEFAULT LABEL OPTIONS(description=...)`.
130
+ string description = 6;
131
+ // Optional (`ai_context` in YAML). AI context metadata (`instructions`,
132
+ // `synonyms`, and `examples`) for the entity.
133
+ GraphAiContextConfig ai_context = 7;
134
+ // Optional. Properties exposed on the entity node table (`PROPERTIES (...)`).
135
+ // In `graph.yaml`, each item may be a bare column name string `"col"` or a
136
+ // `GraphFieldConfig` object. When omitted (and no `metrics` target the
137
+ // entity), `PROPERTIES ALL COLUMNS` is emitted. Required when one or more
138
+ // `metrics` target the entity.
139
+ repeated GraphFieldConfig fields = 8;
140
+ }
141
+
142
+ // Wrapper for a single composite unique key column list on an entity
143
+ // (`unique_keys` in `graph.yaml`). Accepted by the parser so V1 validation can
144
+ // report an explicit unsupported error.
145
+ message GraphUniqueKeyList {
146
+ // Column names comprising the unique key list.
147
+ repeated string columns = 1;
148
+ }
149
+
150
+ // An edge relationship definition connecting two entities in the semantic model
151
+ // (`relationships` in `graph.yaml`). Each relationship is a 1:N foreign-key
152
+ // edge from `from` to `to`, backed by the `from` entity's table and
153
+ // `primary_key`, and compiles to a BigQuery `EDGE TABLES` entry with
154
+ // `NO PROPERTIES` in V1.
155
+ message GraphRelationshipConfig {
156
+ // Required. Unique relationship name within the semantic model (must be a
157
+ // valid identifier).
158
+ string name = 1;
159
+ // Required. Name of the source entity (`SOURCE KEY ... REFERENCES <from>`).
160
+ string from = 5;
161
+ // Required. Name of the destination entity
162
+ // (`DESTINATION KEY ... REFERENCES <to>`).
163
+ string to = 6;
164
+ // Required (`from_columns` in YAML; accepts a single string or list of
165
+ // strings). Physical foreign-key column name(s) on the `from` entity table
166
+ // emitted in `DESTINATION KEY (<from_columns>) REFERENCES <to> (<to_columns>)`.
167
+ repeated string from_columns = 7;
168
+ // Required (`to_columns` in YAML; accepts a single string or list of
169
+ // strings). Referenced key column(s) on the `to` entity table (must match
170
+ // `<to>.primary_key`).
171
+ repeated string to_columns = 8;
172
+ // Optional. Human-readable description of the relationship, emitted in
173
+ // `DEFAULT LABEL OPTIONS(description=...)`.
174
+ string description = 9;
175
+ // Optional (`ai_context` in YAML). AI context metadata (`instructions`,
176
+ // `synonyms`, and `examples`) for the relationship.
177
+ GraphAiContextConfig ai_context = 10;
178
+ }
179
+
180
+ // Dimension metadata for a property field on an entity (`dimension` in
181
+ // `graph.yaml`). In `graph.yaml`, a boolean shorthand `dimension: true` is also
182
+ // accepted and normalized to `{ is_time: true }`.
183
+ message GraphDimensionConfig {
184
+ // Optional (`is_time` in YAML). If true, marks the field as a temporal
185
+ // dimension and folds `"Time dimension."` into `OPTIONS(description=...)`.
186
+ bool is_time = 1;
187
+ }
188
+
189
+ // A property field definition on an entity (`fields` in `graph.yaml`). In
190
+ // `graph.yaml`, a bare string `"col"` is shorthand for
191
+ // `{ name: "col", expression: "col" }`.
192
+ message GraphFieldConfig {
193
+ // Required. Property name exposed on the graph element (must be a valid
194
+ // identifier, unique case-insensitively on the entity).
195
+ string name = 1;
196
+ // Optional. SQL expression computing the property value; defaults to `name`.
197
+ // Passed through as-is into BigQuery DDL (`<expression> AS <name>` when
198
+ // distinct from `name`).
199
+ string expression = 2;
200
+ // Optional. Human-readable description of the field, folded into
201
+ // `OPTIONS(description=...)`.
202
+ string description = 3;
203
+ // Optional. Informational data type metadata. Accepted by the parser but has
204
+ // no effect on the generated BigQuery DDL.
205
+ string datatype = 4;
206
+ // Optional. Dimension metadata (e.g. `dimension: { is_time: true }` or
207
+ // `dimension: true`), folding `"Time dimension."` into
208
+ // `OPTIONS(description=...)` when `is_time` is true.
209
+ GraphDimensionConfig dimension = 5;
210
+ // Optional. Human-readable display label folded into
211
+ // `OPTIONS(description=...)`.
212
+ string label = 6;
213
+ // Optional (`ai_context` in YAML). AI context metadata (`instructions`,
214
+ // `synonyms`, and `examples`) for the field.
215
+ GraphAiContextConfig ai_context = 7;
216
+ }
217
+
218
+ // A top-level aggregate metric definition (`metrics` in `graph.yaml`), compiled
219
+ // as a `MEASURE(<expression>) AS <name>` property inside the target entity's
220
+ // `PROPERTIES (...)` list.
221
+ message GraphMetricConfig {
222
+ // Required. Unique metric name within the semantic model (must be a valid
223
+ // identifier and must not collide case-insensitively with any declared field
224
+ // on the target `entity`).
225
+ string name = 1;
226
+ // Required. Target entity name on whose node table the `MEASURE(...)`
227
+ // property is emitted.
228
+ string entity = 2;
229
+ // Required. Single supported SQL aggregate (`SUM`, `AVG`, `COUNT`, `MIN`, or
230
+ // `MAX`), or `COUNT(*)`. Passed through as-is into
231
+ // `MEASURE(<expression>) AS <name>`, except `COUNT(*)` which is lowered to
232
+ // `COUNT(<primary_key[0]>)`.
233
+ string expression = 3;
234
+ // Optional. Human-readable description of the metric, folded into
235
+ // `OPTIONS(description=...)`.
236
+ string description = 4;
237
+ // Optional. Human-readable display label folded into
238
+ // `OPTIONS(description=...)`.
239
+ string label = 5;
240
+ // Optional (`ai_context` in YAML). AI context metadata (`instructions`,
241
+ // `synonyms`, and `examples`) for the metric.
242
+ GraphAiContextConfig ai_context = 6;
243
+ }
244
+
245
+ // AI context metadata attached to a semantic model, entity, relationship, field,
246
+ // or metric (`ai_context` in `graph.yaml`). On entities and relationships,
247
+ // `DEFAULT LABEL OPTIONS(...)` is emitted before `PROPERTIES (...)`,
248
+ // `PROPERTIES ALL COLUMNS`, or `NO PROPERTIES`. Description sections (`label`,
249
+ // `"Time dimension."`, `description`, `ai_context.instructions`, and
250
+ // `"Examples: a; b"`) are joined with `"\n\n"`.
251
+ message GraphAiContextConfig {
252
+ // Optional. Alternative names emitted as `OPTIONS(synonyms=[...])`.
253
+ repeated string synonyms = 1;
254
+ // Optional. Example values or questions folded into the element's description
255
+ // (`"...\n\nExamples: ex1; ex2"`).
256
+ repeated string examples = 2;
257
+ // Optional. Free-form AI instructions folded into the element's description
258
+ // (`OPTIONS(description=...)`).
259
+ string instructions = 3;
260
+ }
261
+
262
+ // Reference to another Dataform action as a backing table source
263
+ // (`source: { ref: ... }`). In `graph.yaml`, a string shorthand
264
+ // `source: { ref: "<action_name>" }` is also accepted and normalized to
265
+ // `{ name: "<action_name>" }`.
266
+ message DataSourceRef {
267
+ // Required. Target Dataform action name.
268
+ string name = 1;
269
+ // Optional. Target dataset (schema) name.
270
+ string schema = 2;
271
+ // Optional. Target Google Cloud project (database) ID.
272
+ string database = 3;
273
+ // Optional (`include_dependent_assertions` in YAML). If true, depends on
274
+ // assertions of the target action.
275
+ bool include_dependent_assertions = 4;
276
+ }