google-cloud-dataplex-v1 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/google/cloud/dataplex/v1/analyze_pb.rb +26 -81
- data/lib/google/cloud/dataplex/v1/content_pb.rb +26 -33
- data/lib/google/cloud/dataplex/v1/content_service/client.rb +6 -4
- data/lib/google/cloud/dataplex/v1/content_service/rest/client.rb +6 -4
- data/lib/google/cloud/dataplex/v1/content_service/rest/service_stub.rb +8 -8
- data/lib/google/cloud/dataplex/v1/data_profile_pb.rb +26 -50
- data/lib/google/cloud/dataplex/v1/data_quality_pb.rb +25 -74
- data/lib/google/cloud/dataplex/v1/data_scan_service/client.rb +16 -8
- data/lib/google/cloud/dataplex/v1/data_scan_service/operations.rb +5 -3
- data/lib/google/cloud/dataplex/v1/data_scan_service/rest/client.rb +16 -8
- data/lib/google/cloud/dataplex/v1/data_scan_service/rest/operations.rb +9 -7
- data/lib/google/cloud/dataplex/v1/data_scan_service/rest/service_stub.rb +8 -8
- data/lib/google/cloud/dataplex/v1/dataplex_service/client.rb +99 -4
- data/lib/google/cloud/dataplex/v1/dataplex_service/operations.rb +5 -3
- data/lib/google/cloud/dataplex/v1/dataplex_service/rest/client.rb +76 -4
- data/lib/google/cloud/dataplex/v1/dataplex_service/rest/operations.rb +9 -7
- data/lib/google/cloud/dataplex/v1/dataplex_service/rest/service_stub.rb +92 -32
- data/lib/google/cloud/dataplex/v1/datascans_pb.rb +31 -121
- data/lib/google/cloud/dataplex/v1/logs_pb.rb +26 -151
- data/lib/google/cloud/dataplex/v1/metadata_pb.rb +27 -191
- data/lib/google/cloud/dataplex/v1/metadata_service/client.rb +6 -4
- data/lib/google/cloud/dataplex/v1/metadata_service/rest/client.rb +6 -4
- data/lib/google/cloud/dataplex/v1/metadata_service/rest/service_stub.rb +9 -9
- data/lib/google/cloud/dataplex/v1/processing_pb.rb +24 -27
- data/lib/google/cloud/dataplex/v1/resources_pb.rb +27 -238
- data/lib/google/cloud/dataplex/v1/rest.rb +1 -1
- data/lib/google/cloud/dataplex/v1/service_pb.rb +31 -196
- data/lib/google/cloud/dataplex/v1/service_services_pb.rb +2 -0
- data/lib/google/cloud/dataplex/v1/tasks_pb.rb +26 -113
- data/lib/google/cloud/dataplex/v1/version.rb +1 -1
- data/lib/google/cloud/dataplex/v1.rb +1 -1
- data/proto_docs/google/api/client.rb +67 -4
- data/proto_docs/google/cloud/dataplex/v1/analyze.rb +4 -0
- data/proto_docs/google/cloud/dataplex/v1/data_profile.rb +64 -87
- data/proto_docs/google/cloud/dataplex/v1/data_quality.rb +54 -32
- data/proto_docs/google/cloud/dataplex/v1/datascans.rb +10 -2
- data/proto_docs/google/cloud/dataplex/v1/metadata.rb +31 -2
- data/proto_docs/google/cloud/dataplex/v1/processing.rb +23 -13
- data/proto_docs/google/cloud/dataplex/v1/resources.rb +24 -6
- data/proto_docs/google/cloud/dataplex/v1/service.rb +17 -0
- data/proto_docs/google/cloud/dataplex/v1/tasks.rb +2 -3
- data/proto_docs/google/protobuf/any.rb +7 -4
- data/proto_docs/google/protobuf/timestamp.rb +1 -3
- metadata +4 -4
@@ -27,131 +27,113 @@ module Google
|
|
27
27
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
28
28
|
end
|
29
29
|
|
30
|
-
# DataProfileResult defines the output of DataProfileScan.
|
31
|
-
#
|
30
|
+
# DataProfileResult defines the output of DataProfileScan. Each field of the
|
31
|
+
# table will have field type specific profile result.
|
32
32
|
# @!attribute [rw] row_count
|
33
33
|
# @return [::Integer]
|
34
|
-
# The count of
|
35
|
-
# Return 0, if zero rows.
|
34
|
+
# The count of rows scanned.
|
36
35
|
# @!attribute [rw] profile
|
37
36
|
# @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile]
|
38
|
-
#
|
37
|
+
# The profile information per field.
|
39
38
|
# @!attribute [rw] scanned_data
|
40
39
|
# @return [::Google::Cloud::Dataplex::V1::ScannedData]
|
41
|
-
# The data scanned for this
|
40
|
+
# The data scanned for this result.
|
42
41
|
class DataProfileResult
|
43
42
|
include ::Google::Protobuf::MessageExts
|
44
43
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
45
44
|
|
46
|
-
#
|
47
|
-
# and contains the profile info.
|
45
|
+
# Contains name, type, mode and field type specific profile information.
|
48
46
|
# @!attribute [rw] fields
|
49
47
|
# @return [::Array<::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field>]
|
50
|
-
#
|
48
|
+
# List of fields with structural and profile information for each field.
|
51
49
|
class Profile
|
52
50
|
include ::Google::Protobuf::MessageExts
|
53
51
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
54
52
|
|
55
|
-
#
|
53
|
+
# A field within a table.
|
56
54
|
# @!attribute [rw] name
|
57
55
|
# @return [::String]
|
58
56
|
# The name of the field.
|
59
57
|
# @!attribute [rw] type
|
60
58
|
# @return [::String]
|
61
|
-
# The
|
62
|
-
#
|
63
|
-
#
|
64
|
-
#
|
65
|
-
#
|
66
|
-
# * INT32
|
67
|
-
# * INT16
|
68
|
-
# * DOUBLE
|
69
|
-
# * FLOAT
|
70
|
-
# * DECIMAL
|
71
|
-
# * BOOLEAN
|
72
|
-
# * BINARY
|
73
|
-
# * TIMESTAMP
|
74
|
-
# * DATE
|
75
|
-
# * TIME
|
76
|
-
# * NULL
|
77
|
-
# * RECORD
|
59
|
+
# The data type retrieved from the schema of the data source. For
|
60
|
+
# instance, for a BigQuery native table, it is the [BigQuery Table
|
61
|
+
# Schema](https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#tablefieldschema).
|
62
|
+
# For a Dataplex Entity, it is the [Entity
|
63
|
+
# Schema](https://cloud.google.com/dataplex/docs/reference/rpc/google.cloud.dataplex.v1#type_3).
|
78
64
|
# @!attribute [rw] mode
|
79
65
|
# @return [::String]
|
80
|
-
# The mode of the field.
|
81
|
-
#
|
82
|
-
#
|
83
|
-
#
|
66
|
+
# The mode of the field. Possible values include:
|
67
|
+
#
|
68
|
+
# * REQUIRED, if it is a required field.
|
69
|
+
# * NULLABLE, if it is an optional field.
|
70
|
+
# * REPEATED, if it is a repeated field.
|
84
71
|
# @!attribute [rw] profile
|
85
72
|
# @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo]
|
86
|
-
#
|
73
|
+
# Profile information for the corresponding field.
|
87
74
|
class Field
|
88
75
|
include ::Google::Protobuf::MessageExts
|
89
76
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
90
77
|
|
91
|
-
#
|
78
|
+
# The profile information for each field type.
|
92
79
|
# @!attribute [rw] null_ratio
|
93
80
|
# @return [::Float]
|
94
|
-
#
|
81
|
+
# Ratio of rows with null value against total scanned rows.
|
95
82
|
# @!attribute [rw] distinct_ratio
|
96
83
|
# @return [::Float]
|
97
|
-
#
|
98
|
-
#
|
84
|
+
# Ratio of rows with distinct values against total scanned rows.
|
85
|
+
# Not available for complex non-groupable field type RECORD and fields
|
86
|
+
# with REPEATABLE mode.
|
99
87
|
# @!attribute [rw] top_n_values
|
100
88
|
# @return [::Array<::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo::TopNValue>]
|
101
|
-
# The
|
102
|
-
#
|
103
|
-
# whichever is smaller.
|
104
|
-
#
|
89
|
+
# The list of top N non-null values and number of times they occur in
|
90
|
+
# the scanned data. N is 10 or equal to the number of distinct values
|
91
|
+
# in the field, whichever is smaller. Not available for complex
|
92
|
+
# non-groupable field type RECORD and fields with REPEATABLE mode.
|
105
93
|
# @!attribute [rw] string_profile
|
106
94
|
# @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo::StringFieldInfo]
|
107
|
-
#
|
95
|
+
# String type field information.
|
108
96
|
# @!attribute [rw] integer_profile
|
109
97
|
# @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo::IntegerFieldInfo]
|
110
|
-
#
|
98
|
+
# Integer type field information.
|
111
99
|
# @!attribute [rw] double_profile
|
112
100
|
# @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo::DoubleFieldInfo]
|
113
|
-
#
|
101
|
+
# Double type field information.
|
114
102
|
class ProfileInfo
|
115
103
|
include ::Google::Protobuf::MessageExts
|
116
104
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
117
105
|
|
118
|
-
#
|
106
|
+
# The profile information for a string type field.
|
119
107
|
# @!attribute [rw] min_length
|
120
108
|
# @return [::Integer]
|
121
|
-
#
|
122
|
-
# Optional if zero non-null rows.
|
109
|
+
# Minimum length of non-null values in the scanned data.
|
123
110
|
# @!attribute [rw] max_length
|
124
111
|
# @return [::Integer]
|
125
|
-
#
|
126
|
-
# Optional if zero non-null rows.
|
112
|
+
# Maximum length of non-null values in the scanned data.
|
127
113
|
# @!attribute [rw] average_length
|
128
114
|
# @return [::Float]
|
129
|
-
#
|
130
|
-
# Optional if zero non-null rows.
|
115
|
+
# Average length of non-null values in the scanned data.
|
131
116
|
class StringFieldInfo
|
132
117
|
include ::Google::Protobuf::MessageExts
|
133
118
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
134
119
|
end
|
135
120
|
|
136
|
-
#
|
121
|
+
# The profile information for an integer type field.
|
137
122
|
# @!attribute [rw] average
|
138
123
|
# @return [::Float]
|
139
|
-
#
|
140
|
-
#
|
141
|
-
# rows.
|
124
|
+
# Average of non-null values in the scanned data. NaN, if the field
|
125
|
+
# has a NaN.
|
142
126
|
# @!attribute [rw] standard_deviation
|
143
127
|
# @return [::Float]
|
144
|
-
#
|
145
|
-
#
|
146
|
-
# rows.
|
128
|
+
# Standard deviation of non-null values in the scanned data. NaN, if
|
129
|
+
# the field has a NaN.
|
147
130
|
# @!attribute [rw] min
|
148
131
|
# @return [::Integer]
|
149
|
-
#
|
150
|
-
#
|
151
|
-
# rows.
|
132
|
+
# Minimum of non-null values in the scanned data. NaN, if the field
|
133
|
+
# has a NaN.
|
152
134
|
# @!attribute [rw] quartiles
|
153
135
|
# @return [::Array<::Integer>]
|
154
|
-
# A quartile
|
136
|
+
# A quartile divides the number of data points into four parts, or
|
155
137
|
# quarters, of more-or-less equal size. Three main quartiles used
|
156
138
|
# are: The first quartile (Q1) splits off the lowest 25% of data from
|
157
139
|
# the highest 75%. It is also known as the lower or 25th empirical
|
@@ -159,37 +141,34 @@ module Google
|
|
159
141
|
# quartile (Q2) is the median of a data set. So, 50% of the data lies
|
160
142
|
# below this point. The third quartile (Q3) splits off the highest
|
161
143
|
# 25% of data from the lowest 75%. It is known as the upper or 75th
|
162
|
-
# empirical quartile, as 75% of the data lies below this point.
|
163
|
-
#
|
164
|
-
# values, occurring in order Q1, median, Q3.
|
144
|
+
# empirical quartile, as 75% of the data lies below this point.
|
145
|
+
# Here, the quartiles is provided as an ordered list of quartile
|
146
|
+
# values for the scanned data, occurring in order Q1, median, Q3.
|
165
147
|
# @!attribute [rw] max
|
166
148
|
# @return [::Integer]
|
167
|
-
#
|
168
|
-
#
|
169
|
-
# rows.
|
149
|
+
# Maximum of non-null values in the scanned data. NaN, if the field
|
150
|
+
# has a NaN.
|
170
151
|
class IntegerFieldInfo
|
171
152
|
include ::Google::Protobuf::MessageExts
|
172
153
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
173
154
|
end
|
174
155
|
|
175
|
-
#
|
156
|
+
# The profile information for a double type field.
|
176
157
|
# @!attribute [rw] average
|
177
158
|
# @return [::Float]
|
178
|
-
#
|
179
|
-
#
|
159
|
+
# Average of non-null values in the scanned data. NaN, if the field
|
160
|
+
# has a NaN.
|
180
161
|
# @!attribute [rw] standard_deviation
|
181
162
|
# @return [::Float]
|
182
|
-
#
|
183
|
-
#
|
184
|
-
# rows.
|
163
|
+
# Standard deviation of non-null values in the scanned data. NaN, if
|
164
|
+
# the field has a NaN.
|
185
165
|
# @!attribute [rw] min
|
186
166
|
# @return [::Float]
|
187
|
-
#
|
188
|
-
#
|
189
|
-
# rows.
|
167
|
+
# Minimum of non-null values in the scanned data. NaN, if the field
|
168
|
+
# has a NaN.
|
190
169
|
# @!attribute [rw] quartiles
|
191
170
|
# @return [::Array<::Float>]
|
192
|
-
# A quartile
|
171
|
+
# A quartile divides the number of data points into four parts, or
|
193
172
|
# quarters, of more-or-less equal size. Three main quartiles used
|
194
173
|
# are: The first quartile (Q1) splits off the lowest 25% of data from
|
195
174
|
# the highest 75%. It is also known as the lower or 25th empirical
|
@@ -197,27 +176,25 @@ module Google
|
|
197
176
|
# quartile (Q2) is the median of a data set. So, 50% of the data lies
|
198
177
|
# below this point. The third quartile (Q3) splits off the highest
|
199
178
|
# 25% of data from the lowest 75%. It is known as the upper or 75th
|
200
|
-
# empirical quartile, as 75% of the data lies below this point.
|
201
|
-
#
|
202
|
-
# values, occurring in order Q1, median, Q3.
|
179
|
+
# empirical quartile, as 75% of the data lies below this point.
|
180
|
+
# Here, the quartiles is provided as an ordered list of quartile
|
181
|
+
# values for the scanned data, occurring in order Q1, median, Q3.
|
203
182
|
# @!attribute [rw] max
|
204
183
|
# @return [::Float]
|
205
|
-
#
|
206
|
-
#
|
207
|
-
# rows.
|
184
|
+
# Maximum of non-null values in the scanned data. NaN, if the field
|
185
|
+
# has a NaN.
|
208
186
|
class DoubleFieldInfo
|
209
187
|
include ::Google::Protobuf::MessageExts
|
210
188
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
211
189
|
end
|
212
190
|
|
213
|
-
#
|
214
|
-
# field.
|
191
|
+
# Top N non-null values in the scanned data.
|
215
192
|
# @!attribute [rw] value
|
216
193
|
# @return [::String]
|
217
|
-
#
|
194
|
+
# String value of a top N non-null value.
|
218
195
|
# @!attribute [rw] count
|
219
196
|
# @return [::Integer]
|
220
|
-
#
|
197
|
+
# Count of the corresponding value in the scanned data.
|
221
198
|
class TopNValue
|
222
199
|
include ::Google::Protobuf::MessageExts
|
223
200
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -37,7 +37,7 @@ module Google
|
|
37
37
|
# Overall data quality result -- `true` if all rules passed.
|
38
38
|
# @!attribute [rw] dimensions
|
39
39
|
# @return [::Array<::Google::Cloud::Dataplex::V1::DataQualityDimensionResult>]
|
40
|
-
# A list of results at the dimension
|
40
|
+
# A list of results at the dimension level.
|
41
41
|
# @!attribute [rw] rules
|
42
42
|
# @return [::Array<::Google::Cloud::Dataplex::V1::DataQualityRuleResult>]
|
43
43
|
# A list of all the rules in a job, and their results.
|
@@ -52,8 +52,7 @@ module Google
|
|
52
52
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
53
53
|
end
|
54
54
|
|
55
|
-
# DataQualityRuleResult provides a more detailed, per-rule
|
56
|
-
# results.
|
55
|
+
# DataQualityRuleResult provides a more detailed, per-rule view of the results.
|
57
56
|
# @!attribute [rw] rule
|
58
57
|
# @return [::Google::Cloud::Dataplex::V1::DataQualityRule]
|
59
58
|
# The rule specified in the DataQualitySpec, as is.
|
@@ -62,12 +61,15 @@ module Google
|
|
62
61
|
# Whether the rule passed or failed.
|
63
62
|
# @!attribute [rw] evaluated_count
|
64
63
|
# @return [::Integer]
|
65
|
-
# The number of rows a rule was evaluated against.
|
66
|
-
#
|
64
|
+
# The number of rows a rule was evaluated against. This field is only valid
|
65
|
+
# for ColumnMap type rules.
|
66
|
+
#
|
67
67
|
# Evaluated count can be configured to either
|
68
|
-
#
|
69
|
-
#
|
70
|
-
#
|
68
|
+
#
|
69
|
+
# * include all rows (default) - with `null` rows automatically failing rule
|
70
|
+
# evaluation, or
|
71
|
+
# * exclude `null` rows from the `evaluated_count`, by setting
|
72
|
+
# `ignore_nulls = true`.
|
71
73
|
# @!attribute [rw] passed_count
|
72
74
|
# @return [::Integer]
|
73
75
|
# The number of rows which passed a rule evaluation.
|
@@ -77,7 +79,7 @@ module Google
|
|
77
79
|
# The number of rows with null values in the specified column.
|
78
80
|
# @!attribute [rw] pass_ratio
|
79
81
|
# @return [::Float]
|
80
|
-
# The ratio of passed_count / evaluated_count
|
82
|
+
# The ratio of **passed_count / evaluated_count**.
|
81
83
|
# This field is only valid for ColumnMap type rules.
|
82
84
|
# @!attribute [rw] failing_rows_query
|
83
85
|
# @return [::String]
|
@@ -88,8 +90,8 @@ module Google
|
|
88
90
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
89
91
|
end
|
90
92
|
|
91
|
-
# DataQualityDimensionResult provides a more detailed, per-dimension
|
92
|
-
#
|
93
|
+
# DataQualityDimensionResult provides a more detailed, per-dimension view of
|
94
|
+
# the results.
|
93
95
|
# @!attribute [rw] passed
|
94
96
|
# @return [::Boolean]
|
95
97
|
# Whether the dimension passed or failed.
|
@@ -133,20 +135,22 @@ module Google
|
|
133
135
|
# Optional. The unnested column which this rule is evaluated against.
|
134
136
|
# @!attribute [rw] ignore_null
|
135
137
|
# @return [::Boolean]
|
136
|
-
# Optional. Rows with null values will automatically fail a rule, unless
|
137
|
-
# ignore_null is true
|
138
|
-
# passing.
|
138
|
+
# Optional. Rows with `null` values will automatically fail a rule, unless
|
139
|
+
# `ignore_null` is `true`. In that case, such `null` rows are trivially
|
140
|
+
# considered passing.
|
141
|
+
#
|
142
|
+
# Only applicable to ColumnMap rules.
|
139
143
|
# @!attribute [rw] dimension
|
140
144
|
# @return [::String]
|
141
145
|
# Required. The dimension a rule belongs to. Results are also aggregated at
|
142
|
-
# the dimension
|
143
|
-
# "CONSISTENCY", "VALIDITY", "UNIQUENESS", "INTEGRITY"]
|
146
|
+
# the dimension level. Supported dimensions are **["COMPLETENESS",
|
147
|
+
# "ACCURACY", "CONSISTENCY", "VALIDITY", "UNIQUENESS", "INTEGRITY"]**
|
144
148
|
# @!attribute [rw] threshold
|
145
149
|
# @return [::Float]
|
146
|
-
# Optional. The minimum ratio of passing_rows / total_rows required to
|
147
|
-
# this rule, with a range of [0.0, 1.0]
|
150
|
+
# Optional. The minimum ratio of **passing_rows / total_rows** required to
|
151
|
+
# pass this rule, with a range of [0.0, 1.0].
|
148
152
|
#
|
149
|
-
# 0 indicates default value (i.e. 1.0)
|
153
|
+
# 0 indicates default value (i.e. 1.0).
|
150
154
|
class DataQualityRule
|
151
155
|
include ::Google::Protobuf::MessageExts
|
152
156
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -155,21 +159,25 @@ module Google
|
|
155
159
|
# @!attribute [rw] min_value
|
156
160
|
# @return [::String]
|
157
161
|
# Optional. The minimum column value allowed for a row to pass this
|
158
|
-
# validation. At least one of min_value and max_value need to be
|
162
|
+
# validation. At least one of `min_value` and `max_value` need to be
|
163
|
+
# provided.
|
159
164
|
# @!attribute [rw] max_value
|
160
165
|
# @return [::String]
|
161
166
|
# Optional. The maximum column value allowed for a row to pass this
|
162
|
-
# validation. At least one of min_value and max_value need to be
|
167
|
+
# validation. At least one of `min_value` and `max_value` need to be
|
168
|
+
# provided.
|
163
169
|
# @!attribute [rw] strict_min_enabled
|
164
170
|
# @return [::Boolean]
|
165
171
|
# Optional. Whether each value needs to be strictly greater than ('>') the
|
166
|
-
# minimum, or if equality is allowed.
|
167
|
-
#
|
172
|
+
# minimum, or if equality is allowed.
|
173
|
+
#
|
174
|
+
# Only relevant if a `min_value` has been defined. Default = false.
|
168
175
|
# @!attribute [rw] strict_max_enabled
|
169
176
|
# @return [::Boolean]
|
170
177
|
# Optional. Whether each value needs to be strictly lesser than ('<') the
|
171
|
-
# maximum, or if equality is allowed.
|
172
|
-
#
|
178
|
+
# maximum, or if equality is allowed.
|
179
|
+
#
|
180
|
+
# Only relevant if a `max_value` has been defined. Default = false.
|
173
181
|
class RangeExpectation
|
174
182
|
include ::Google::Protobuf::MessageExts
|
175
183
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -184,6 +192,7 @@ module Google
|
|
184
192
|
# Evaluates whether each column value is contained by a specified set.
|
185
193
|
# @!attribute [rw] values
|
186
194
|
# @return [::Array<::String>]
|
195
|
+
# Expected values for the column value.
|
187
196
|
class SetExpectation
|
188
197
|
include ::Google::Protobuf::MessageExts
|
189
198
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -192,6 +201,7 @@ module Google
|
|
192
201
|
# Evaluates whether each column value matches a specified regex.
|
193
202
|
# @!attribute [rw] regex
|
194
203
|
# @return [::String]
|
204
|
+
# A regular expression the column value is expected to match.
|
195
205
|
class RegexExpectation
|
196
206
|
include ::Google::Protobuf::MessageExts
|
197
207
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -207,30 +217,36 @@ module Google
|
|
207
217
|
# range.
|
208
218
|
# @!attribute [rw] statistic
|
209
219
|
# @return [::Google::Cloud::Dataplex::V1::DataQualityRule::StatisticRangeExpectation::ColumnStatistic]
|
220
|
+
# The aggregate metric to evaluate.
|
210
221
|
# @!attribute [rw] min_value
|
211
222
|
# @return [::String]
|
212
223
|
# The minimum column statistic value allowed for a row to pass this
|
213
224
|
# validation.
|
214
|
-
#
|
225
|
+
#
|
226
|
+
# At least one of `min_value` and `max_value` need to be provided.
|
215
227
|
# @!attribute [rw] max_value
|
216
228
|
# @return [::String]
|
217
229
|
# The maximum column statistic value allowed for a row to pass this
|
218
230
|
# validation.
|
219
|
-
#
|
231
|
+
#
|
232
|
+
# At least one of `min_value` and `max_value` need to be provided.
|
220
233
|
# @!attribute [rw] strict_min_enabled
|
221
234
|
# @return [::Boolean]
|
222
235
|
# Whether column statistic needs to be strictly greater than ('>')
|
223
|
-
# the minimum, or if equality is allowed.
|
224
|
-
#
|
236
|
+
# the minimum, or if equality is allowed.
|
237
|
+
#
|
238
|
+
# Only relevant if a `min_value` has been defined. Default = false.
|
225
239
|
# @!attribute [rw] strict_max_enabled
|
226
240
|
# @return [::Boolean]
|
227
241
|
# Whether column statistic needs to be strictly lesser than ('<') the
|
228
|
-
# maximum, or if equality is allowed.
|
229
|
-
#
|
242
|
+
# maximum, or if equality is allowed.
|
243
|
+
#
|
244
|
+
# Only relevant if a `max_value` has been defined. Default = false.
|
230
245
|
class StatisticRangeExpectation
|
231
246
|
include ::Google::Protobuf::MessageExts
|
232
247
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
233
248
|
|
249
|
+
# The list of aggregate metrics a rule can be evaluated against.
|
234
250
|
module ColumnStatistic
|
235
251
|
# Unspecified statistic type
|
236
252
|
STATISTIC_UNDEFINED = 0
|
@@ -247,22 +263,28 @@ module Google
|
|
247
263
|
end
|
248
264
|
|
249
265
|
# Evaluates whether each row passes the specified condition.
|
266
|
+
#
|
250
267
|
# The SQL expression needs to use BigQuery standard SQL syntax and should
|
251
|
-
# produce a boolean per row as the result.
|
268
|
+
# produce a boolean value per row as the result.
|
269
|
+
#
|
252
270
|
# Example: col1 >= 0 AND col2 < 10
|
253
271
|
# @!attribute [rw] sql_expression
|
254
272
|
# @return [::String]
|
273
|
+
# The SQL expression.
|
255
274
|
class RowConditionExpectation
|
256
275
|
include ::Google::Protobuf::MessageExts
|
257
276
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
258
277
|
end
|
259
278
|
|
260
279
|
# Evaluates whether the provided expression is true.
|
280
|
+
#
|
261
281
|
# The SQL expression needs to use BigQuery standard SQL syntax and should
|
262
282
|
# produce a scalar boolean result.
|
283
|
+
#
|
263
284
|
# Example: MIN(col1) >= 0
|
264
285
|
# @!attribute [rw] sql_expression
|
265
286
|
# @return [::String]
|
287
|
+
# The SQL expression.
|
266
288
|
class TableConditionExpectation
|
267
289
|
include ::Google::Protobuf::MessageExts
|
268
290
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -40,6 +40,10 @@ module Google
|
|
40
40
|
# * Must end with a number or a letter.
|
41
41
|
# * Must be between 1-63 characters.
|
42
42
|
# * Must be unique within the customer project / location.
|
43
|
+
# @!attribute [rw] validate_only
|
44
|
+
# @return [::Boolean]
|
45
|
+
# Optional. Only validate the request, but do not perform mutations.
|
46
|
+
# The default is `false`.
|
43
47
|
class CreateDataScanRequest
|
44
48
|
include ::Google::Protobuf::MessageExts
|
45
49
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -54,6 +58,10 @@ module Google
|
|
54
58
|
# @!attribute [rw] update_mask
|
55
59
|
# @return [::Google::Protobuf::FieldMask]
|
56
60
|
# Required. Mask of fields to update.
|
61
|
+
# @!attribute [rw] validate_only
|
62
|
+
# @return [::Boolean]
|
63
|
+
# Optional. Only validate the request, but do not perform mutations.
|
64
|
+
# The default is `false`.
|
57
65
|
class UpdateDataScanRequest
|
58
66
|
include ::Google::Protobuf::MessageExts
|
59
67
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -108,7 +116,7 @@ module Google
|
|
108
116
|
# @!attribute [rw] page_size
|
109
117
|
# @return [::Integer]
|
110
118
|
# Optional. Maximum number of dataScans to return. The service may return
|
111
|
-
# fewer than this value. If unspecified, at most
|
119
|
+
# fewer than this value. If unspecified, at most 500 scans will be returned.
|
112
120
|
# The maximum value is 1000; values above 1000 will be coerced to 1000.
|
113
121
|
# @!attribute [rw] page_token
|
114
122
|
# @return [::String]
|
@@ -171,7 +179,7 @@ module Google
|
|
171
179
|
# @!attribute [rw] name
|
172
180
|
# @return [::String]
|
173
181
|
# Required. The resource name of the DataScanJob:
|
174
|
-
# `projects/{project}/locations/{location_id}/dataScans/{data_scan_id}/
|
182
|
+
# `projects/{project}/locations/{location_id}/dataScans/{data_scan_id}/jobs/{data_scan_job_id}`
|
175
183
|
# where `project` refers to a *project_id* or *project_number* and
|
176
184
|
# `location_id` refers to a GCP region.
|
177
185
|
# @!attribute [rw] view
|
@@ -283,8 +283,7 @@ module Google
|
|
283
283
|
# published table name. Specifying a new ID in an update entity
|
284
284
|
# request will override the existing value.
|
285
285
|
# The ID must contain only letters (a-z, A-Z), numbers (0-9), and
|
286
|
-
# underscores
|
287
|
-
# characters.
|
286
|
+
# underscores, and consist of 256 or fewer characters.
|
288
287
|
# @!attribute [rw] etag
|
289
288
|
# @return [::String]
|
290
289
|
# Optional. The etag associated with the entity, which can be retrieved with
|
@@ -322,6 +321,14 @@ module Google
|
|
322
321
|
# @!attribute [r] compatibility
|
323
322
|
# @return [::Google::Cloud::Dataplex::V1::Entity::CompatibilityStatus]
|
324
323
|
# Output only. Metadata stores that the entity is compatible with.
|
324
|
+
# @!attribute [r] access
|
325
|
+
# @return [::Google::Cloud::Dataplex::V1::StorageAccess]
|
326
|
+
# Output only. Identifies the access mechanism to the entity. Not user
|
327
|
+
# settable.
|
328
|
+
# @!attribute [r] uid
|
329
|
+
# @return [::String]
|
330
|
+
# Output only. System generated unique ID for the Entity. This ID will be
|
331
|
+
# different if the Entity is deleted and re-created with the same name.
|
325
332
|
# @!attribute [rw] schema
|
326
333
|
# @return [::Google::Cloud::Dataplex::V1::Schema]
|
327
334
|
# Required. The description of the data structure and layout.
|
@@ -685,6 +692,28 @@ module Google
|
|
685
692
|
end
|
686
693
|
end
|
687
694
|
|
695
|
+
# Describes the access mechanism of the data within its storage location.
|
696
|
+
# @!attribute [r] read
|
697
|
+
# @return [::Google::Cloud::Dataplex::V1::StorageAccess::AccessMode]
|
698
|
+
# Output only. Describes the read access mechanism of the data. Not user
|
699
|
+
# settable.
|
700
|
+
class StorageAccess
|
701
|
+
include ::Google::Protobuf::MessageExts
|
702
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
703
|
+
|
704
|
+
# Access Mode determines how data stored within the Entity is read.
|
705
|
+
module AccessMode
|
706
|
+
# Access mode unspecified.
|
707
|
+
ACCESS_MODE_UNSPECIFIED = 0
|
708
|
+
|
709
|
+
# Default. Data is accessed directly using storage APIs.
|
710
|
+
DIRECT = 1
|
711
|
+
|
712
|
+
# Data is accessed through a managed interface using BigQuery APIs.
|
713
|
+
MANAGED = 2
|
714
|
+
end
|
715
|
+
end
|
716
|
+
|
688
717
|
# Identifies the cloud system that manages the data storage.
|
689
718
|
module StorageSystem
|
690
719
|
# Storage system unspecified.
|
@@ -24,7 +24,7 @@ module Google
|
|
24
24
|
# DataScan scheduling and trigger settings.
|
25
25
|
# @!attribute [rw] on_demand
|
26
26
|
# @return [::Google::Cloud::Dataplex::V1::Trigger::OnDemand]
|
27
|
-
# The scan runs
|
27
|
+
# The scan runs once via `RunDataScan` API.
|
28
28
|
# @!attribute [rw] schedule
|
29
29
|
# @return [::Google::Cloud::Dataplex::V1::Trigger::Schedule]
|
30
30
|
# The scan is scheduled to run periodically.
|
@@ -32,7 +32,7 @@ module Google
|
|
32
32
|
include ::Google::Protobuf::MessageExts
|
33
33
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
34
34
|
|
35
|
-
# The scan runs
|
35
|
+
# The scan runs once via `RunDataScan` API.
|
36
36
|
class OnDemand
|
37
37
|
include ::Google::Protobuf::MessageExts
|
38
38
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -41,13 +41,17 @@ module Google
|
|
41
41
|
# The scan is scheduled to run periodically.
|
42
42
|
# @!attribute [rw] cron
|
43
43
|
# @return [::String]
|
44
|
-
# Required. Cron
|
44
|
+
# Required. [Cron](https://en.wikipedia.org/wiki/Cron) schedule for running
|
45
45
|
# scans periodically.
|
46
|
-
#
|
47
|
-
# cron tab
|
48
|
-
#
|
49
|
-
#
|
50
|
-
#
|
46
|
+
#
|
47
|
+
# To explicitly set a timezone in the cron tab, apply a prefix in the
|
48
|
+
# cron tab: **"CRON_TZ=$\\{IANA_TIME_ZONE}"** or **"TZ=$\\{IANA_TIME_ZONE}"**.
|
49
|
+
# The **$\\{IANA_TIME_ZONE}** may only be a valid string from IANA time zone
|
50
|
+
# database
|
51
|
+
# ([wikipedia](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List)).
|
52
|
+
# For example, `CRON_TZ=America/New_York 1 * * * *`, or
|
53
|
+
# `TZ=America/New_York 1 * * * *`.
|
54
|
+
#
|
51
55
|
# This field is required for Schedule scans.
|
52
56
|
class Schedule
|
53
57
|
include ::Google::Protobuf::MessageExts
|
@@ -58,9 +62,15 @@ module Google
|
|
58
62
|
# The data source for DataScan.
|
59
63
|
# @!attribute [rw] entity
|
60
64
|
# @return [::String]
|
61
|
-
# Immutable. The
|
62
|
-
# the form:
|
65
|
+
# Immutable. The Dataplex entity that represents the data source (e.g.
|
66
|
+
# BigQuery table) for DataScan, of the form:
|
63
67
|
# `projects/{project_number}/locations/{location_id}/lakes/{lake_id}/zones/{zone_id}/entities/{entity_id}`.
|
68
|
+
# @!attribute [rw] resource
|
69
|
+
# @return [::String]
|
70
|
+
# Immutable. The service-qualified full resource name of the cloud resource
|
71
|
+
# for a DataScan job to scan against. The field could be: BigQuery table of
|
72
|
+
# type "TABLE" for DataProfileScan/DataQualityScan Format:
|
73
|
+
# //bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID
|
64
74
|
class DataSource
|
65
75
|
include ::Google::Protobuf::MessageExts
|
66
76
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -78,13 +88,13 @@ module Google
|
|
78
88
|
# @!attribute [rw] field
|
79
89
|
# @return [::String]
|
80
90
|
# The field that contains values which monotonically increases over time
|
81
|
-
# (e.g. timestamp).
|
91
|
+
# (e.g. a timestamp column).
|
82
92
|
# @!attribute [rw] start
|
83
93
|
# @return [::String]
|
84
|
-
# Value that marks the start of the range
|
94
|
+
# Value that marks the start of the range.
|
85
95
|
# @!attribute [rw] end
|
86
96
|
# @return [::String]
|
87
|
-
# Value that marks the end of the range
|
97
|
+
# Value that marks the end of the range.
|
88
98
|
class IncrementalField
|
89
99
|
include ::Google::Protobuf::MessageExts
|
90
100
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|