google-cloud-dataplex-v1 0.7.0 → 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/google/cloud/dataplex/v1/analyze_pb.rb +26 -81
- data/lib/google/cloud/dataplex/v1/content_pb.rb +26 -33
- data/lib/google/cloud/dataplex/v1/content_service/client.rb +6 -4
- data/lib/google/cloud/dataplex/v1/content_service/rest/client.rb +6 -4
- data/lib/google/cloud/dataplex/v1/content_service/rest/service_stub.rb +8 -8
- data/lib/google/cloud/dataplex/v1/data_profile_pb.rb +26 -50
- data/lib/google/cloud/dataplex/v1/data_quality_pb.rb +25 -74
- data/lib/google/cloud/dataplex/v1/data_scan_service/client.rb +16 -8
- data/lib/google/cloud/dataplex/v1/data_scan_service/operations.rb +5 -3
- data/lib/google/cloud/dataplex/v1/data_scan_service/rest/client.rb +16 -8
- data/lib/google/cloud/dataplex/v1/data_scan_service/rest/operations.rb +9 -7
- data/lib/google/cloud/dataplex/v1/data_scan_service/rest/service_stub.rb +8 -8
- data/lib/google/cloud/dataplex/v1/dataplex_service/client.rb +99 -4
- data/lib/google/cloud/dataplex/v1/dataplex_service/operations.rb +5 -3
- data/lib/google/cloud/dataplex/v1/dataplex_service/rest/client.rb +76 -4
- data/lib/google/cloud/dataplex/v1/dataplex_service/rest/operations.rb +9 -7
- data/lib/google/cloud/dataplex/v1/dataplex_service/rest/service_stub.rb +92 -32
- data/lib/google/cloud/dataplex/v1/datascans_pb.rb +31 -121
- data/lib/google/cloud/dataplex/v1/logs_pb.rb +26 -151
- data/lib/google/cloud/dataplex/v1/metadata_pb.rb +27 -191
- data/lib/google/cloud/dataplex/v1/metadata_service/client.rb +6 -4
- data/lib/google/cloud/dataplex/v1/metadata_service/rest/client.rb +6 -4
- data/lib/google/cloud/dataplex/v1/metadata_service/rest/service_stub.rb +9 -9
- data/lib/google/cloud/dataplex/v1/processing_pb.rb +24 -27
- data/lib/google/cloud/dataplex/v1/resources_pb.rb +27 -238
- data/lib/google/cloud/dataplex/v1/rest.rb +1 -1
- data/lib/google/cloud/dataplex/v1/service_pb.rb +31 -196
- data/lib/google/cloud/dataplex/v1/service_services_pb.rb +2 -0
- data/lib/google/cloud/dataplex/v1/tasks_pb.rb +26 -113
- data/lib/google/cloud/dataplex/v1/version.rb +1 -1
- data/lib/google/cloud/dataplex/v1.rb +1 -1
- data/proto_docs/google/api/client.rb +67 -4
- data/proto_docs/google/cloud/dataplex/v1/analyze.rb +4 -0
- data/proto_docs/google/cloud/dataplex/v1/data_profile.rb +64 -87
- data/proto_docs/google/cloud/dataplex/v1/data_quality.rb +54 -32
- data/proto_docs/google/cloud/dataplex/v1/datascans.rb +10 -2
- data/proto_docs/google/cloud/dataplex/v1/metadata.rb +31 -2
- data/proto_docs/google/cloud/dataplex/v1/processing.rb +23 -13
- data/proto_docs/google/cloud/dataplex/v1/resources.rb +24 -6
- data/proto_docs/google/cloud/dataplex/v1/service.rb +17 -0
- data/proto_docs/google/cloud/dataplex/v1/tasks.rb +2 -3
- data/proto_docs/google/protobuf/any.rb +7 -4
- data/proto_docs/google/protobuf/timestamp.rb +1 -3
- metadata +4 -4
@@ -27,131 +27,113 @@ module Google
|
|
27
27
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
28
28
|
end
|
29
29
|
|
30
|
-
# DataProfileResult defines the output of DataProfileScan.
|
31
|
-
#
|
30
|
+
# DataProfileResult defines the output of DataProfileScan. Each field of the
|
31
|
+
# table will have field type specific profile result.
|
32
32
|
# @!attribute [rw] row_count
|
33
33
|
# @return [::Integer]
|
34
|
-
# The count of
|
35
|
-
# Return 0, if zero rows.
|
34
|
+
# The count of rows scanned.
|
36
35
|
# @!attribute [rw] profile
|
37
36
|
# @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile]
|
38
|
-
#
|
37
|
+
# The profile information per field.
|
39
38
|
# @!attribute [rw] scanned_data
|
40
39
|
# @return [::Google::Cloud::Dataplex::V1::ScannedData]
|
41
|
-
# The data scanned for this
|
40
|
+
# The data scanned for this result.
|
42
41
|
class DataProfileResult
|
43
42
|
include ::Google::Protobuf::MessageExts
|
44
43
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
45
44
|
|
46
|
-
#
|
47
|
-
# and contains the profile info.
|
45
|
+
# Contains name, type, mode and field type specific profile information.
|
48
46
|
# @!attribute [rw] fields
|
49
47
|
# @return [::Array<::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field>]
|
50
|
-
#
|
48
|
+
# List of fields with structural and profile information for each field.
|
51
49
|
class Profile
|
52
50
|
include ::Google::Protobuf::MessageExts
|
53
51
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
54
52
|
|
55
|
-
#
|
53
|
+
# A field within a table.
|
56
54
|
# @!attribute [rw] name
|
57
55
|
# @return [::String]
|
58
56
|
# The name of the field.
|
59
57
|
# @!attribute [rw] type
|
60
58
|
# @return [::String]
|
61
|
-
# The
|
62
|
-
#
|
63
|
-
#
|
64
|
-
#
|
65
|
-
#
|
66
|
-
# * INT32
|
67
|
-
# * INT16
|
68
|
-
# * DOUBLE
|
69
|
-
# * FLOAT
|
70
|
-
# * DECIMAL
|
71
|
-
# * BOOLEAN
|
72
|
-
# * BINARY
|
73
|
-
# * TIMESTAMP
|
74
|
-
# * DATE
|
75
|
-
# * TIME
|
76
|
-
# * NULL
|
77
|
-
# * RECORD
|
59
|
+
# The data type retrieved from the schema of the data source. For
|
60
|
+
# instance, for a BigQuery native table, it is the [BigQuery Table
|
61
|
+
# Schema](https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#tablefieldschema).
|
62
|
+
# For a Dataplex Entity, it is the [Entity
|
63
|
+
# Schema](https://cloud.google.com/dataplex/docs/reference/rpc/google.cloud.dataplex.v1#type_3).
|
78
64
|
# @!attribute [rw] mode
|
79
65
|
# @return [::String]
|
80
|
-
# The mode of the field.
|
81
|
-
#
|
82
|
-
#
|
83
|
-
#
|
66
|
+
# The mode of the field. Possible values include:
|
67
|
+
#
|
68
|
+
# * REQUIRED, if it is a required field.
|
69
|
+
# * NULLABLE, if it is an optional field.
|
70
|
+
# * REPEATED, if it is a repeated field.
|
84
71
|
# @!attribute [rw] profile
|
85
72
|
# @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo]
|
86
|
-
#
|
73
|
+
# Profile information for the corresponding field.
|
87
74
|
class Field
|
88
75
|
include ::Google::Protobuf::MessageExts
|
89
76
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
90
77
|
|
91
|
-
#
|
78
|
+
# The profile information for each field type.
|
92
79
|
# @!attribute [rw] null_ratio
|
93
80
|
# @return [::Float]
|
94
|
-
#
|
81
|
+
# Ratio of rows with null value against total scanned rows.
|
95
82
|
# @!attribute [rw] distinct_ratio
|
96
83
|
# @return [::Float]
|
97
|
-
#
|
98
|
-
#
|
84
|
+
# Ratio of rows with distinct values against total scanned rows.
|
85
|
+
# Not available for complex non-groupable field type RECORD and fields
|
86
|
+
# with REPEATABLE mode.
|
99
87
|
# @!attribute [rw] top_n_values
|
100
88
|
# @return [::Array<::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo::TopNValue>]
|
101
|
-
# The
|
102
|
-
#
|
103
|
-
# whichever is smaller.
|
104
|
-
#
|
89
|
+
# The list of top N non-null values and number of times they occur in
|
90
|
+
# the scanned data. N is 10 or equal to the number of distinct values
|
91
|
+
# in the field, whichever is smaller. Not available for complex
|
92
|
+
# non-groupable field type RECORD and fields with REPEATABLE mode.
|
105
93
|
# @!attribute [rw] string_profile
|
106
94
|
# @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo::StringFieldInfo]
|
107
|
-
#
|
95
|
+
# String type field information.
|
108
96
|
# @!attribute [rw] integer_profile
|
109
97
|
# @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo::IntegerFieldInfo]
|
110
|
-
#
|
98
|
+
# Integer type field information.
|
111
99
|
# @!attribute [rw] double_profile
|
112
100
|
# @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo::DoubleFieldInfo]
|
113
|
-
#
|
101
|
+
# Double type field information.
|
114
102
|
class ProfileInfo
|
115
103
|
include ::Google::Protobuf::MessageExts
|
116
104
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
117
105
|
|
118
|
-
#
|
106
|
+
# The profile information for a string type field.
|
119
107
|
# @!attribute [rw] min_length
|
120
108
|
# @return [::Integer]
|
121
|
-
#
|
122
|
-
# Optional if zero non-null rows.
|
109
|
+
# Minimum length of non-null values in the scanned data.
|
123
110
|
# @!attribute [rw] max_length
|
124
111
|
# @return [::Integer]
|
125
|
-
#
|
126
|
-
# Optional if zero non-null rows.
|
112
|
+
# Maximum length of non-null values in the scanned data.
|
127
113
|
# @!attribute [rw] average_length
|
128
114
|
# @return [::Float]
|
129
|
-
#
|
130
|
-
# Optional if zero non-null rows.
|
115
|
+
# Average length of non-null values in the scanned data.
|
131
116
|
class StringFieldInfo
|
132
117
|
include ::Google::Protobuf::MessageExts
|
133
118
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
134
119
|
end
|
135
120
|
|
136
|
-
#
|
121
|
+
# The profile information for an integer type field.
|
137
122
|
# @!attribute [rw] average
|
138
123
|
# @return [::Float]
|
139
|
-
#
|
140
|
-
#
|
141
|
-
# rows.
|
124
|
+
# Average of non-null values in the scanned data. NaN, if the field
|
125
|
+
# has a NaN.
|
142
126
|
# @!attribute [rw] standard_deviation
|
143
127
|
# @return [::Float]
|
144
|
-
#
|
145
|
-
#
|
146
|
-
# rows.
|
128
|
+
# Standard deviation of non-null values in the scanned data. NaN, if
|
129
|
+
# the field has a NaN.
|
147
130
|
# @!attribute [rw] min
|
148
131
|
# @return [::Integer]
|
149
|
-
#
|
150
|
-
#
|
151
|
-
# rows.
|
132
|
+
# Minimum of non-null values in the scanned data. NaN, if the field
|
133
|
+
# has a NaN.
|
152
134
|
# @!attribute [rw] quartiles
|
153
135
|
# @return [::Array<::Integer>]
|
154
|
-
# A quartile
|
136
|
+
# A quartile divides the number of data points into four parts, or
|
155
137
|
# quarters, of more-or-less equal size. Three main quartiles used
|
156
138
|
# are: The first quartile (Q1) splits off the lowest 25% of data from
|
157
139
|
# the highest 75%. It is also known as the lower or 25th empirical
|
@@ -159,37 +141,34 @@ module Google
|
|
159
141
|
# quartile (Q2) is the median of a data set. So, 50% of the data lies
|
160
142
|
# below this point. The third quartile (Q3) splits off the highest
|
161
143
|
# 25% of data from the lowest 75%. It is known as the upper or 75th
|
162
|
-
# empirical quartile, as 75% of the data lies below this point.
|
163
|
-
#
|
164
|
-
# values, occurring in order Q1, median, Q3.
|
144
|
+
# empirical quartile, as 75% of the data lies below this point.
|
145
|
+
# Here, the quartiles is provided as an ordered list of quartile
|
146
|
+
# values for the scanned data, occurring in order Q1, median, Q3.
|
165
147
|
# @!attribute [rw] max
|
166
148
|
# @return [::Integer]
|
167
|
-
#
|
168
|
-
#
|
169
|
-
# rows.
|
149
|
+
# Maximum of non-null values in the scanned data. NaN, if the field
|
150
|
+
# has a NaN.
|
170
151
|
class IntegerFieldInfo
|
171
152
|
include ::Google::Protobuf::MessageExts
|
172
153
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
173
154
|
end
|
174
155
|
|
175
|
-
#
|
156
|
+
# The profile information for a double type field.
|
176
157
|
# @!attribute [rw] average
|
177
158
|
# @return [::Float]
|
178
|
-
#
|
179
|
-
#
|
159
|
+
# Average of non-null values in the scanned data. NaN, if the field
|
160
|
+
# has a NaN.
|
180
161
|
# @!attribute [rw] standard_deviation
|
181
162
|
# @return [::Float]
|
182
|
-
#
|
183
|
-
#
|
184
|
-
# rows.
|
163
|
+
# Standard deviation of non-null values in the scanned data. NaN, if
|
164
|
+
# the field has a NaN.
|
185
165
|
# @!attribute [rw] min
|
186
166
|
# @return [::Float]
|
187
|
-
#
|
188
|
-
#
|
189
|
-
# rows.
|
167
|
+
# Minimum of non-null values in the scanned data. NaN, if the field
|
168
|
+
# has a NaN.
|
190
169
|
# @!attribute [rw] quartiles
|
191
170
|
# @return [::Array<::Float>]
|
192
|
-
# A quartile
|
171
|
+
# A quartile divides the number of data points into four parts, or
|
193
172
|
# quarters, of more-or-less equal size. Three main quartiles used
|
194
173
|
# are: The first quartile (Q1) splits off the lowest 25% of data from
|
195
174
|
# the highest 75%. It is also known as the lower or 25th empirical
|
@@ -197,27 +176,25 @@ module Google
|
|
197
176
|
# quartile (Q2) is the median of a data set. So, 50% of the data lies
|
198
177
|
# below this point. The third quartile (Q3) splits off the highest
|
199
178
|
# 25% of data from the lowest 75%. It is known as the upper or 75th
|
200
|
-
# empirical quartile, as 75% of the data lies below this point.
|
201
|
-
#
|
202
|
-
# values, occurring in order Q1, median, Q3.
|
179
|
+
# empirical quartile, as 75% of the data lies below this point.
|
180
|
+
# Here, the quartiles is provided as an ordered list of quartile
|
181
|
+
# values for the scanned data, occurring in order Q1, median, Q3.
|
203
182
|
# @!attribute [rw] max
|
204
183
|
# @return [::Float]
|
205
|
-
#
|
206
|
-
#
|
207
|
-
# rows.
|
184
|
+
# Maximum of non-null values in the scanned data. NaN, if the field
|
185
|
+
# has a NaN.
|
208
186
|
class DoubleFieldInfo
|
209
187
|
include ::Google::Protobuf::MessageExts
|
210
188
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
211
189
|
end
|
212
190
|
|
213
|
-
#
|
214
|
-
# field.
|
191
|
+
# Top N non-null values in the scanned data.
|
215
192
|
# @!attribute [rw] value
|
216
193
|
# @return [::String]
|
217
|
-
#
|
194
|
+
# String value of a top N non-null value.
|
218
195
|
# @!attribute [rw] count
|
219
196
|
# @return [::Integer]
|
220
|
-
#
|
197
|
+
# Count of the corresponding value in the scanned data.
|
221
198
|
class TopNValue
|
222
199
|
include ::Google::Protobuf::MessageExts
|
223
200
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -37,7 +37,7 @@ module Google
|
|
37
37
|
# Overall data quality result -- `true` if all rules passed.
|
38
38
|
# @!attribute [rw] dimensions
|
39
39
|
# @return [::Array<::Google::Cloud::Dataplex::V1::DataQualityDimensionResult>]
|
40
|
-
# A list of results at the dimension
|
40
|
+
# A list of results at the dimension level.
|
41
41
|
# @!attribute [rw] rules
|
42
42
|
# @return [::Array<::Google::Cloud::Dataplex::V1::DataQualityRuleResult>]
|
43
43
|
# A list of all the rules in a job, and their results.
|
@@ -52,8 +52,7 @@ module Google
|
|
52
52
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
53
53
|
end
|
54
54
|
|
55
|
-
# DataQualityRuleResult provides a more detailed, per-rule
|
56
|
-
# results.
|
55
|
+
# DataQualityRuleResult provides a more detailed, per-rule view of the results.
|
57
56
|
# @!attribute [rw] rule
|
58
57
|
# @return [::Google::Cloud::Dataplex::V1::DataQualityRule]
|
59
58
|
# The rule specified in the DataQualitySpec, as is.
|
@@ -62,12 +61,15 @@ module Google
|
|
62
61
|
# Whether the rule passed or failed.
|
63
62
|
# @!attribute [rw] evaluated_count
|
64
63
|
# @return [::Integer]
|
65
|
-
# The number of rows a rule was evaluated against.
|
66
|
-
#
|
64
|
+
# The number of rows a rule was evaluated against. This field is only valid
|
65
|
+
# for ColumnMap type rules.
|
66
|
+
#
|
67
67
|
# Evaluated count can be configured to either
|
68
|
-
#
|
69
|
-
#
|
70
|
-
#
|
68
|
+
#
|
69
|
+
# * include all rows (default) - with `null` rows automatically failing rule
|
70
|
+
# evaluation, or
|
71
|
+
# * exclude `null` rows from the `evaluated_count`, by setting
|
72
|
+
# `ignore_nulls = true`.
|
71
73
|
# @!attribute [rw] passed_count
|
72
74
|
# @return [::Integer]
|
73
75
|
# The number of rows which passed a rule evaluation.
|
@@ -77,7 +79,7 @@ module Google
|
|
77
79
|
# The number of rows with null values in the specified column.
|
78
80
|
# @!attribute [rw] pass_ratio
|
79
81
|
# @return [::Float]
|
80
|
-
# The ratio of passed_count / evaluated_count
|
82
|
+
# The ratio of **passed_count / evaluated_count**.
|
81
83
|
# This field is only valid for ColumnMap type rules.
|
82
84
|
# @!attribute [rw] failing_rows_query
|
83
85
|
# @return [::String]
|
@@ -88,8 +90,8 @@ module Google
|
|
88
90
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
89
91
|
end
|
90
92
|
|
91
|
-
# DataQualityDimensionResult provides a more detailed, per-dimension
|
92
|
-
#
|
93
|
+
# DataQualityDimensionResult provides a more detailed, per-dimension view of
|
94
|
+
# the results.
|
93
95
|
# @!attribute [rw] passed
|
94
96
|
# @return [::Boolean]
|
95
97
|
# Whether the dimension passed or failed.
|
@@ -133,20 +135,22 @@ module Google
|
|
133
135
|
# Optional. The unnested column which this rule is evaluated against.
|
134
136
|
# @!attribute [rw] ignore_null
|
135
137
|
# @return [::Boolean]
|
136
|
-
# Optional. Rows with null values will automatically fail a rule, unless
|
137
|
-
# ignore_null is true
|
138
|
-
# passing.
|
138
|
+
# Optional. Rows with `null` values will automatically fail a rule, unless
|
139
|
+
# `ignore_null` is `true`. In that case, such `null` rows are trivially
|
140
|
+
# considered passing.
|
141
|
+
#
|
142
|
+
# Only applicable to ColumnMap rules.
|
139
143
|
# @!attribute [rw] dimension
|
140
144
|
# @return [::String]
|
141
145
|
# Required. The dimension a rule belongs to. Results are also aggregated at
|
142
|
-
# the dimension
|
143
|
-
# "CONSISTENCY", "VALIDITY", "UNIQUENESS", "INTEGRITY"]
|
146
|
+
# the dimension level. Supported dimensions are **["COMPLETENESS",
|
147
|
+
# "ACCURACY", "CONSISTENCY", "VALIDITY", "UNIQUENESS", "INTEGRITY"]**
|
144
148
|
# @!attribute [rw] threshold
|
145
149
|
# @return [::Float]
|
146
|
-
# Optional. The minimum ratio of passing_rows / total_rows required to
|
147
|
-
# this rule, with a range of [0.0, 1.0]
|
150
|
+
# Optional. The minimum ratio of **passing_rows / total_rows** required to
|
151
|
+
# pass this rule, with a range of [0.0, 1.0].
|
148
152
|
#
|
149
|
-
# 0 indicates default value (i.e. 1.0)
|
153
|
+
# 0 indicates default value (i.e. 1.0).
|
150
154
|
class DataQualityRule
|
151
155
|
include ::Google::Protobuf::MessageExts
|
152
156
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -155,21 +159,25 @@ module Google
|
|
155
159
|
# @!attribute [rw] min_value
|
156
160
|
# @return [::String]
|
157
161
|
# Optional. The minimum column value allowed for a row to pass this
|
158
|
-
# validation. At least one of min_value and max_value need to be
|
162
|
+
# validation. At least one of `min_value` and `max_value` need to be
|
163
|
+
# provided.
|
159
164
|
# @!attribute [rw] max_value
|
160
165
|
# @return [::String]
|
161
166
|
# Optional. The maximum column value allowed for a row to pass this
|
162
|
-
# validation. At least one of min_value and max_value need to be
|
167
|
+
# validation. At least one of `min_value` and `max_value` need to be
|
168
|
+
# provided.
|
163
169
|
# @!attribute [rw] strict_min_enabled
|
164
170
|
# @return [::Boolean]
|
165
171
|
# Optional. Whether each value needs to be strictly greater than ('>') the
|
166
|
-
# minimum, or if equality is allowed.
|
167
|
-
#
|
172
|
+
# minimum, or if equality is allowed.
|
173
|
+
#
|
174
|
+
# Only relevant if a `min_value` has been defined. Default = false.
|
168
175
|
# @!attribute [rw] strict_max_enabled
|
169
176
|
# @return [::Boolean]
|
170
177
|
# Optional. Whether each value needs to be strictly lesser than ('<') the
|
171
|
-
# maximum, or if equality is allowed.
|
172
|
-
#
|
178
|
+
# maximum, or if equality is allowed.
|
179
|
+
#
|
180
|
+
# Only relevant if a `max_value` has been defined. Default = false.
|
173
181
|
class RangeExpectation
|
174
182
|
include ::Google::Protobuf::MessageExts
|
175
183
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -184,6 +192,7 @@ module Google
|
|
184
192
|
# Evaluates whether each column value is contained by a specified set.
|
185
193
|
# @!attribute [rw] values
|
186
194
|
# @return [::Array<::String>]
|
195
|
+
# Expected values for the column value.
|
187
196
|
class SetExpectation
|
188
197
|
include ::Google::Protobuf::MessageExts
|
189
198
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -192,6 +201,7 @@ module Google
|
|
192
201
|
# Evaluates whether each column value matches a specified regex.
|
193
202
|
# @!attribute [rw] regex
|
194
203
|
# @return [::String]
|
204
|
+
# A regular expression the column value is expected to match.
|
195
205
|
class RegexExpectation
|
196
206
|
include ::Google::Protobuf::MessageExts
|
197
207
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -207,30 +217,36 @@ module Google
|
|
207
217
|
# range.
|
208
218
|
# @!attribute [rw] statistic
|
209
219
|
# @return [::Google::Cloud::Dataplex::V1::DataQualityRule::StatisticRangeExpectation::ColumnStatistic]
|
220
|
+
# The aggregate metric to evaluate.
|
210
221
|
# @!attribute [rw] min_value
|
211
222
|
# @return [::String]
|
212
223
|
# The minimum column statistic value allowed for a row to pass this
|
213
224
|
# validation.
|
214
|
-
#
|
225
|
+
#
|
226
|
+
# At least one of `min_value` and `max_value` need to be provided.
|
215
227
|
# @!attribute [rw] max_value
|
216
228
|
# @return [::String]
|
217
229
|
# The maximum column statistic value allowed for a row to pass this
|
218
230
|
# validation.
|
219
|
-
#
|
231
|
+
#
|
232
|
+
# At least one of `min_value` and `max_value` need to be provided.
|
220
233
|
# @!attribute [rw] strict_min_enabled
|
221
234
|
# @return [::Boolean]
|
222
235
|
# Whether column statistic needs to be strictly greater than ('>')
|
223
|
-
# the minimum, or if equality is allowed.
|
224
|
-
#
|
236
|
+
# the minimum, or if equality is allowed.
|
237
|
+
#
|
238
|
+
# Only relevant if a `min_value` has been defined. Default = false.
|
225
239
|
# @!attribute [rw] strict_max_enabled
|
226
240
|
# @return [::Boolean]
|
227
241
|
# Whether column statistic needs to be strictly lesser than ('<') the
|
228
|
-
# maximum, or if equality is allowed.
|
229
|
-
#
|
242
|
+
# maximum, or if equality is allowed.
|
243
|
+
#
|
244
|
+
# Only relevant if a `max_value` has been defined. Default = false.
|
230
245
|
class StatisticRangeExpectation
|
231
246
|
include ::Google::Protobuf::MessageExts
|
232
247
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
233
248
|
|
249
|
+
# The list of aggregate metrics a rule can be evaluated against.
|
234
250
|
module ColumnStatistic
|
235
251
|
# Unspecified statistic type
|
236
252
|
STATISTIC_UNDEFINED = 0
|
@@ -247,22 +263,28 @@ module Google
|
|
247
263
|
end
|
248
264
|
|
249
265
|
# Evaluates whether each row passes the specified condition.
|
266
|
+
#
|
250
267
|
# The SQL expression needs to use BigQuery standard SQL syntax and should
|
251
|
-
# produce a boolean per row as the result.
|
268
|
+
# produce a boolean value per row as the result.
|
269
|
+
#
|
252
270
|
# Example: col1 >= 0 AND col2 < 10
|
253
271
|
# @!attribute [rw] sql_expression
|
254
272
|
# @return [::String]
|
273
|
+
# The SQL expression.
|
255
274
|
class RowConditionExpectation
|
256
275
|
include ::Google::Protobuf::MessageExts
|
257
276
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
258
277
|
end
|
259
278
|
|
260
279
|
# Evaluates whether the provided expression is true.
|
280
|
+
#
|
261
281
|
# The SQL expression needs to use BigQuery standard SQL syntax and should
|
262
282
|
# produce a scalar boolean result.
|
283
|
+
#
|
263
284
|
# Example: MIN(col1) >= 0
|
264
285
|
# @!attribute [rw] sql_expression
|
265
286
|
# @return [::String]
|
287
|
+
# The SQL expression.
|
266
288
|
class TableConditionExpectation
|
267
289
|
include ::Google::Protobuf::MessageExts
|
268
290
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -40,6 +40,10 @@ module Google
|
|
40
40
|
# * Must end with a number or a letter.
|
41
41
|
# * Must be between 1-63 characters.
|
42
42
|
# * Must be unique within the customer project / location.
|
43
|
+
# @!attribute [rw] validate_only
|
44
|
+
# @return [::Boolean]
|
45
|
+
# Optional. Only validate the request, but do not perform mutations.
|
46
|
+
# The default is `false`.
|
43
47
|
class CreateDataScanRequest
|
44
48
|
include ::Google::Protobuf::MessageExts
|
45
49
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -54,6 +58,10 @@ module Google
|
|
54
58
|
# @!attribute [rw] update_mask
|
55
59
|
# @return [::Google::Protobuf::FieldMask]
|
56
60
|
# Required. Mask of fields to update.
|
61
|
+
# @!attribute [rw] validate_only
|
62
|
+
# @return [::Boolean]
|
63
|
+
# Optional. Only validate the request, but do not perform mutations.
|
64
|
+
# The default is `false`.
|
57
65
|
class UpdateDataScanRequest
|
58
66
|
include ::Google::Protobuf::MessageExts
|
59
67
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -108,7 +116,7 @@ module Google
|
|
108
116
|
# @!attribute [rw] page_size
|
109
117
|
# @return [::Integer]
|
110
118
|
# Optional. Maximum number of dataScans to return. The service may return
|
111
|
-
# fewer than this value. If unspecified, at most
|
119
|
+
# fewer than this value. If unspecified, at most 500 scans will be returned.
|
112
120
|
# The maximum value is 1000; values above 1000 will be coerced to 1000.
|
113
121
|
# @!attribute [rw] page_token
|
114
122
|
# @return [::String]
|
@@ -171,7 +179,7 @@ module Google
|
|
171
179
|
# @!attribute [rw] name
|
172
180
|
# @return [::String]
|
173
181
|
# Required. The resource name of the DataScanJob:
|
174
|
-
# `projects/{project}/locations/{location_id}/dataScans/{data_scan_id}/
|
182
|
+
# `projects/{project}/locations/{location_id}/dataScans/{data_scan_id}/jobs/{data_scan_job_id}`
|
175
183
|
# where `project` refers to a *project_id* or *project_number* and
|
176
184
|
# `location_id` refers to a GCP region.
|
177
185
|
# @!attribute [rw] view
|
@@ -283,8 +283,7 @@ module Google
|
|
283
283
|
# published table name. Specifying a new ID in an update entity
|
284
284
|
# request will override the existing value.
|
285
285
|
# The ID must contain only letters (a-z, A-Z), numbers (0-9), and
|
286
|
-
# underscores
|
287
|
-
# characters.
|
286
|
+
# underscores, and consist of 256 or fewer characters.
|
288
287
|
# @!attribute [rw] etag
|
289
288
|
# @return [::String]
|
290
289
|
# Optional. The etag associated with the entity, which can be retrieved with
|
@@ -322,6 +321,14 @@ module Google
|
|
322
321
|
# @!attribute [r] compatibility
|
323
322
|
# @return [::Google::Cloud::Dataplex::V1::Entity::CompatibilityStatus]
|
324
323
|
# Output only. Metadata stores that the entity is compatible with.
|
324
|
+
# @!attribute [r] access
|
325
|
+
# @return [::Google::Cloud::Dataplex::V1::StorageAccess]
|
326
|
+
# Output only. Identifies the access mechanism to the entity. Not user
|
327
|
+
# settable.
|
328
|
+
# @!attribute [r] uid
|
329
|
+
# @return [::String]
|
330
|
+
# Output only. System generated unique ID for the Entity. This ID will be
|
331
|
+
# different if the Entity is deleted and re-created with the same name.
|
325
332
|
# @!attribute [rw] schema
|
326
333
|
# @return [::Google::Cloud::Dataplex::V1::Schema]
|
327
334
|
# Required. The description of the data structure and layout.
|
@@ -685,6 +692,28 @@ module Google
|
|
685
692
|
end
|
686
693
|
end
|
687
694
|
|
695
|
+
# Describes the access mechanism of the data within its storage location.
|
696
|
+
# @!attribute [r] read
|
697
|
+
# @return [::Google::Cloud::Dataplex::V1::StorageAccess::AccessMode]
|
698
|
+
# Output only. Describes the read access mechanism of the data. Not user
|
699
|
+
# settable.
|
700
|
+
class StorageAccess
|
701
|
+
include ::Google::Protobuf::MessageExts
|
702
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
703
|
+
|
704
|
+
# Access Mode determines how data stored within the Entity is read.
|
705
|
+
module AccessMode
|
706
|
+
# Access mode unspecified.
|
707
|
+
ACCESS_MODE_UNSPECIFIED = 0
|
708
|
+
|
709
|
+
# Default. Data is accessed directly using storage APIs.
|
710
|
+
DIRECT = 1
|
711
|
+
|
712
|
+
# Data is accessed through a managed interface using BigQuery APIs.
|
713
|
+
MANAGED = 2
|
714
|
+
end
|
715
|
+
end
|
716
|
+
|
688
717
|
# Identifies the cloud system that manages the data storage.
|
689
718
|
module StorageSystem
|
690
719
|
# Storage system unspecified.
|
@@ -24,7 +24,7 @@ module Google
|
|
24
24
|
# DataScan scheduling and trigger settings.
|
25
25
|
# @!attribute [rw] on_demand
|
26
26
|
# @return [::Google::Cloud::Dataplex::V1::Trigger::OnDemand]
|
27
|
-
# The scan runs
|
27
|
+
# The scan runs once via `RunDataScan` API.
|
28
28
|
# @!attribute [rw] schedule
|
29
29
|
# @return [::Google::Cloud::Dataplex::V1::Trigger::Schedule]
|
30
30
|
# The scan is scheduled to run periodically.
|
@@ -32,7 +32,7 @@ module Google
|
|
32
32
|
include ::Google::Protobuf::MessageExts
|
33
33
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
34
34
|
|
35
|
-
# The scan runs
|
35
|
+
# The scan runs once via `RunDataScan` API.
|
36
36
|
class OnDemand
|
37
37
|
include ::Google::Protobuf::MessageExts
|
38
38
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -41,13 +41,17 @@ module Google
|
|
41
41
|
# The scan is scheduled to run periodically.
|
42
42
|
# @!attribute [rw] cron
|
43
43
|
# @return [::String]
|
44
|
-
# Required. Cron
|
44
|
+
# Required. [Cron](https://en.wikipedia.org/wiki/Cron) schedule for running
|
45
45
|
# scans periodically.
|
46
|
-
#
|
47
|
-
# cron tab
|
48
|
-
#
|
49
|
-
#
|
50
|
-
#
|
46
|
+
#
|
47
|
+
# To explicitly set a timezone in the cron tab, apply a prefix in the
|
48
|
+
# cron tab: **"CRON_TZ=$\\{IANA_TIME_ZONE}"** or **"TZ=$\\{IANA_TIME_ZONE}"**.
|
49
|
+
# The **$\\{IANA_TIME_ZONE}** may only be a valid string from IANA time zone
|
50
|
+
# database
|
51
|
+
# ([wikipedia](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List)).
|
52
|
+
# For example, `CRON_TZ=America/New_York 1 * * * *`, or
|
53
|
+
# `TZ=America/New_York 1 * * * *`.
|
54
|
+
#
|
51
55
|
# This field is required for Schedule scans.
|
52
56
|
class Schedule
|
53
57
|
include ::Google::Protobuf::MessageExts
|
@@ -58,9 +62,15 @@ module Google
|
|
58
62
|
# The data source for DataScan.
|
59
63
|
# @!attribute [rw] entity
|
60
64
|
# @return [::String]
|
61
|
-
# Immutable. The
|
62
|
-
# the form:
|
65
|
+
# Immutable. The Dataplex entity that represents the data source (e.g.
|
66
|
+
# BigQuery table) for DataScan, of the form:
|
63
67
|
# `projects/{project_number}/locations/{location_id}/lakes/{lake_id}/zones/{zone_id}/entities/{entity_id}`.
|
68
|
+
# @!attribute [rw] resource
|
69
|
+
# @return [::String]
|
70
|
+
# Immutable. The service-qualified full resource name of the cloud resource
|
71
|
+
# for a DataScan job to scan against. The field could be: BigQuery table of
|
72
|
+
# type "TABLE" for DataProfileScan/DataQualityScan Format:
|
73
|
+
# //bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID
|
64
74
|
class DataSource
|
65
75
|
include ::Google::Protobuf::MessageExts
|
66
76
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -78,13 +88,13 @@ module Google
|
|
78
88
|
# @!attribute [rw] field
|
79
89
|
# @return [::String]
|
80
90
|
# The field that contains values which monotonically increases over time
|
81
|
-
# (e.g. timestamp).
|
91
|
+
# (e.g. a timestamp column).
|
82
92
|
# @!attribute [rw] start
|
83
93
|
# @return [::String]
|
84
|
-
# Value that marks the start of the range
|
94
|
+
# Value that marks the start of the range.
|
85
95
|
# @!attribute [rw] end
|
86
96
|
# @return [::String]
|
87
|
-
# Value that marks the end of the range
|
97
|
+
# Value that marks the end of the range.
|
88
98
|
class IncrementalField
|
89
99
|
include ::Google::Protobuf::MessageExts
|
90
100
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|