google-cloud-dataplex-v1 0.5.1 → 0.6.0

data/proto_docs/google/cloud/dataplex/v1/data_profile.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Auto-generated by gapic-generator-ruby. DO NOT EDIT!
+
+
+module Google
+  module Cloud
+    module Dataplex
+      module V1
+        # DataProfileScan related setting.
+        class DataProfileSpec
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+        end
+
+        # DataProfileResult defines the output of DataProfileScan.
+        # Each field of the table will have field type specific profile result.
+        # @!attribute [rw] row_count
+        #   @return [::Integer]
+        #     The count of all rows in the sampled data.
+        #     Return 0, if zero rows.
+        # @!attribute [rw] profile
+        #   @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile]
+        #     This represents the profile information per field.
+        # @!attribute [rw] scanned_data
+        #   @return [::Google::Cloud::Dataplex::V1::ScannedData]
+        #     The data scanned for this profile.
+        class DataProfileResult
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+
+          # Profile information describing the structure and layout of the data
+          # and contains the profile info.
+          # @!attribute [rw] fields
+          #   @return [::Array<::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field>]
+          #     The sequence of fields describing data in table entities.
+          class Profile
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+
+            # Represents a column field within a table schema.
+            # @!attribute [rw] name
+            #   @return [::String]
+            #     The name of the field.
+            # @!attribute [rw] type
+            #   @return [::String]
+            #     The field data type. Possible values include:
+            #
+            #     * STRING
+            #     * BYTE
+            #     * INT64
+            #     * INT32
+            #     * INT16
+            #     * DOUBLE
+            #     * FLOAT
+            #     * DECIMAL
+            #     * BOOLEAN
+            #     * BINARY
+            #     * TIMESTAMP
+            #     * DATE
+            #     * TIME
+            #     * NULL
+            #     * RECORD
+            # @!attribute [rw] mode
+            #   @return [::String]
+            #     The mode of the field. Its value will be:
+            #     REQUIRED, if it is a required field.
+            #     NULLABLE, if it is an optional field.
+            #     REPEATED, if it is a repeated field.
+            # @!attribute [rw] profile
+            #   @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo]
+            #     The profile information for the corresponding field.
+            class Field
+              include ::Google::Protobuf::MessageExts
+              extend ::Google::Protobuf::MessageExts::ClassMethods
+
+              # ProfileInfo defines the profile information for each schema field type.
+              # @!attribute [rw] null_ratio
+              #   @return [::Float]
+              #     The ratio of null rows against the rows in the sampled data.
+              # @!attribute [rw] distinct_ratio
+              #   @return [::Float]
+              #     The ratio of rows that are distinct against the rows in the sampled
+              #     data.
+              # @!attribute [rw] top_n_values
+              #   @return [::Array<::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo::TopNValue>]
+              #     The array of top N values of the field in the sampled data.
+              #     Currently N is set as 10 or equal to distinct values in the field,
+              #     whichever is smaller. This will be optional for complex non-groupable
+              #     data-types such as JSON, ARRAY, JSON, STRUCT.
+              # @!attribute [rw] string_profile
+              #   @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo::StringFieldInfo]
+              #     The corresponding string field profile.
+              # @!attribute [rw] integer_profile
+              #   @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo::IntegerFieldInfo]
+              #     The corresponding integer field profile.
+              # @!attribute [rw] double_profile
+              #   @return [::Google::Cloud::Dataplex::V1::DataProfileResult::Profile::Field::ProfileInfo::DoubleFieldInfo]
+              #     The corresponding double field profile.
+              class ProfileInfo
+                include ::Google::Protobuf::MessageExts
+                extend ::Google::Protobuf::MessageExts::ClassMethods
+
+                # StringFieldInfo defines output info for any string type field.
+                # @!attribute [rw] min_length
+                #   @return [::Integer]
+                #     The minimum length of the string field in the sampled data.
+                #     Optional if zero non-null rows.
+                # @!attribute [rw] max_length
+                #   @return [::Integer]
+                #     The maximum length of a string field in the sampled data.
+                #     Optional if zero non-null rows.
+                # @!attribute [rw] average_length
+                #   @return [::Float]
+                #     The average length of a string field in the sampled data.
+                #     Optional if zero non-null rows.
+                class StringFieldInfo
+                  include ::Google::Protobuf::MessageExts
+                  extend ::Google::Protobuf::MessageExts::ClassMethods
+                end
+
+                # IntegerFieldInfo defines output for any integer type field.
+                # @!attribute [rw] average
+                #   @return [::Float]
+                #     The average of non-null values of integer field in the sampled
+                #     data. Return NaN, if the field has a NaN. Optional if zero non-null
+                #     rows.
+                # @!attribute [rw] standard_deviation
+                #   @return [::Float]
+                #     The standard deviation of non-null of integer field in the sampled
+                #     data. Return NaN, if the field has a NaN. Optional if zero non-null
+                #     rows.
+                # @!attribute [rw] min
+                #   @return [::Integer]
+                #     The minimum value of an integer field in the sampled data.
+                #     Return NaN, if the field has a NaN. Optional if zero non-null
+                #     rows.
+                # @!attribute [rw] quartiles
+                #   @return [::Array<::Integer>]
+                #     A quartile divide the number of data points into four parts, or
+                #     quarters, of more-or-less equal size. Three main quartiles used
+                #     are: The first quartile (Q1) splits off the lowest 25% of data from
+                #     the highest 75%. It is also known as the lower or 25th empirical
+                #     quartile, as 25% of the data is below this point. The second
+                #     quartile (Q2) is the median of a data set. So, 50% of the data lies
+                #     below this point. The third quartile (Q3) splits off the highest
+                #     25% of data from the lowest 75%. It is known as the upper or 75th
+                #     empirical quartile, as 75% of the data lies below this point. So,
+                #     here the quartiles is provided as an ordered list of quartile
+                #     values, occurring in order Q1, median, Q3.
+                # @!attribute [rw] max
+                #   @return [::Integer]
+                #     The maximum value of an integer field in the sampled data.
+                #     Return NaN, if the field has a NaN. Optional if zero non-null
+                #     rows.
+                class IntegerFieldInfo
+                  include ::Google::Protobuf::MessageExts
+                  extend ::Google::Protobuf::MessageExts::ClassMethods
+                end
+
+                # DoubleFieldInfo defines output for any double type field.
+                # @!attribute [rw] average
+                #   @return [::Float]
+                #     The average of non-null values of double field in the sampled data.
+                #     Return NaN, if the field has a NaN. Optional if zero non-null rows.
+                # @!attribute [rw] standard_deviation
+                #   @return [::Float]
+                #     The standard deviation of non-null of double field in the sampled
+                #     data. Return NaN, if the field has a NaN. Optional if zero non-null
+                #     rows.
+                # @!attribute [rw] min
+                #   @return [::Float]
+                #     The minimum value of a double field in the sampled data.
+                #     Return NaN, if the field has a NaN. Optional if zero non-null
+                #     rows.
+                # @!attribute [rw] quartiles
+                #   @return [::Array<::Float>]
+                #     A quartile divide the numebr of data points into four parts, or
+                #     quarters, of more-or-less equal size. Three main quartiles used
+                #     are: The first quartile (Q1) splits off the lowest 25% of data from
+                #     the highest 75%. It is also known as the lower or 25th empirical
+                #     quartile, as 25% of the data is below this point. The second
+                #     quartile (Q2) is the median of a data set. So, 50% of the data lies
+                #     below this point. The third quartile (Q3) splits off the highest
+                #     25% of data from the lowest 75%. It is known as the upper or 75th
+                #     empirical quartile, as 75% of the data lies below this point. So,
+                #     here the quartiles is provided as an ordered list of quartile
+                #     values, occurring in order Q1, median, Q3.
+                # @!attribute [rw] max
+                #   @return [::Float]
+                #     The maximum value of a double field in the sampled data.
+                #     Return NaN, if the field has a NaN. Optional if zero non-null
+                #     rows.
+                class DoubleFieldInfo
+                  include ::Google::Protobuf::MessageExts
+                  extend ::Google::Protobuf::MessageExts::ClassMethods
+                end
+
+                # The TopNValue defines the structure of output of top N values of a
+                # field.
+                # @!attribute [rw] value
+                #   @return [::String]
+                #     The value is the string value of the actual value from the field.
+                # @!attribute [rw] count
+                #   @return [::Integer]
+                #     The frequency count of the corresponding value in the field.
+                class TopNValue
+                  include ::Google::Protobuf::MessageExts
+                  extend ::Google::Protobuf::MessageExts::ClassMethods
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/proto_docs/google/cloud/dataplex/v1/data_quality.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Auto-generated by gapic-generator-ruby. DO NOT EDIT!
+
+
+module Google
+  module Cloud
+    module Dataplex
+      module V1
+        # DataQualityScan related setting.
+        # @!attribute [rw] rules
+        #   @return [::Array<::Google::Cloud::Dataplex::V1::DataQualityRule>]
+        #     The list of rules to evaluate against a data source. At least one rule is
+        #     required.
+        class DataQualitySpec
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+        end
+
+        # The output of a DataQualityScan.
+        # @!attribute [rw] passed
+        #   @return [::Boolean]
+        #     Overall data quality result -- `true` if all rules passed.
+        # @!attribute [rw] dimensions
+        #   @return [::Array<::Google::Cloud::Dataplex::V1::DataQualityDimensionResult>]
+        #     A list of results at the dimension-level.
+        # @!attribute [rw] rules
+        #   @return [::Array<::Google::Cloud::Dataplex::V1::DataQualityRuleResult>]
+        #     A list of all the rules in a job, and their results.
+        # @!attribute [rw] row_count
+        #   @return [::Integer]
+        #     The count of rows processed.
+        # @!attribute [rw] scanned_data
+        #   @return [::Google::Cloud::Dataplex::V1::ScannedData]
+        #     The data scanned for this result.
+        class DataQualityResult
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+        end
+
+        # DataQualityRuleResult provides a more detailed, per-rule level view of the
+        # results.
+        # @!attribute [rw] rule
+        #   @return [::Google::Cloud::Dataplex::V1::DataQualityRule]
+        #     The rule specified in the DataQualitySpec, as is.
+        # @!attribute [rw] passed
+        #   @return [::Boolean]
+        #     Whether the rule passed or failed.
+        # @!attribute [rw] evaluated_count
+        #   @return [::Integer]
+        #     The number of rows a rule was evaluated against.
+        #     This field is only valid for ColumnMap type rules.
+        #     Evaluated count can be configured to either
+        #     (1) include all rows (default) - with null rows automatically failing rule
+        #     evaluation  OR (2) exclude null rows from the evaluated_count, by setting
+        #     ignore_nulls = true
+        # @!attribute [rw] passed_count
+        #   @return [::Integer]
+        #     The number of rows which passed a rule evaluation.
+        #     This field is only valid for ColumnMap type rules.
+        # @!attribute [rw] null_count
+        #   @return [::Integer]
+        #     The number of rows with null values in the specified column.
+        # @!attribute [rw] pass_ratio
+        #   @return [::Float]
+        #     The ratio of passed_count / evaluated_count.
+        #     This field is only valid for ColumnMap type rules.
+        # @!attribute [rw] failing_rows_query
+        #   @return [::String]
+        #     The query to find rows that did not pass this rule.
+        #     Only applies to ColumnMap and RowCondition rules.
+        class DataQualityRuleResult
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+        end
+
+        # DataQualityDimensionResult provides a more detailed, per-dimension level view
+        # of the results.
+        # @!attribute [rw] passed
+        #   @return [::Boolean]
+        #     Whether the dimension passed or failed.
+        class DataQualityDimensionResult
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+        end
+
+        # A rule captures data quality intent about a data source.
+        # @!attribute [rw] range_expectation
+        #   @return [::Google::Cloud::Dataplex::V1::DataQualityRule::RangeExpectation]
+        #     ColumnMap rule which evaluates whether each column value lies between a
+        #     specified range.
+        # @!attribute [rw] non_null_expectation
+        #   @return [::Google::Cloud::Dataplex::V1::DataQualityRule::NonNullExpectation]
+        #     ColumnMap rule which evaluates whether each column value is null.
+        # @!attribute [rw] set_expectation
+        #   @return [::Google::Cloud::Dataplex::V1::DataQualityRule::SetExpectation]
+        #     ColumnMap rule which evaluates whether each column value is contained by
+        #     a specified set.
+        # @!attribute [rw] regex_expectation
+        #   @return [::Google::Cloud::Dataplex::V1::DataQualityRule::RegexExpectation]
+        #     ColumnMap rule which evaluates whether each column value matches a
+        #     specified regex.
+        # @!attribute [rw] uniqueness_expectation
+        #   @return [::Google::Cloud::Dataplex::V1::DataQualityRule::UniquenessExpectation]
+        #     ColumnAggregate rule which evaluates whether the column has duplicates.
+        # @!attribute [rw] statistic_range_expectation
+        #   @return [::Google::Cloud::Dataplex::V1::DataQualityRule::StatisticRangeExpectation]
+        #     ColumnAggregate rule which evaluates whether the column aggregate
+        #     statistic lies between a specified range.
+        # @!attribute [rw] row_condition_expectation
+        #   @return [::Google::Cloud::Dataplex::V1::DataQualityRule::RowConditionExpectation]
+        #     Table rule which evaluates whether each row passes the specified
+        #     condition.
+        # @!attribute [rw] table_condition_expectation
+        #   @return [::Google::Cloud::Dataplex::V1::DataQualityRule::TableConditionExpectation]
+        #     Table rule which evaluates whether the provided expression is true.
+        # @!attribute [rw] column
+        #   @return [::String]
+        #     Optional. The unnested column which this rule is evaluated against.
+        # @!attribute [rw] ignore_null
+        #   @return [::Boolean]
+        #     Optional. Rows with null values will automatically fail a rule, unless
+        #     ignore_null is true. In that case, such null rows are trivially considered
+        #     passing. Only applicable to ColumnMap rules.
+        # @!attribute [rw] dimension
+        #   @return [::String]
+        #     Required. The dimension a rule belongs to. Results are also aggregated at
+        #     the dimension-level. Supported dimensions are ["COMPLETENESS", "ACCURACY",
+        #     "CONSISTENCY", "VALIDITY", "UNIQUENESS", "INTEGRITY"]
+        # @!attribute [rw] threshold
+        #   @return [::Float]
+        #     Optional. The minimum ratio of passing_rows / total_rows required to pass
+        #     this rule, with a range of [0.0, 1.0]
+        #
+        #     0 indicates default value (i.e. 1.0)
+        class DataQualityRule
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+
+          # Evaluates whether each column value lies between a specified range.
+          # @!attribute [rw] min_value
+          #   @return [::String]
+          #     Optional. The minimum column value allowed for a row to pass this
+          #     validation. At least one of min_value and max_value need to be provided.
+          # @!attribute [rw] max_value
+          #   @return [::String]
+          #     Optional. The maximum column value allowed for a row to pass this
+          #     validation. At least one of min_value and max_value need to be provided.
+          # @!attribute [rw] strict_min_enabled
+          #   @return [::Boolean]
+          #     Optional. Whether each value needs to be strictly greater than ('>') the
+          #     minimum, or if equality is allowed. Only relevant if a min_value has been
+          #     defined. Default = false.
+          # @!attribute [rw] strict_max_enabled
+          #   @return [::Boolean]
+          #     Optional. Whether each value needs to be strictly lesser than ('<') the
+          #     maximum, or if equality is allowed. Only relevant if a max_value has been
+          #     defined. Default = false.
+          class RangeExpectation
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+
+          # Evaluates whether each column value is null.
+          class NonNullExpectation
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+
+          # Evaluates whether each column value is contained by a specified set.
+          # @!attribute [rw] values
+          #   @return [::Array<::String>]
+          class SetExpectation
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+
+          # Evaluates whether each column value matches a specified regex.
+          # @!attribute [rw] regex
+          #   @return [::String]
+          class RegexExpectation
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+
+          # Evaluates whether the column has duplicates.
+          class UniquenessExpectation
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+
+          # Evaluates whether the column aggregate statistic lies between a specified
+          # range.
+          # @!attribute [rw] statistic
+          #   @return [::Google::Cloud::Dataplex::V1::DataQualityRule::StatisticRangeExpectation::ColumnStatistic]
+          # @!attribute [rw] min_value
+          #   @return [::String]
+          #     The minimum column statistic value allowed for a row to pass this
+          #     validation.
+          #     At least one of min_value and max_value need to be provided.
+          # @!attribute [rw] max_value
+          #   @return [::String]
+          #     The maximum column statistic value allowed for a row to pass this
+          #     validation.
+          #     At least one of min_value and max_value need to be provided.
+          # @!attribute [rw] strict_min_enabled
+          #   @return [::Boolean]
+          #     Whether column statistic needs to be strictly greater than ('>')
+          #     the minimum, or if equality is allowed. Only relevant if a min_value has
+          #     been defined. Default = false.
+          # @!attribute [rw] strict_max_enabled
+          #   @return [::Boolean]
+          #     Whether column statistic needs to be strictly lesser than ('<') the
+          #     maximum, or if equality is allowed. Only relevant if a max_value has been
+          #     defined. Default = false.
+          class StatisticRangeExpectation
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+
+            module ColumnStatistic
+              # Unspecified statistic type
+              STATISTIC_UNDEFINED = 0
+
+              # Evaluate the column mean
+              MEAN = 1
+
+              # Evaluate the column min
+              MIN = 2
+
+              # Evaluate the column max
+              MAX = 3
+            end
+          end
+
+          # Evaluates whether each row passes the specified condition.
+          # The SQL expression needs to use BigQuery standard SQL syntax and should
+          # produce a boolean per row as the result.
+          # Example: col1 >= 0 AND col2 < 10
+          # @!attribute [rw] sql_expression
+          #   @return [::String]
+          class RowConditionExpectation
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+
+          # Evaluates whether the provided expression is true.
+          # The SQL expression needs to use BigQuery standard SQL syntax and should
+          # produce a scalar boolean result.
+          # Example: MIN(col1) >= 0
+          # @!attribute [rw] sql_expression
+          #   @return [::String]
+          class TableConditionExpectation
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+        end
+      end
+    end
+  end
+end