google-cloud-dataproc 0.3.1 → 0.4.0

data/lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service_client_config.json

@@ -0,0 +1,51 @@
+{
+  "interfaces": {
+    "google.cloud.dataproc.v1beta2.AutoscalingPolicyService": {
+      "retry_codes": {
+        "idempotent": [
+          "DEADLINE_EXCEEDED",
+          "UNAVAILABLE"
+        ],
+        "non_idempotent": []
+      },
+      "retry_params": {
+        "default": {
+          "initial_retry_delay_millis": 100,
+          "retry_delay_multiplier": 1.3,
+          "max_retry_delay_millis": 60000,
+          "initial_rpc_timeout_millis": 20000,
+          "rpc_timeout_multiplier": 1.0,
+          "max_rpc_timeout_millis": 20000,
+          "total_timeout_millis": 600000
+        }
+      },
+      "methods": {
+        "CreateAutoscalingPolicy": {
+          "timeout_millis": 60000,
+          "retry_codes_name": "non_idempotent",
+          "retry_params_name": "default"
+        },
+        "UpdateAutoscalingPolicy": {
+          "timeout_millis": 60000,
+          "retry_codes_name": "idempotent",
+          "retry_params_name": "default"
+        },
+        "GetAutoscalingPolicy": {
+          "timeout_millis": 60000,
+          "retry_codes_name": "idempotent",
+          "retry_params_name": "default"
+        },
+        "ListAutoscalingPolicies": {
+          "timeout_millis": 60000,
+          "retry_codes_name": "idempotent",
+          "retry_params_name": "default"
+        },
+        "DeleteAutoscalingPolicy": {
+          "timeout_millis": 60000,
+          "retry_codes_name": "non_idempotent",
+          "retry_params_name": "default"
+        }
+      }
+    }
+  }
+}

data/lib/google/cloud/dataproc/v1beta2/cluster_controller_client.rb

@@ -29,6 +29,7 @@ require "google/longrunning/operations_client"
 
 require "google/cloud/dataproc/v1beta2/clusters_pb"
 require "google/cloud/dataproc/v1beta2/credentials"
+require "google/cloud/dataproc/version"
 
 module Google
   module Cloud
@@ -146,7 +147,7 @@ module Google
               updater_proc = credentials.updater_proc
             end
 
-            package_version = Gem.loaded_specs['google-cloud-dataproc'].version.version
+            package_version = Google::Cloud::Dataproc::VERSION
 
             google_api_client = "gl-ruby/#{RUBY_VERSION}"
             google_api_client << " #{lib_name}/#{lib_version}" if lib_name
@@ -234,11 +235,10 @@ module Google
           #   can also be provided.
           # @param request_id [String]
           #   Optional. A unique id used to identify the request. If the server
-          #   receives two
-          #   {Google::Cloud::Dataproc::V1beta2::CreateClusterRequest CreateClusterRequest}
-          #   requests  with the same id, then the second request will be ignored and the
-          #   first {Google::Longrunning::Operation} created
-          #   and stored in the backend is returned.
+          #   receives two {Google::Cloud::Dataproc::V1beta2::CreateClusterRequest CreateClusterRequest} requests  with the same
+          #   id, then the second request will be ignored and the
+          #   first {Google::Longrunning::Operation} created and stored in the backend
+          #   is returned.
           #
           #   It is recommended to always set this value to a
           #   [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
@@ -384,6 +384,10 @@ module Google
           #   <td>config.lifecycle_config.idle_delete_ttl</td><td>Update Idle TTL
           #   duration</td>
           #   </tr>
+          #   <tr>
+          #   <td>config.autoscaling_config.policy_uri</td><td>Use, stop using, or change
+          #   autoscaling policies</td>
+          #   </tr>
           #   </table>
           #   A hash of the same form as `Google::Protobuf::FieldMask`
           #   can also be provided.
@@ -400,11 +404,10 @@ module Google
           #   can also be provided.
           # @param request_id [String]
           #   Optional. A unique id used to identify the request. If the server
-          #   receives two
-          #   {Google::Cloud::Dataproc::V1beta2::UpdateClusterRequest UpdateClusterRequest}
-          #   requests  with the same id, then the second request will be ignored and the
-          #   first {Google::Longrunning::Operation} created
-          #   and stored in the backend is returned.
+          #   receives two {Google::Cloud::Dataproc::V1beta2::UpdateClusterRequest UpdateClusterRequest} requests  with the same
+          #   id, then the second request will be ignored and the
+          #   first {Google::Longrunning::Operation} created and stored in the
+          #   backend is returned.
           #
           #   It is recommended to always set this value to a
           #   [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
@@ -507,11 +510,10 @@ module Google
           #   (with error NOT_FOUND) if cluster with specified UUID does not exist.
           # @param request_id [String]
           #   Optional. A unique id used to identify the request. If the server
-          #   receives two
-          #   {Google::Cloud::Dataproc::V1beta2::DeleteClusterRequest DeleteClusterRequest}
-          #   requests  with the same id, then the second request will be ignored and the
-          #   first {Google::Longrunning::Operation} created
-          #   and stored in the backend is returned.
+          #   receives two {Google::Cloud::Dataproc::V1beta2::DeleteClusterRequest DeleteClusterRequest} requests  with the same
+          #   id, then the second request will be ignored and the
+          #   first {Google::Longrunning::Operation} created and stored in the
+          #   backend is returned.
           #
           #   It is recommended to always set this value to a
           #   [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).

data/lib/google/cloud/dataproc/v1beta2/clusters_pb.rb

@@ -32,6 +32,16 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
     optional :lifecycle_config, :message, 14, "google.cloud.dataproc.v1beta2.LifecycleConfig"
     repeated :initialization_actions, :message, 11, "google.cloud.dataproc.v1beta2.NodeInitializationAction"
     optional :encryption_config, :message, 15, "google.cloud.dataproc.v1beta2.EncryptionConfig"
+    optional :autoscaling_config, :message, 16, "google.cloud.dataproc.v1beta2.AutoscalingConfig"
+    optional :endpoint_config, :message, 17, "google.cloud.dataproc.v1beta2.EndpointConfig"
+    optional :security_config, :message, 18, "google.cloud.dataproc.v1beta2.SecurityConfig"
+  end
+  add_message "google.cloud.dataproc.v1beta2.EndpointConfig" do
+    map :http_ports, :string, :string, 1
+    optional :enable_http_port_access, :bool, 2
+  end
+  add_message "google.cloud.dataproc.v1beta2.AutoscalingConfig" do
+    optional :policy_uri, :string, 1
   end
   add_message "google.cloud.dataproc.v1beta2.EncryptionConfig" do
     optional :gce_pd_kms_key_name, :string, 1
@@ -45,6 +55,7 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
     repeated :service_account_scopes, :string, 3
     repeated :tags, :string, 4
     map :metadata, :string, :string, 5
+    optional :reservation_affinity, :message, 11, "google.cloud.dataproc.v1beta2.ReservationAffinity"
   end
   add_message "google.cloud.dataproc.v1beta2.InstanceGroupConfig" do
     optional :num_instances, :int32, 1
@@ -77,6 +88,25 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
       optional :auto_delete_ttl, :message, 3, "google.protobuf.Duration"
     end
   end
+  add_message "google.cloud.dataproc.v1beta2.SecurityConfig" do
+    optional :kerberos_config, :message, 1, "google.cloud.dataproc.v1beta2.KerberosConfig"
+  end
+  add_message "google.cloud.dataproc.v1beta2.KerberosConfig" do
+    optional :enable_kerberos, :bool, 1
+    optional :root_principal_password_uri, :string, 2
+    optional :kms_key_uri, :string, 3
+    optional :keystore_uri, :string, 4
+    optional :truststore_uri, :string, 5
+    optional :keystore_password_uri, :string, 6
+    optional :key_password_uri, :string, 7
+    optional :truststore_password_uri, :string, 8
+    optional :cross_realm_trust_realm, :string, 9
+    optional :cross_realm_trust_kdc, :string, 10
+    optional :cross_realm_trust_admin_server, :string, 11
+    optional :cross_realm_trust_shared_password_uri, :string, 12
+    optional :kdc_db_key_uri, :string, 13
+    optional :tgt_lifetime_hours, :int32, 14
+  end
   add_message "google.cloud.dataproc.v1beta2.NodeInitializationAction" do
     optional :executable_file, :string, 1
     optional :execution_timeout, :message, 2, "google.protobuf.Duration"
@@ -103,6 +133,7 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
   add_message "google.cloud.dataproc.v1beta2.SoftwareConfig" do
     optional :image_version, :string, 1
     map :properties, :string, :string, 2
+    repeated :optional_components, :enum, 3, "google.cloud.dataproc.v1beta2.Component"
   end
   add_message "google.cloud.dataproc.v1beta2.ClusterMetrics" do
     map :hdfs_metrics, :string, :int64, 1
@@ -154,6 +185,17 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
   add_message "google.cloud.dataproc.v1beta2.DiagnoseClusterResults" do
     optional :output_uri, :string, 1
   end
+  add_message "google.cloud.dataproc.v1beta2.ReservationAffinity" do
+    optional :consume_reservation_type, :enum, 1, "google.cloud.dataproc.v1beta2.ReservationAffinity.Type"
+    optional :key, :string, 2
+    repeated :values, :string, 3
+  end
+  add_enum "google.cloud.dataproc.v1beta2.ReservationAffinity.Type" do
+    value :TYPE_UNSPECIFIED, 0
+    value :NO_RESERVATION, 1
+    value :ANY_RESERVATION, 2
+    value :SPECIFIC_RESERVATION, 3
+  end
 end
 
 module Google
@@ -162,6 +204,8 @@ module Google
       module V1beta2
         Cluster = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.Cluster").msgclass
         ClusterConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.ClusterConfig").msgclass
+        EndpointConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.EndpointConfig").msgclass
+        AutoscalingConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.AutoscalingConfig").msgclass
         EncryptionConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.EncryptionConfig").msgclass
         GceClusterConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.GceClusterConfig").msgclass
         InstanceGroupConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.InstanceGroupConfig").msgclass
@@ -169,6 +213,8 @@ module Google
         AcceleratorConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.AcceleratorConfig").msgclass
         DiskConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.DiskConfig").msgclass
         LifecycleConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.LifecycleConfig").msgclass
+        SecurityConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.SecurityConfig").msgclass
+        KerberosConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.KerberosConfig").msgclass
         NodeInitializationAction = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.NodeInitializationAction").msgclass
         ClusterStatus = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.ClusterStatus").msgclass
         ClusterStatus::State = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.ClusterStatus.State").enummodule
@@ -183,6 +229,8 @@ module Google
         ListClustersResponse = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.ListClustersResponse").msgclass
         DiagnoseClusterRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.DiagnoseClusterRequest").msgclass
         DiagnoseClusterResults = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.DiagnoseClusterResults").msgclass
+        ReservationAffinity = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.ReservationAffinity").msgclass
+        ReservationAffinity::Type = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.ReservationAffinity.Type").enummodule
       end
     end
   end

data/lib/google/cloud/dataproc/v1beta2/clusters_services_pb.rb

@@ -1,7 +1,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # Source: google/cloud/dataproc/v1beta2/clusters.proto for package 'google.cloud.dataproc.v1beta2'
 # Original file comments:
-# Copyright 2018 Google LLC.
+# Copyright 2019 Google LLC.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/autoscaling_policies.rb

@@ -0,0 +1,202 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+module Google
+  module Cloud
+    module Dataproc
+      module V1beta2
+        # Describes an autoscaling policy for Dataproc cluster autoscaler.
+        # @!attribute [rw] id
+        #   @return [String]
+        #     Required. The policy id.
+        #
+        #     The id must contain only letters (a-z, A-Z), numbers (0-9),
+        #     underscores (_), and hyphens (-). Cannot begin or end with underscore
+        #     or hyphen. Must consist of between 3 and 50 characters.
+        # @!attribute [rw] name
+        #   @return [String]
+        #     Output only. The "resource name" of the policy, as described
+        #     in https://cloud.google.com/apis/design/resource_names of the form
+        #     `projects/{project_id}/regions/{region}/autoscalingPolicies/{policy_id}`.
+        # @!attribute [rw] basic_algorithm
+        #   @return [Google::Cloud::Dataproc::V1beta2::BasicAutoscalingAlgorithm]
+        # @!attribute [rw] worker_config
+        #   @return [Google::Cloud::Dataproc::V1beta2::InstanceGroupAutoscalingPolicyConfig]
+        #     Required. Describes how the autoscaler will operate for primary workers.
+        # @!attribute [rw] secondary_worker_config
+        #   @return [Google::Cloud::Dataproc::V1beta2::InstanceGroupAutoscalingPolicyConfig]
+        #     Optional. Describes how the autoscaler will operate for secondary workers.
+        class AutoscalingPolicy; end
+
+        # Basic algorithm for autoscaling.
+        # @!attribute [rw] yarn_config
+        #   @return [Google::Cloud::Dataproc::V1beta2::BasicYarnAutoscalingConfig]
+        #     Required. YARN autoscaling configuration.
+        # @!attribute [rw] cooldown_period
+        #   @return [Google::Protobuf::Duration]
+        #     Optional. Duration between scaling events. A scaling period starts after
+        #     the update operation from the previous event has completed.
+        #
+        #     Bounds: [2m, 1d]. Default: 2m.
+        class BasicAutoscalingAlgorithm; end
+
+        # Basic autoscaling configurations for YARN.
+        # @!attribute [rw] graceful_decommission_timeout
+        #   @return [Google::Protobuf::Duration]
+        #     Required. Timeout for YARN graceful decommissioning of Node Managers.
+        #     Specifies the duration to wait for jobs to complete before forcefully
+        #     removing workers (and potentially interrupting jobs). Only applicable to
+        #     downscaling operations.
+        #
+        #     Bounds: [0s, 1d].
+        # @!attribute [rw] scale_up_factor
+        #   @return [Float]
+        #     Required. Fraction of average pending memory in the last cooldown period
+        #     for which to add workers. A scale-up factor of 1.0 will result in scaling
+        #     up so that there is no pending memory remaining after the update (more
+        #     aggressive scaling). A scale-up factor closer to 0 will result in a smaller
+        #     magnitude of scaling up (less aggressive scaling).
+        #
+        #     Bounds: [0.0, 1.0].
+        # @!attribute [rw] scale_down_factor
+        #   @return [Float]
+        #     Required. Fraction of average pending memory in the last cooldown period
+        #     for which to remove workers. A scale-down factor of 1 will result in
+        #     scaling down so that there is no available memory remaining after the
+        #     update (more aggressive scaling). A scale-down factor of 0 disables
+        #     removing workers, which can be beneficial for autoscaling a single job.
+        #
+        #     Bounds: [0.0, 1.0].
+        # @!attribute [rw] scale_up_min_worker_fraction
+        #   @return [Float]
+        #     Optional. Minimum scale-up threshold as a fraction of total cluster size
+        #     before scaling occurs. For example, in a 20-worker cluster, a threshold of
+        #     0.1 means the autoscaler must recommend at least a 2-worker scale-up for
+        #     the cluster to scale. A threshold of 0 means the autoscaler will scale up
+        #     on any recommended change.
+        #
+        #     Bounds: [0.0, 1.0]. Default: 0.0.
+        # @!attribute [rw] scale_down_min_worker_fraction
+        #   @return [Float]
+        #     Optional. Minimum scale-down threshold as a fraction of total cluster size
+        #     before scaling occurs. For example, in a 20-worker cluster, a threshold of
+        #     0.1 means the autoscaler must recommend at least a 2 worker scale-down for
+        #     the cluster to scale. A threshold of 0 means the autoscaler will scale down
+        #     on any recommended change.
+        #
+        #     Bounds: [0.0, 1.0]. Default: 0.0.
+        class BasicYarnAutoscalingConfig; end
+
+        # Configuration for the size bounds of an instance group, including its
+        # proportional size to other groups.
+        # @!attribute [rw] min_instances
+        #   @return [Integer]
+        #     Optional. Minimum number of instances for this group.
+        #
+        #     Primary workers - Bounds: [2, max_instances]. Default: 2.
+        #     Secondary workers - Bounds: [0, max_instances]. Default: 0.
+        # @!attribute [rw] max_instances
+        #   @return [Integer]
+        #     Optional. Maximum number of instances for this group. Required for primary
+        #     workers. Note that by default, clusters will not use secondary workers.
+        #     Required for secondary workers if the minimum secondary instances is set.
+        #
+        #     Primary workers - Bounds: [min_instances, ). Required.
+        #     Secondary workers - Bounds: [min_instances, ). Default: 0.
+        # @!attribute [rw] weight
+        #   @return [Integer]
+        #     Optional. Weight for the instance group, which is used to determine the
+        #     fraction of total workers in the cluster from this instance group.
+        #     For example, if primary workers have weight 2, and secondary workers have
+        #     weight 1, the cluster will have approximately 2 primary workers for each
+        #     secondary worker.
+        #
+        #     The cluster may not reach the specified balance if constrained
+        #     by min/max bounds or other autoscaling settings. For example, if
+        #     `max_instances` for secondary workers is 0, then only primary workers will
+        #     be added. The cluster can also be out of balance when created.
+        #
+        #     If weight is not set on any instance group, the cluster will default to
+        #     equal weight for all groups: the cluster will attempt to maintain an equal
+        #     number of workers in each group within the configured size bounds for each
+        #     group. If weight is set for one group only, the cluster will default to
+        #     zero weight on the unset group. For example if weight is set only on
+        #     primary workers, the cluster will use primary workers only and no
+        #     secondary workers.
+        class InstanceGroupAutoscalingPolicyConfig; end
+
+        # A request to create an autoscaling policy.
+        # @!attribute [rw] parent
+        #   @return [String]
+        #     Required. The "resource name" of the region, as described
+        #     in https://cloud.google.com/apis/design/resource_names of the form
+        #     `projects/{project_id}/regions/{region}`.
+        # @!attribute [rw] policy
+        #   @return [Google::Cloud::Dataproc::V1beta2::AutoscalingPolicy]
+        #     The autoscaling policy to create.
+        class CreateAutoscalingPolicyRequest; end
+
+        # A request to fetch an autoscaling policy.
+        # @!attribute [rw] name
+        #   @return [String]
+        #     Required. The "resource name" of the autoscaling policy, as described
+        #     in https://cloud.google.com/apis/design/resource_names of the form
+        #     `projects/{project_id}/regions/{region}/autoscalingPolicies/{policy_id}`.
+        class GetAutoscalingPolicyRequest; end
+
+        # A request to update an autoscaling policy.
+        # @!attribute [rw] policy
+        #   @return [Google::Cloud::Dataproc::V1beta2::AutoscalingPolicy]
+        #     Required. The updated autoscaling policy.
+        class UpdateAutoscalingPolicyRequest; end
+
+        # A request to delete an autoscaling policy.
+        #
+        # Autoscaling policies in use by one or more clusters will not be deleted.
+        # @!attribute [rw] name
+        #   @return [String]
+        #     Required. The "resource name" of the autoscaling policy, as described
+        #     in https://cloud.google.com/apis/design/resource_names of the form
+        #     `projects/{project_id}/regions/{region}/autoscalingPolicies/{policy_id}`.
+        class DeleteAutoscalingPolicyRequest; end
+
+        # A request to list autoscaling policies in a project.
+        # @!attribute [rw] parent
+        #   @return [String]
+        #     Required. The "resource name" of the region, as described
+        #     in https://cloud.google.com/apis/design/resource_names of the form
+        #     `projects/{project_id}/regions/{region}`
+        # @!attribute [rw] page_size
+        #   @return [Integer]
+        #     Optional. The maximum number of results to return in each response.
+        # @!attribute [rw] page_token
+        #   @return [String]
+        #     Optional. The page token, returned by a previous call, to request the
+        #     next page of results.
+        class ListAutoscalingPoliciesRequest; end
+
+        # A response to a request to list autoscaling policies in a project.
+        # @!attribute [rw] policies
+        #   @return [Array<Google::Cloud::Dataproc::V1beta2::AutoscalingPolicy>]
+        #     Output only. Autoscaling policies list.
+        # @!attribute [rw] next_page_token
+        #   @return [String]
+        #     Output only. This token is included in the response if there are more
+        #     results to fetch.
+        class ListAutoscalingPoliciesResponse; end
+      end
+    end
+  end
+end

data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/clusters.rb

@@ -60,15 +60,18 @@ module Google
         # The cluster config.
         # @!attribute [rw] config_bucket
         #   @return [String]
-        #     Optional. A Cloud Storage staging bucket used for sharing generated
-        #     SSH keys and config. If you do not specify a staging bucket, Cloud
-        #     Dataproc will determine an appropriate Cloud Storage location (US,
+        #     Optional. A Google Cloud Storage bucket used to stage job
+        #     dependencies, config files, and job driver console output.
+        #     If you do not specify a staging bucket, Cloud
+        #     Dataproc will determine a Cloud Storage location (US,
         #     ASIA, or EU) for your cluster's staging bucket according to the Google
-        #     Compute Engine zone where your cluster is deployed, and then it will create
-        #     and manage this project-level, per-location bucket for you.
+        #     Compute Engine zone where your cluster is deployed, and then create
+        #     and manage this project-level, per-location bucket (see
+        #     [Cloud Dataproc staging
+        #     bucket](/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
         # @!attribute [rw] gce_cluster_config
         #   @return [Google::Cloud::Dataproc::V1beta2::GceClusterConfig]
-        #     Required. The shared Compute Engine config settings for
+        #     Optional. The shared Compute Engine config settings for
         #     all instances in a cluster.
         # @!attribute [rw] master_config
         #   @return [Google::Cloud::Dataproc::V1beta2::InstanceGroupConfig]
@@ -106,8 +109,43 @@ module Google
         # @!attribute [rw] encryption_config
         #   @return [Google::Cloud::Dataproc::V1beta2::EncryptionConfig]
         #     Optional. Encryption settings for the cluster.
+        # @!attribute [rw] autoscaling_config
+        #   @return [Google::Cloud::Dataproc::V1beta2::AutoscalingConfig]
+        #     Optional. Autoscaling config for the policy associated with the cluster.
+        #     Cluster does not autoscale if this field is unset.
+        # @!attribute [rw] endpoint_config
+        #   @return [Google::Cloud::Dataproc::V1beta2::EndpointConfig]
+        #     Optional. Port/endpoint configuration for this cluster
+        # @!attribute [rw] security_config
+        #   @return [Google::Cloud::Dataproc::V1beta2::SecurityConfig]
+        #     Optional. Security related configuration.
         class ClusterConfig; end
 
+        # Endpoint config for this cluster
+        # @!attribute [rw] http_ports
+        #   @return [Hash{String => String}]
+        #     Output only. The map of port descriptions to URLs. Will only be populated
+        #     if enable_http_port_access is true.
+        # @!attribute [rw] enable_http_port_access
+        #   @return [true, false]
+        #     Optional. If true, enable http access to specific ports on the cluster
+        #     from external sources. Defaults to false.
+        class EndpointConfig; end
+
+        # Autoscaling Policy config associated with the cluster.
+        # @!attribute [rw] policy_uri
+        #   @return [String]
+        #     Optional. The autoscaling policy used by the cluster.
+        #
+        #     Only resource names including projectid and location (region) are valid.
+        #     Examples:
+        #
+        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/locations/[dataproc_region]/autoscalingPolicies/[policy_id]`
+        #     * `projects/[project_id]/locations/[dataproc_region]/autoscalingPolicies/[policy_id]`
+        #
+        #     Note that the policy must be in the same project and Cloud Dataproc region.
+        class AutoscalingConfig; end
+
         # Encryption settings for the cluster.
         # @!attribute [rw] gce_pd_kms_key_name
         #   @return [String]
@@ -150,8 +188,8 @@ module Google
         #
         #     A full URL, partial URI, or short name are valid. Examples:
         #
-        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/us-east1/sub0`
-        #     * `projects/[project_id]/regions/us-east1/sub0`
+        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/us-east1/subnetworks/sub0`
+        #     * `projects/[project_id]/regions/us-east1/subnetworks/sub0`
         #     * `sub0`
         # @!attribute [rw] internal_ip_only
         #   @return [true, false]
@@ -199,6 +237,9 @@ module Google
         #     The Compute Engine metadata entries to add to all instances (see
         #     [Project and instance
         #     metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).
+        # @!attribute [rw] reservation_affinity
+        #   @return [Google::Cloud::Dataproc::V1beta2::ReservationAffinity]
+        #     Optional. Reservation Affinity for consuming Zonal reservation.
         class GceClusterConfig; end
 
         # Optional. The config settings for Compute Engine resources in
@@ -272,8 +313,9 @@ module Google
         # @!attribute [rw] accelerator_type_uri
         #   @return [String]
         #     Full URL, partial URI, or short name of the accelerator type resource to
-        #     expose to this instance. See [Compute Engine AcceleratorTypes](
-        #     /compute/docs/reference/beta/acceleratorTypes)
+        #     expose to this instance. See
+        #     [Compute Engine
+        #     AcceleratorTypes](/compute/docs/reference/beta/acceleratorTypes)
         #
         #     Examples
         #     * `https://www.googleapis.com/compute/beta/projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80`
@@ -329,6 +371,77 @@ module Google
         #     Example: **"1d"**, to delete the cluster 1 day after its creation..
         class LifecycleConfig; end
 
+        # Security related configuration, including encryption, Kerberos, etc.
+        # @!attribute [rw] kerberos_config
+        #   @return [Google::Cloud::Dataproc::V1beta2::KerberosConfig]
+        #     Kerberos related configuration.
+        class SecurityConfig; end
+
+        # Specifies Kerberos related configuration.
+        # @!attribute [rw] enable_kerberos
+        #   @return [true, false]
+        #     Optional. Flag to indicate whether to Kerberize the cluster.
+        # @!attribute [rw] root_principal_password_uri
+        #   @return [String]
+        #     Required. The Cloud Storage URI of a KMS encrypted file containing the root
+        #     principal password.
+        # @!attribute [rw] kms_key_uri
+        #   @return [String]
+        #     Required. The uri of the KMS key used to encrypt various sensitive
+        #     files.
+        # @!attribute [rw] keystore_uri
+        #   @return [String]
+        #     Optional. The Cloud Storage URI of the keystore file used for SSL
+        #     encryption. If not provided, Dataproc will provide a self-signed
+        #     certificate.
+        # @!attribute [rw] truststore_uri
+        #   @return [String]
+        #     Optional. The Cloud Storage URI of the truststore file used for SSL
+        #     encryption. If not provided, Dataproc will provide a self-signed
+        #     certificate.
+        # @!attribute [rw] keystore_password_uri
+        #   @return [String]
+        #     Optional. The Cloud Storage URI of a KMS encrypted file containing the
+        #     password to the user provided keystore. For the self-signed certificate,
+        #     this password is generated by Dataproc.
+        # @!attribute [rw] key_password_uri
+        #   @return [String]
+        #     Optional. The Cloud Storage URI of a KMS encrypted file containing the
+        #     password to the user provided key. For the self-signed certificate, this
+        #     password is generated by Dataproc.
+        # @!attribute [rw] truststore_password_uri
+        #   @return [String]
+        #     Optional. The Cloud Storage URI of a KMS encrypted file containing the
+        #     password to the user provided truststore. For the self-signed certificate,
+        #     this password is generated by Dataproc.
+        # @!attribute [rw] cross_realm_trust_realm
+        #   @return [String]
+        #     Optional. The remote realm the Dataproc on-cluster KDC will trust, should
+        #     the user enable cross realm trust.
+        # @!attribute [rw] cross_realm_trust_kdc
+        #   @return [String]
+        #     Optional. The KDC (IP or hostname) for the remote trusted realm in a cross
+        #     realm trust relationship.
+        # @!attribute [rw] cross_realm_trust_admin_server
+        #   @return [String]
+        #     Optional. The admin server (IP or hostname) for the remote trusted realm in
+        #     a cross realm trust relationship.
+        # @!attribute [rw] cross_realm_trust_shared_password_uri
+        #   @return [String]
+        #     Optional. The Cloud Storage URI of a KMS encrypted file containing the
+        #     shared password between the on-cluster Kerberos realm and the remote
+        #     trusted realm, in a cross realm trust relationship.
+        # @!attribute [rw] kdc_db_key_uri
+        #   @return [String]
+        #     Optional. The Cloud Storage URI of a KMS encrypted file containing the
+        #     master key of the KDC database.
+        # @!attribute [rw] tgt_lifetime_hours
+        #   @return [Integer]
+        #     Optional. The lifetime of the ticket granting ticket, in hours.
+        #     If not specified, or user specifies 0, then default value 10
+        #     will be used.
+        class KerberosConfig; end
+
         # Specifies an executable to run on a fully configured node and a
         # timeout period for executable completion.
         # @!attribute [rw] executable_file
@@ -407,13 +520,13 @@ module Google
         #     such as "1.2" (including a subminor version, such as "1.2.29"), or the
         #     ["preview"
         #     version](/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
-        #     If unspecified, it defaults to the latest version.
+        #     If unspecified, it defaults to the latest Debian version.
         # @!attribute [rw] properties
         #   @return [Hash{String => String}]
         #     Optional. The properties to set on daemon config files.
         #
-        #     Property keys are specified in `prefix:property` format, such as
-        #     `core:fs.defaultFS`. The following are supported prefixes
+        #     Property keys are specified in `prefix:property` format, for example
+        #     `core:hadoop.tmp.dir`. The following are supported prefixes
         #     and their mappings:
         #
         #     * capacity-scheduler: `capacity-scheduler.xml`
@@ -428,6 +541,9 @@ module Google
         #
         #     For more information, see
         #     [Cluster properties](https://cloud.google.com/dataproc/docs/concepts/cluster-properties).
+        # @!attribute [rw] optional_components
+        #   @return [Array<Google::Cloud::Dataproc::V1beta2::Component>]
+        #     The set of optional components to activate on the cluster.
         class SoftwareConfig; end
 
         # Contains cluster daemon metrics, such as HDFS and YARN stats.
@@ -456,11 +572,10 @@ module Google
         # @!attribute [rw] request_id
         #   @return [String]
         #     Optional. A unique id used to identify the request. If the server
-        #     receives two
-        #     {Google::Cloud::Dataproc::V1beta2::CreateClusterRequest CreateClusterRequest}
-        #     requests  with the same id, then the second request will be ignored and the
-        #     first {Google::Longrunning::Operation} created
-        #     and stored in the backend is returned.
+        #     receives two {Google::Cloud::Dataproc::V1beta2::CreateClusterRequest CreateClusterRequest} requests  with the same
+        #     id, then the second request will be ignored and the
+        #     first {Google::Longrunning::Operation} created and stored in the backend
+        #     is returned.
         #
         #     It is recommended to always set this value to a
         #     [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
@@ -550,15 +665,18 @@ module Google
         #     <td>config.lifecycle_config.idle_delete_ttl</td><td>Update Idle TTL
         #     duration</td>
         #     </tr>
+        #     <tr>
+        #     <td>config.autoscaling_config.policy_uri</td><td>Use, stop using, or change
+        #     autoscaling policies</td>
+        #     </tr>
         #     </table>
         # @!attribute [rw] request_id
         #   @return [String]
         #     Optional. A unique id used to identify the request. If the server
-        #     receives two
-        #     {Google::Cloud::Dataproc::V1beta2::UpdateClusterRequest UpdateClusterRequest}
-        #     requests  with the same id, then the second request will be ignored and the
-        #     first {Google::Longrunning::Operation} created
-        #     and stored in the backend is returned.
+        #     receives two {Google::Cloud::Dataproc::V1beta2::UpdateClusterRequest UpdateClusterRequest} requests  with the same
+        #     id, then the second request will be ignored and the
+        #     first {Google::Longrunning::Operation} created and stored in the
+        #     backend is returned.
         #
         #     It is recommended to always set this value to a
         #     [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
@@ -585,11 +703,10 @@ module Google
         # @!attribute [rw] request_id
         #   @return [String]
         #     Optional. A unique id used to identify the request. If the server
-        #     receives two
-        #     {Google::Cloud::Dataproc::V1beta2::DeleteClusterRequest DeleteClusterRequest}
-        #     requests  with the same id, then the second request will be ignored and the
-        #     first {Google::Longrunning::Operation} created
-        #     and stored in the backend is returned.
+        #     receives two {Google::Cloud::Dataproc::V1beta2::DeleteClusterRequest DeleteClusterRequest} requests  with the same
+        #     id, then the second request will be ignored and the
+        #     first {Google::Longrunning::Operation} created and stored in the
+        #     backend is returned.
         #
         #     It is recommended to always set this value to a
         #     [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
@@ -679,6 +796,33 @@ module Google
         #     The output report is a plain text file with a summary of collected
         #     diagnostics.
         class DiagnoseClusterResults; end
+
+        # Reservation Affinity for consuming Zonal reservation.
+        # @!attribute [rw] consume_reservation_type
+        #   @return [Google::Cloud::Dataproc::V1beta2::ReservationAffinity::Type]
+        #     Optional. Type of reservation to consume
+        # @!attribute [rw] key
+        #   @return [String]
+        #     Optional. Corresponds to the label key of reservation resource.
+        # @!attribute [rw] values
+        #   @return [Array<String>]
+        #     Optional. Corresponds to the label values of reservation resource.
+        class ReservationAffinity
+          # Indicates whether to consume capacity from an reservation or not.
+          module Type
+            TYPE_UNSPECIFIED = 0
+
+            # Do not consume from any allocated capacity.
+            NO_RESERVATION = 1
+
+            # Consume any reservation available.
+            ANY_RESERVATION = 2
+
+            # Must consume from a specific reservation. Must specify key value fields
+            # for specifying the reservations.
+            SPECIFIC_RESERVATION = 3
+          end
+        end
       end
     end
   end