google-api-client 0.28.4 → 0.28.5

data/generated/google/apis/dataproc_v1.rb

@@ -25,7 +25,7 @@ module Google
     # @see https://cloud.google.com/dataproc/
     module DataprocV1
       VERSION = 'V1'
-      REVISION = '20181126'
+      REVISION = '20190301'
 
       # View and manage your data across Google Cloud Platform services
       AUTH_CLOUD_PLATFORM = 'https://www.googleapis.com/auth/cloud-platform'

data/generated/google/apis/dataproc_v1/classes.rb

@@ -79,8 +79,8 @@ module Google
         # For example, my-other-app@appspot.gserviceaccount.com.
         # group:`emailid`: An email address that represents a Google group.  For example,
         # admins@example.com.
-        # domain:`domain`: A Google Apps domain name that represents all the  users of
-        # that domain. For example, google.com or example.com.
+        # domain:`domain`: The G Suite domain (primary) that represents all the  users
+        # of that domain. For example, google.com or example.com.
         # Corresponds to the JSON property `members`
         # @return [Array<String>]
         attr_accessor :members
@@ -190,12 +190,12 @@ module Google
       class ClusterConfig
         include Google::Apis::Core::Hashable
       
-        # Optional. A Cloud Storage staging bucket used for sharing generated SSH keys
-        # and config. If you do not specify a staging bucket, Cloud Dataproc will
-        # determine an appropriate Cloud Storage location (US, ASIA, or EU) for your
-        # cluster's staging bucket according to the Google Compute Engine zone where
-        # your cluster is deployed, and then it will create and manage this project-
-        # level, per-location bucket for you.
+        # Optional. A Google Cloud Storage bucket used to stage job dependencies, config
+        # files, and job driver console output. If you do not specify a staging bucket,
+        # Cloud Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for
+        # your cluster's staging bucket according to the Google Compute Engine zone
+        # where your cluster is deployed, and then create and manage this project-level,
+        # per-location bucket (see Cloud Dataproc staging bucket).
         # Corresponds to the JSON property `configBucket`
         # @return [String]
         attr_accessor :config_bucket
@@ -1170,11 +1170,10 @@ module Google
       class JobReference
         include Google::Apis::Core::Hashable
       
-        # Optional. The job ID, which must be unique within the project. The job ID is
-        # generated by the server upon job submission or provided by the user as a means
-        # to perform retries without creating duplicate jobs. The ID must contain only
-        # letters (a-z, A-Z), numbers (0-9), underscores (_), or hyphens (-). The
-        # maximum length is 100 characters.
+        # Optional. The job ID, which must be unique within the project.The ID must
+        # contain only letters (a-z, A-Z), numbers (0-9), underscores (_), or hyphens (-)
+        # . The maximum length is 100 characters.If not specified by the caller, the job
+        # ID will be provided by the server.
         # Corresponds to the JSON property `jobId`
         # @return [String]
         attr_accessor :job_id
@@ -2009,9 +2008,14 @@ module Google
         # @return [String]
         attr_accessor :image_version
       
+        # The set of optional components to activate on the cluster.
+        # Corresponds to the JSON property `optionalComponents`
+        # @return [Array<String>]
+        attr_accessor :optional_components
+      
         # Optional. The properties to set on daemon config files.Property keys are
-        # specified in prefix:property format, such as core:fs.defaultFS. The following
-        # are supported prefixes and their mappings:
+        # specified in prefix:property format, for example core:hadoop.tmp.dir. The
+        # following are supported prefixes and their mappings:
         # capacity-scheduler: capacity-scheduler.xml
         # core: core-site.xml
         # distcp: distcp-default.xml
@@ -2032,6 +2036,7 @@ module Google
         # Update properties of this object
         def update!(**args)
           @image_version = args[:image_version] if args.key?(:image_version)
+          @optional_components = args[:optional_components] if args.key?(:optional_components)
           @properties = args[:properties] if args.key?(:properties)
         end
       end

data/generated/google/apis/dataproc_v1/representations.rb

@@ -872,6 +872,7 @@ module Google
         # @private
         class Representation < Google::Apis::Core::JsonRepresentation
           property :image_version, as: 'imageVersion'
+          collection :optional_components, as: 'optionalComponents'
           hash :properties, as: 'properties'
         end
       end

data/generated/google/apis/dataproc_v1beta2.rb

@@ -25,7 +25,7 @@ module Google
     # @see https://cloud.google.com/dataproc/
     module DataprocV1beta2
       VERSION = 'V1beta2'
-      REVISION = '20181207'
+      REVISION = '20190313'
 
       # View and manage your data across Google Cloud Platform services
       AUTH_CLOUD_PLATFORM = 'https://www.googleapis.com/auth/cloud-platform'

data/generated/google/apis/dataproc_v1beta2/classes.rb

@@ -33,13 +33,12 @@ module Google
         attr_accessor :accelerator_count
       
         # Full URL, partial URI, or short name of the accelerator type resource to
-        # expose to this instance. See Compute Engine AcceleratorTypes( /compute/docs/
-        # reference/beta/acceleratorTypes)Examples * https://www.googleapis.com/compute/
-        # beta/projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80 *
-        # projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80 *
-        # nvidia-tesla-k80Auto Zone Exception: If you are using the Cloud Dataproc Auto
-        # Zone Placement feature, you must use the short name of the accelerator type
-        # resource, for example, nvidia-tesla-k80.
+        # expose to this instance. See Compute Engine AcceleratorTypesExamples * https://
+        # www.googleapis.com/compute/beta/projects/[project_id]/zones/us-east1-a/
+        # acceleratorTypes/nvidia-tesla-k80 * projects/[project_id]/zones/us-east1-a/
+        # acceleratorTypes/nvidia-tesla-k80 * nvidia-tesla-k80Auto Zone Exception: If
+        # you are using the Cloud Dataproc Auto Zone Placement feature, you must use the
+        # short name of the accelerator type resource, for example, nvidia-tesla-k80.
         # Corresponds to the JSON property `acceleratorTypeUri`
         # @return [String]
         attr_accessor :accelerator_type_uri
@@ -55,24 +54,65 @@ module Google
         end
       end
       
-      # Allocation Affinity for consuming Zonal allocation.
-      class AllocationAffinity
+      # Autoscaling Policy config associated with the cluster.
+      class AutoscalingConfig
         include Google::Apis::Core::Hashable
       
-        # 
-        # Corresponds to the JSON property `consumeAllocationType`
+        # Optional. The autoscaling policy used by the cluster.Only resource names
+        # including projectid and location (region) are valid. Examples:
+        # https://www.googleapis.com/compute/v1/projects/[project_id]/locations/[
+        # dataproc_region]/autoscalingPolicies/[policy_id]
+        # projects/[project_id]/locations/[dataproc_region]/autoscalingPolicies/[
+        # policy_id]Note that the policy must be in the same project and Cloud Dataproc
+        # region.
+        # Corresponds to the JSON property `policyUri`
         # @return [String]
-        attr_accessor :consume_allocation_type
+        attr_accessor :policy_uri
       
-        # Corresponds to the label key of Allocation resource.
-        # Corresponds to the JSON property `key`
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @policy_uri = args[:policy_uri] if args.key?(:policy_uri)
+        end
+      end
+      
+      # Describes an autoscaling policy for Dataproc cluster autoscaler.
+      class AutoscalingPolicy
+        include Google::Apis::Core::Hashable
+      
+        # Basic algorithm for autoscaling.
+        # Corresponds to the JSON property `basicAlgorithm`
+        # @return [Google::Apis::DataprocV1beta2::BasicAutoscalingAlgorithm]
+        attr_accessor :basic_algorithm
+      
+        # Required. The policy id.The id must contain only letters (a-z, A-Z), numbers (
+        # 0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or
+        # hyphen. Must consist of between 3 and 50 characters.
+        # Corresponds to the JSON property `id`
         # @return [String]
-        attr_accessor :key
+        attr_accessor :id
       
-        # Corresponds to the label values of allocation resource.
-        # Corresponds to the JSON property `values`
-        # @return [Array<String>]
-        attr_accessor :values
+        # Output only. The "resource name" of the policy, as described in https://cloud.
+        # google.com/apis/design/resource_names of the form projects/`project_id`/
+        # regions/`region`/autoscalingPolicies/`policy_id`.
+        # Corresponds to the JSON property `name`
+        # @return [String]
+        attr_accessor :name
+      
+        # Configuration for the size bounds of an instance group, including its
+        # proportional size to other groups.
+        # Corresponds to the JSON property `secondaryWorkerConfig`
+        # @return [Google::Apis::DataprocV1beta2::InstanceGroupAutoscalingPolicyConfig]
+        attr_accessor :secondary_worker_config
+      
+        # Configuration for the size bounds of an instance group, including its
+        # proportional size to other groups.
+        # Corresponds to the JSON property `workerConfig`
+        # @return [Google::Apis::DataprocV1beta2::InstanceGroupAutoscalingPolicyConfig]
+        attr_accessor :worker_config
       
         def initialize(**args)
            update!(**args)
@@ -80,9 +120,100 @@ module Google
       
         # Update properties of this object
         def update!(**args)
-          @consume_allocation_type = args[:consume_allocation_type] if args.key?(:consume_allocation_type)
-          @key = args[:key] if args.key?(:key)
-          @values = args[:values] if args.key?(:values)
+          @basic_algorithm = args[:basic_algorithm] if args.key?(:basic_algorithm)
+          @id = args[:id] if args.key?(:id)
+          @name = args[:name] if args.key?(:name)
+          @secondary_worker_config = args[:secondary_worker_config] if args.key?(:secondary_worker_config)
+          @worker_config = args[:worker_config] if args.key?(:worker_config)
+        end
+      end
+      
+      # Basic algorithm for autoscaling.
+      class BasicAutoscalingAlgorithm
+        include Google::Apis::Core::Hashable
+      
+        # Optional. Duration between scaling events. A scaling period starts after the
+        # update operation from the previous event has completed.Bounds: 2m, 1d. Default:
+        # 2m.
+        # Corresponds to the JSON property `cooldownPeriod`
+        # @return [String]
+        attr_accessor :cooldown_period
+      
+        # Basic autoscaling configurations for YARN.
+        # Corresponds to the JSON property `yarnConfig`
+        # @return [Google::Apis::DataprocV1beta2::BasicYarnAutoscalingConfig]
+        attr_accessor :yarn_config
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @cooldown_period = args[:cooldown_period] if args.key?(:cooldown_period)
+          @yarn_config = args[:yarn_config] if args.key?(:yarn_config)
+        end
+      end
+      
+      # Basic autoscaling configurations for YARN.
+      class BasicYarnAutoscalingConfig
+        include Google::Apis::Core::Hashable
+      
+        # Required. Timeout for YARN graceful decommissioning of Node Managers.
+        # Specifies the duration to wait for jobs to complete before forcefully removing
+        # workers (and potentially interrupting jobs). Only applicable to downscaling
+        # operations.Bounds: 0s, 1d.
+        # Corresponds to the JSON property `gracefulDecommissionTimeout`
+        # @return [String]
+        attr_accessor :graceful_decommission_timeout
+      
+        # Required. Fraction of average pending memory in the last cooldown period for
+        # which to remove workers. A scale-down factor of 1 will result in scaling down
+        # so that there is no available memory remaining after the update (more
+        # aggressive scaling). A scale-down factor of 0 disables removing workers, which
+        # can be beneficial for autoscaling a single job.Bounds: 0.0, 1.0.
+        # Corresponds to the JSON property `scaleDownFactor`
+        # @return [Float]
+        attr_accessor :scale_down_factor
+      
+        # Optional. Minimum scale-down threshold as a fraction of total cluster size
+        # before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1
+        # means the autoscaler must recommend at least a 2 worker scale-down for the
+        # cluster to scale. A threshold of 0 means the autoscaler will scale down on any
+        # recommended change.Bounds: 0.0, 1.0. Default: 0.0.
+        # Corresponds to the JSON property `scaleDownMinWorkerFraction`
+        # @return [Float]
+        attr_accessor :scale_down_min_worker_fraction
+      
+        # Required. Fraction of average pending memory in the last cooldown period for
+        # which to add workers. A scale-up factor of 1.0 will result in scaling up so
+        # that there is no pending memory remaining after the update (more aggressive
+        # scaling). A scale-up factor closer to 0 will result in a smaller magnitude of
+        # scaling up (less aggressive scaling).Bounds: 0.0, 1.0.
+        # Corresponds to the JSON property `scaleUpFactor`
+        # @return [Float]
+        attr_accessor :scale_up_factor
+      
+        # Optional. Minimum scale-up threshold as a fraction of total cluster size
+        # before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1
+        # means the autoscaler must recommend at least a 2-worker scale-up for the
+        # cluster to scale. A threshold of 0 means the autoscaler will scale up on any
+        # recommended change.Bounds: 0.0, 1.0. Default: 0.0.
+        # Corresponds to the JSON property `scaleUpMinWorkerFraction`
+        # @return [Float]
+        attr_accessor :scale_up_min_worker_fraction
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @graceful_decommission_timeout = args[:graceful_decommission_timeout] if args.key?(:graceful_decommission_timeout)
+          @scale_down_factor = args[:scale_down_factor] if args.key?(:scale_down_factor)
+          @scale_down_min_worker_fraction = args[:scale_down_min_worker_fraction] if args.key?(:scale_down_min_worker_fraction)
+          @scale_up_factor = args[:scale_up_factor] if args.key?(:scale_up_factor)
+          @scale_up_min_worker_fraction = args[:scale_up_min_worker_fraction] if args.key?(:scale_up_min_worker_fraction)
         end
       end
       
@@ -110,8 +241,8 @@ module Google
         # For example, my-other-app@appspot.gserviceaccount.com.
         # group:`emailid`: An email address that represents a Google group.  For example,
         # admins@example.com.
-        # domain:`domain`: A Google Apps domain name that represents all the  users of
-        # that domain. For example, google.com or example.com.
+        # domain:`domain`: The G Suite domain (primary) that represents all the  users
+        # of that domain. For example, google.com or example.com.
         # Corresponds to the JSON property `members`
         # @return [Array<String>]
         attr_accessor :members
@@ -221,12 +352,17 @@ module Google
       class ClusterConfig
         include Google::Apis::Core::Hashable
       
-        # Optional. A Cloud Storage staging bucket used for sharing generated SSH keys
-        # and config. If you do not specify a staging bucket, Cloud Dataproc will
-        # determine an appropriate Cloud Storage location (US, ASIA, or EU) for your
-        # cluster's staging bucket according to the Google Compute Engine zone where
-        # your cluster is deployed, and then it will create and manage this project-
-        # level, per-location bucket for you.
+        # Autoscaling Policy config associated with the cluster.
+        # Corresponds to the JSON property `autoscalingConfig`
+        # @return [Google::Apis::DataprocV1beta2::AutoscalingConfig]
+        attr_accessor :autoscaling_config
+      
+        # Optional. A Google Cloud Storage bucket used to stage job dependencies, config
+        # files, and job driver console output. If you do not specify a staging bucket,
+        # Cloud Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for
+        # your cluster's staging bucket according to the Google Compute Engine zone
+        # where your cluster is deployed, and then create and manage this project-level,
+        # per-location bucket (see Cloud Dataproc staging bucket).
         # Corresponds to the JSON property `configBucket`
         # @return [String]
         attr_accessor :config_bucket
@@ -236,6 +372,11 @@ module Google
         # @return [Google::Apis::DataprocV1beta2::EncryptionConfig]
         attr_accessor :encryption_config
       
+        # Endpoint config for this cluster
+        # Corresponds to the JSON property `endpointConfig`
+        # @return [Google::Apis::DataprocV1beta2::EndpointConfig]
+        attr_accessor :endpoint_config
+      
         # Common config settings for resources of Compute Engine cluster instances,
         # applicable to all instances in the cluster.
         # Corresponds to the JSON property `gceClusterConfig`
@@ -246,8 +387,8 @@ module Google
         # default, executables are run on master and all worker nodes. You can test a
         # node's <code>role</code> metadata to run an executable on a master or worker
         # node, as shown below using curl (you can also use wget):
-        # ROLE=$(curl -H Metadata-Flavor:Google http://metadata/computeMetadata/v1beta2/
-        # instance/attributes/dataproc-role)
+        # ROLE=$(curl -H Metadata-Flavor:Google
+        # http://metadata/computeMetadata/v1beta2/instance/attributes/dataproc-role)
         # if [[ "$`ROLE`" == 'Master' ]]; then
         # ... master specific actions ...
         # else
@@ -274,6 +415,11 @@ module Google
         # @return [Google::Apis::DataprocV1beta2::InstanceGroupConfig]
         attr_accessor :secondary_worker_config
       
+        # Security related configuration, including encryption, Kerberos, etc.
+        # Corresponds to the JSON property `securityConfig`
+        # @return [Google::Apis::DataprocV1beta2::SecurityConfig]
+        attr_accessor :security_config
+      
         # Specifies the selection and config of software inside the cluster.
         # Corresponds to the JSON property `softwareConfig`
         # @return [Google::Apis::DataprocV1beta2::SoftwareConfig]
@@ -291,13 +437,16 @@ module Google
       
         # Update properties of this object
         def update!(**args)
+          @autoscaling_config = args[:autoscaling_config] if args.key?(:autoscaling_config)
           @config_bucket = args[:config_bucket] if args.key?(:config_bucket)
           @encryption_config = args[:encryption_config] if args.key?(:encryption_config)
+          @endpoint_config = args[:endpoint_config] if args.key?(:endpoint_config)
           @gce_cluster_config = args[:gce_cluster_config] if args.key?(:gce_cluster_config)
           @initialization_actions = args[:initialization_actions] if args.key?(:initialization_actions)
           @lifecycle_config = args[:lifecycle_config] if args.key?(:lifecycle_config)
           @master_config = args[:master_config] if args.key?(:master_config)
           @secondary_worker_config = args[:secondary_worker_config] if args.key?(:secondary_worker_config)
+          @security_config = args[:security_config] if args.key?(:security_config)
           @software_config = args[:software_config] if args.key?(:software_config)
           @worker_config = args[:worker_config] if args.key?(:worker_config)
         end
@@ -634,6 +783,34 @@ module Google
         end
       end
       
+      # Endpoint config for this cluster
+      class EndpointConfig
+        include Google::Apis::Core::Hashable
+      
+        # Optional. If true, enable http access to specific ports on the cluster from
+        # external sources. Defaults to false.
+        # Corresponds to the JSON property `enableHttpPortAccess`
+        # @return [Boolean]
+        attr_accessor :enable_http_port_access
+        alias_method :enable_http_port_access?, :enable_http_port_access
+      
+        # Output only. The map of port descriptions to URLs. Will only be populated if
+        # enable_http_port_access is true.
+        # Corresponds to the JSON property `httpPorts`
+        # @return [Hash<String,String>]
+        attr_accessor :http_ports
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @enable_http_port_access = args[:enable_http_port_access] if args.key?(:enable_http_port_access)
+          @http_ports = args[:http_ports] if args.key?(:http_ports)
+        end
+      end
+      
       # Represents an expression text. Example:
       # title: "User account presence"
       # description: "Determines whether the request has a user account"
@@ -684,11 +861,6 @@ module Google
       class GceClusterConfig
         include Google::Apis::Core::Hashable
       
-        # Allocation Affinity for consuming Zonal allocation.
-        # Corresponds to the JSON property `allocationAffinity`
-        # @return [Google::Apis::DataprocV1beta2::AllocationAffinity]
-        attr_accessor :allocation_affinity
-      
         # Optional. If true, all instances in the cluster will only have internal IP
         # addresses. By default, clusters are not restricted to internal IP addresses,
         # and will have ephemeral external IP addresses assigned to each instance. This
@@ -720,6 +892,11 @@ module Google
         # @return [String]
         attr_accessor :network_uri
       
+        # Reservation Affinity for consuming Zonal reservation.
+        # Corresponds to the JSON property `reservationAffinity`
+        # @return [Google::Apis::DataprocV1beta2::ReservationAffinity]
+        attr_accessor :reservation_affinity
+      
         # Optional. The service account of the instances. Defaults to the default
         # Compute Engine service account. Custom service accounts need permissions
         # equivalent to the following IAM roles:
@@ -779,10 +956,10 @@ module Google
       
         # Update properties of this object
         def update!(**args)
-          @allocation_affinity = args[:allocation_affinity] if args.key?(:allocation_affinity)
           @internal_ip_only = args[:internal_ip_only] if args.key?(:internal_ip_only)
           @metadata = args[:metadata] if args.key?(:metadata)
           @network_uri = args[:network_uri] if args.key?(:network_uri)
+          @reservation_affinity = args[:reservation_affinity] if args.key?(:reservation_affinity)
           @service_account = args[:service_account] if args.key?(:service_account)
           @service_account_scopes = args[:service_account_scopes] if args.key?(:service_account_scopes)
           @subnetwork_uri = args[:subnetwork_uri] if args.key?(:subnetwork_uri)
@@ -940,6 +1117,44 @@ module Google
         end
       end
       
+      # Configuration for the size bounds of an instance group, including its
+      # proportional size to other groups.
+      class InstanceGroupAutoscalingPolicyConfig
+        include Google::Apis::Core::Hashable
+      
+        # Optional. Maximum number of instances for this group. Required for primary
+        # workers. Note that by default, clusters will not use secondary workers.
+        # Required for secondary workers if the minimum secondary instances is set.
+        # Primary workers - Bounds: [min_instances, ). Required. Secondary workers -
+        # Bounds: [min_instances, ). Default: 0.
+        # Corresponds to the JSON property `maxInstances`
+        # @return [Fixnum]
+        attr_accessor :max_instances
+      
+        # Optional. Minimum number of instances for this group.Primary workers - Bounds:
+        # 2, max_instances. Default: 2. Secondary workers - Bounds: 0, max_instances.
+        # Default: 0.
+        # Corresponds to the JSON property `minInstances`
+        # @return [Fixnum]
+        attr_accessor :min_instances
+      
+        # 
+        # Corresponds to the JSON property `weight`
+        # @return [Fixnum]
+        attr_accessor :weight
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @max_instances = args[:max_instances] if args.key?(:max_instances)
+          @min_instances = args[:min_instances] if args.key?(:min_instances)
+          @weight = args[:weight] if args.key?(:weight)
+        end
+      end
+      
       # Optional. The config settings for Compute Engine resources in an instance
       # group, such as a master or worker group.
       class InstanceGroupConfig
@@ -1126,6 +1341,11 @@ module Google
         # @return [Google::Apis::DataprocV1beta2::JobPlacement]
         attr_accessor :placement
       
+        # A Cloud Dataproc job for running Presto (https://prestosql.io/) queries
+        # Corresponds to the JSON property `prestoJob`
+        # @return [Google::Apis::DataprocV1beta2::PrestoJob]
+        attr_accessor :presto_job
+      
         # A Cloud Dataproc job for running Apache PySpark (https://spark.apache.org/docs/
         # 0.9.0/python-programming-guide.html) applications on YARN.
         # Corresponds to the JSON property `pysparkJob`
@@ -1197,6 +1417,7 @@ module Google
           @labels = args[:labels] if args.key?(:labels)
           @pig_job = args[:pig_job] if args.key?(:pig_job)
           @placement = args[:placement] if args.key?(:placement)
+          @presto_job = args[:presto_job] if args.key?(:presto_job)
           @pyspark_job = args[:pyspark_job] if args.key?(:pyspark_job)
           @reference = args[:reference] if args.key?(:reference)
           @scheduling = args[:scheduling] if args.key?(:scheduling)
@@ -1240,11 +1461,10 @@ module Google
       class JobReference
         include Google::Apis::Core::Hashable
       
-        # Optional. The job ID, which must be unique within the project. The job ID is
-        # generated by the server upon job submission or provided by the user as a means
-        # to perform retries without creating duplicate jobs. The ID must contain only
-        # letters (a-z, A-Z), numbers (0-9), underscores (_), or hyphens (-). The
-        # maximum length is 100 characters.
+        # Optional. The job ID, which must be unique within the project.The ID must
+        # contain only letters (a-z, A-Z), numbers (0-9), underscores (_), or hyphens (-)
+        # . The maximum length is 100 characters.If not specified by the caller, the job
+        # ID will be provided by the server.
         # Corresponds to the JSON property `jobId`
         # @return [String]
         attr_accessor :job_id
@@ -1326,6 +1546,120 @@ module Google
         end
       end
       
+      # Specifies Kerberos related configuration.
+      class KerberosConfig
+        include Google::Apis::Core::Hashable
+      
+        # Optional. The admin server (IP or hostname) for the remote trusted realm in a
+        # cross realm trust relationship.
+        # Corresponds to the JSON property `crossRealmTrustAdminServer`
+        # @return [String]
+        attr_accessor :cross_realm_trust_admin_server
+      
+        # Optional. The KDC (IP or hostname) for the remote trusted realm in a cross
+        # realm trust relationship.
+        # Corresponds to the JSON property `crossRealmTrustKdc`
+        # @return [String]
+        attr_accessor :cross_realm_trust_kdc
+      
+        # Optional. The remote realm the Dataproc on-cluster KDC will trust, should the
+        # user enable cross realm trust.
+        # Corresponds to the JSON property `crossRealmTrustRealm`
+        # @return [String]
+        attr_accessor :cross_realm_trust_realm
+      
+        # Optional. The GCS uri of a KMS encrypted file containing the shared password
+        # between the on-cluster Kerberos realm and the remote trusted realm, in a cross
+        # realm trust relationship.
+        # Corresponds to the JSON property `crossRealmTrustSharedPasswordUri`
+        # @return [String]
+        attr_accessor :cross_realm_trust_shared_password_uri
+      
+        # Optional. Flag to indicate whether to Kerberize the cluster.
+        # Corresponds to the JSON property `enableKerberos`
+        # @return [Boolean]
+        attr_accessor :enable_kerberos
+        alias_method :enable_kerberos?, :enable_kerberos
+      
+        # Optional. The GCS uri of a KMS encrypted file containing the master key of the
+        # KDC database.
+        # Corresponds to the JSON property `kdcDbKeyUri`
+        # @return [String]
+        attr_accessor :kdc_db_key_uri
+      
+        # Optional. The GCS uri of a KMS encrypted file containing the password to the
+        # user provided key. For the self-signed certificate, this password is generated
+        # by Dataproc.
+        # Corresponds to the JSON property `keyPasswordUri`
+        # @return [String]
+        attr_accessor :key_password_uri
+      
+        # Optional. The GCS uri of a KMS encrypted file containing the password to the
+        # user provided keystore. For the self-signed certificate, this password is
+        # generated by Dataproc.
+        # Corresponds to the JSON property `keystorePasswordUri`
+        # @return [String]
+        attr_accessor :keystore_password_uri
+      
+        # Optional. The GCS uri of the keystore file used for SSL encryption. If not
+        # provided, Dataproc will provide a self-signed certificate.
+        # Corresponds to the JSON property `keystoreUri`
+        # @return [String]
+        attr_accessor :keystore_uri
+      
+        # Required. The uri of the KMS key used to encrypt various sensitive files.
+        # Corresponds to the JSON property `kmsKeyUri`
+        # @return [String]
+        attr_accessor :kms_key_uri
+      
+        # Required. The GCS uri of a KMS encrypted file containing the root principal
+        # password.
+        # Corresponds to the JSON property `rootPrincipalPasswordUri`
+        # @return [String]
+        attr_accessor :root_principal_password_uri
+      
+        # Optional. The lifetime of the ticket granting ticket, in hours. If not
+        # specified, or user specifies 0, then default value 10 will be used.
+        # Corresponds to the JSON property `tgtLifetimeHours`
+        # @return [Fixnum]
+        attr_accessor :tgt_lifetime_hours
+      
+        # Optional. The GCS uri of a KMS encrypted file containing the password to the
+        # user provided truststore. For the self-signed certificate, this password is
+        # generated by Dataproc.
+        # Corresponds to the JSON property `truststorePasswordUri`
+        # @return [String]
+        attr_accessor :truststore_password_uri
+      
+        # Optional. The GCS uri of the truststore file used for SSL encryption. If not
+        # provided, Dataproc will provide a self-signed certificate.
+        # Corresponds to the JSON property `truststoreUri`
+        # @return [String]
+        attr_accessor :truststore_uri
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @cross_realm_trust_admin_server = args[:cross_realm_trust_admin_server] if args.key?(:cross_realm_trust_admin_server)
+          @cross_realm_trust_kdc = args[:cross_realm_trust_kdc] if args.key?(:cross_realm_trust_kdc)
+          @cross_realm_trust_realm = args[:cross_realm_trust_realm] if args.key?(:cross_realm_trust_realm)
+          @cross_realm_trust_shared_password_uri = args[:cross_realm_trust_shared_password_uri] if args.key?(:cross_realm_trust_shared_password_uri)
+          @enable_kerberos = args[:enable_kerberos] if args.key?(:enable_kerberos)
+          @kdc_db_key_uri = args[:kdc_db_key_uri] if args.key?(:kdc_db_key_uri)
+          @key_password_uri = args[:key_password_uri] if args.key?(:key_password_uri)
+          @keystore_password_uri = args[:keystore_password_uri] if args.key?(:keystore_password_uri)
+          @keystore_uri = args[:keystore_uri] if args.key?(:keystore_uri)
+          @kms_key_uri = args[:kms_key_uri] if args.key?(:kms_key_uri)
+          @root_principal_password_uri = args[:root_principal_password_uri] if args.key?(:root_principal_password_uri)
+          @tgt_lifetime_hours = args[:tgt_lifetime_hours] if args.key?(:tgt_lifetime_hours)
+          @truststore_password_uri = args[:truststore_password_uri] if args.key?(:truststore_password_uri)
+          @truststore_uri = args[:truststore_uri] if args.key?(:truststore_uri)
+        end
+      end
+      
       # Specifies the cluster auto-delete schedule configuration.
       class LifecycleConfig
         include Google::Apis::Core::Hashable
@@ -1362,6 +1696,32 @@ module Google
         end
       end
       
+      # A response to a request to list autoscaling policies in a project.
+      class ListAutoscalingPoliciesResponse
+        include Google::Apis::Core::Hashable
+      
+        # Output only. This token is included in the response if there are more results
+        # to fetch.
+        # Corresponds to the JSON property `nextPageToken`
+        # @return [String]
+        attr_accessor :next_page_token
+      
+        # Output only. Autoscaling policies list.
+        # Corresponds to the JSON property `policies`
+        # @return [Array<Google::Apis::DataprocV1beta2::AutoscalingPolicy>]
+        attr_accessor :policies
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @next_page_token = args[:next_page_token] if args.key?(:next_page_token)
+          @policies = args[:policies] if args.key?(:policies)
+        end
+      end
+      
       # The list of all clusters in a project.
       class ListClustersResponse
         include Google::Apis::Core::Hashable
@@ -1926,6 +2286,67 @@ module Google
         end
       end
       
+      # A Cloud Dataproc job for running Presto (https://prestosql.io/) queries
+      class PrestoJob
+        include Google::Apis::Core::Hashable
+      
+        # Optional. Presto client tags to attach to this query
+        # Corresponds to the JSON property `clientTags`
+        # @return [Array<String>]
+        attr_accessor :client_tags
+      
+        # Optional. Whether to continue executing queries if a query fails. The default
+        # value is false. Setting to true can be useful when executing independent
+        # parallel queries.
+        # Corresponds to the JSON property `continueOnFailure`
+        # @return [Boolean]
+        attr_accessor :continue_on_failure
+        alias_method :continue_on_failure?, :continue_on_failure
+      
+        # The runtime logging config of the job.
+        # Corresponds to the JSON property `loggingConfig`
+        # @return [Google::Apis::DataprocV1beta2::LoggingConfig]
+        attr_accessor :logging_config
+      
+        # Optional. The format in which query output will be displayed. See the Presto
+        # documentation for supported output formats
+        # Corresponds to the JSON property `outputFormat`
+        # @return [String]
+        attr_accessor :output_format
+      
+        # Optional. A mapping of property names to values. Used to set Presto session
+        # properties (https://prestodb.io/docs/current/sql/set-session.html) Equivalent
+        # to using the --session flag in the Presto CLI
+        # Corresponds to the JSON property `properties`
+        # @return [Hash<String,String>]
+        attr_accessor :properties
+      
+        # The HCFS URI of the script that contains SQL queries.
+        # Corresponds to the JSON property `queryFileUri`
+        # @return [String]
+        attr_accessor :query_file_uri
+      
+        # A list of queries to run on a cluster.
+        # Corresponds to the JSON property `queryList`
+        # @return [Google::Apis::DataprocV1beta2::QueryList]
+        attr_accessor :query_list
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @client_tags = args[:client_tags] if args.key?(:client_tags)
+          @continue_on_failure = args[:continue_on_failure] if args.key?(:continue_on_failure)
+          @logging_config = args[:logging_config] if args.key?(:logging_config)
+          @output_format = args[:output_format] if args.key?(:output_format)
+          @properties = args[:properties] if args.key?(:properties)
+          @query_file_uri = args[:query_file_uri] if args.key?(:query_file_uri)
+          @query_list = args[:query_list] if args.key?(:query_list)
+        end
+      end
+      
       # A Cloud Dataproc job for running Apache PySpark (https://spark.apache.org/docs/
       # 0.9.0/python-programming-guide.html) applications on YARN.
       class PySparkJob
@@ -2050,6 +2471,56 @@ module Google
         end
       end
       
+      # Reservation Affinity for consuming Zonal reservation.
+      class ReservationAffinity
+        include Google::Apis::Core::Hashable
+      
+        # Optional. Type of reservation to consume
+        # Corresponds to the JSON property `consumeReservationType`
+        # @return [String]
+        attr_accessor :consume_reservation_type
+      
+        # Optional. Corresponds to the label key of reservation resource.
+        # Corresponds to the JSON property `key`
+        # @return [String]
+        attr_accessor :key
+      
+        # Optional. Corresponds to the label values of reservation resource.
+        # Corresponds to the JSON property `values`
+        # @return [Array<String>]
+        attr_accessor :values
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @consume_reservation_type = args[:consume_reservation_type] if args.key?(:consume_reservation_type)
+          @key = args[:key] if args.key?(:key)
+          @values = args[:values] if args.key?(:values)
+        end
+      end
+      
+      # Security related configuration, including encryption, Kerberos, etc.
+      class SecurityConfig
+        include Google::Apis::Core::Hashable
+      
+        # Specifies Kerberos related configuration.
+        # Corresponds to the JSON property `kerberosConfig`
+        # @return [Google::Apis::DataprocV1beta2::KerberosConfig]
+        attr_accessor :kerberos_config
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @kerberos_config = args[:kerberos_config] if args.key?(:kerberos_config)
+        end
+      end
+      
       # Request message for SetIamPolicy method.
       class SetIamPolicyRequest
         include Google::Apis::Core::Hashable
@@ -2121,8 +2592,8 @@ module Google
         attr_accessor :optional_components
       
         # Optional. The properties to set on daemon config files.Property keys are
-        # specified in prefix:property format, such as core:fs.defaultFS. The following
-        # are supported prefixes and their mappings:
+        # specified in prefix:property format, for example core:hadoop.tmp.dir. The
+        # following are supported prefixes and their mappings:
         # capacity-scheduler: capacity-scheduler.xml
         # core: core-site.xml
         # distcp: distcp-default.xml