google-apis-dataproc_v1 0.15.0 → 0.19.0

data/lib/google/apis/dataproc_v1/classes.rb

@@ -98,6 +98,16 @@ module Google
         # @return [String]
         attr_accessor :id
       
+        # Optional. The labels to associate with this autoscaling policy. Label keys
+        # must contain 1 to 63 characters, and must conform to RFC 1035 (https://www.
+        # ietf.org/rfc/rfc1035.txt). Label values may be empty, but, if present, must
+        # contain 1 to 63 characters, and must conform to RFC 1035 (https://www.ietf.org/
+        # rfc/rfc1035.txt). No more than 32 labels can be associated with an autoscaling
+        # policy.
+        # Corresponds to the JSON property `labels`
+        # @return [Hash<String,String>]
+        attr_accessor :labels
+      
         # Output only. The "resource name" of the autoscaling policy, as described in
         # https://cloud.google.com/apis/design/resource_names. For projects.regions.
         # autoscalingPolicies, the resource name of the policy has the following format:
@@ -129,6 +139,7 @@ module Google
         def update!(**args)
           @basic_algorithm = args[:basic_algorithm] if args.key?(:basic_algorithm)
           @id = args[:id] if args.key?(:id)
+          @labels = args[:labels] if args.key?(:labels)
           @name = args[:name] if args.key?(:name)
           @secondary_worker_config = args[:secondary_worker_config] if args.key?(:secondary_worker_config)
           @worker_config = args[:worker_config] if args.key?(:worker_config)
@@ -146,6 +157,11 @@ module Google
         # @return [String]
         attr_accessor :cooldown_period
       
+        # Basic autoscaling configurations for Spark Standalone.
+        # Corresponds to the JSON property `sparkStandaloneConfig`
+        # @return [Google::Apis::DataprocV1::SparkStandaloneAutoscalingConfig]
+        attr_accessor :spark_standalone_config
+      
         # Basic autoscaling configurations for YARN.
         # Corresponds to the JSON property `yarnConfig`
         # @return [Google::Apis::DataprocV1::BasicYarnAutoscalingConfig]
@@ -158,6 +174,7 @@ module Google
         # Update properties of this object
         def update!(**args)
           @cooldown_period = args[:cooldown_period] if args.key?(:cooldown_period)
+          @spark_standalone_config = args[:spark_standalone_config] if args.key?(:spark_standalone_config)
           @yarn_config = args[:yarn_config] if args.key?(:yarn_config)
         end
       end
@@ -228,6 +245,131 @@ module Google
         end
       end
       
+      # A representation of a batch workload in the service.
+      class Batch
+        include Google::Apis::Core::Hashable
+      
+        # Output only. The time when the batch was created.
+        # Corresponds to the JSON property `createTime`
+        # @return [String]
+        attr_accessor :create_time
+      
+        # Output only. The email address of the user who created the batch.
+        # Corresponds to the JSON property `creator`
+        # @return [String]
+        attr_accessor :creator
+      
+        # Environment configuration for a workload.
+        # Corresponds to the JSON property `environmentConfig`
+        # @return [Google::Apis::DataprocV1::EnvironmentConfig]
+        attr_accessor :environment_config
+      
+        # Optional. The labels to associate with this batch. Label keys must contain 1
+        # to 63 characters, and must conform to RFC 1035 (https://www.ietf.org/rfc/
+        # rfc1035.txt). Label values may be empty, but, if present, must contain 1 to 63
+        # characters, and must conform to RFC 1035 (https://www.ietf.org/rfc/rfc1035.txt)
+        # . No more than 32 labels can be associated with a batch.
+        # Corresponds to the JSON property `labels`
+        # @return [Hash<String,String>]
+        attr_accessor :labels
+      
+        # Output only. The resource name of the batch.
+        # Corresponds to the JSON property `name`
+        # @return [String]
+        attr_accessor :name
+      
+        # Output only. The resource name of the operation associated with this batch.
+        # Corresponds to the JSON property `operation`
+        # @return [String]
+        attr_accessor :operation
+      
+        # A configuration for running an Apache PySpark (https://spark.apache.org/docs/
+        # latest/api/python/getting_started/quickstart.html) batch workload.
+        # Corresponds to the JSON property `pysparkBatch`
+        # @return [Google::Apis::DataprocV1::PySparkBatch]
+        attr_accessor :pyspark_batch
+      
+        # Runtime configuration for a workload.
+        # Corresponds to the JSON property `runtimeConfig`
+        # @return [Google::Apis::DataprocV1::RuntimeConfig]
+        attr_accessor :runtime_config
+      
+        # Runtime information about workload execution.
+        # Corresponds to the JSON property `runtimeInfo`
+        # @return [Google::Apis::DataprocV1::RuntimeInfo]
+        attr_accessor :runtime_info
+      
+        # A configuration for running an Apache Spark (http://spark.apache.org/) batch
+        # workload.
+        # Corresponds to the JSON property `sparkBatch`
+        # @return [Google::Apis::DataprocV1::SparkBatch]
+        attr_accessor :spark_batch
+      
+        # A configuration for running an Apache SparkR (https://spark.apache.org/docs/
+        # latest/sparkr.html) batch workload.
+        # Corresponds to the JSON property `sparkRBatch`
+        # @return [Google::Apis::DataprocV1::SparkRBatch]
+        attr_accessor :spark_r_batch
+      
+        # A configuration for running Apache Spark SQL (http://spark.apache.org/sql/)
+        # queries as a batch workload.
+        # Corresponds to the JSON property `sparkSqlBatch`
+        # @return [Google::Apis::DataprocV1::SparkSqlBatch]
+        attr_accessor :spark_sql_batch
+      
+        # Output only. The state of the batch.
+        # Corresponds to the JSON property `state`
+        # @return [String]
+        attr_accessor :state
+      
+        # Output only. Historical state information for the batch.
+        # Corresponds to the JSON property `stateHistory`
+        # @return [Array<Google::Apis::DataprocV1::StateHistory>]
+        attr_accessor :state_history
+      
+        # Output only. Batch state details, such as a failure description if the state
+        # is FAILED.
+        # Corresponds to the JSON property `stateMessage`
+        # @return [String]
+        attr_accessor :state_message
+      
+        # Output only. The time when the batch entered a current state.
+        # Corresponds to the JSON property `stateTime`
+        # @return [String]
+        attr_accessor :state_time
+      
+        # Output only. A batch UUID (Unique Universal Identifier). The service generates
+        # this value when it creates the batch.
+        # Corresponds to the JSON property `uuid`
+        # @return [String]
+        attr_accessor :uuid
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @create_time = args[:create_time] if args.key?(:create_time)
+          @creator = args[:creator] if args.key?(:creator)
+          @environment_config = args[:environment_config] if args.key?(:environment_config)
+          @labels = args[:labels] if args.key?(:labels)
+          @name = args[:name] if args.key?(:name)
+          @operation = args[:operation] if args.key?(:operation)
+          @pyspark_batch = args[:pyspark_batch] if args.key?(:pyspark_batch)
+          @runtime_config = args[:runtime_config] if args.key?(:runtime_config)
+          @runtime_info = args[:runtime_info] if args.key?(:runtime_info)
+          @spark_batch = args[:spark_batch] if args.key?(:spark_batch)
+          @spark_r_batch = args[:spark_r_batch] if args.key?(:spark_r_batch)
+          @spark_sql_batch = args[:spark_sql_batch] if args.key?(:spark_sql_batch)
+          @state = args[:state] if args.key?(:state)
+          @state_history = args[:state_history] if args.key?(:state_history)
+          @state_message = args[:state_message] if args.key?(:state_message)
+          @state_time = args[:state_time] if args.key?(:state_time)
+          @uuid = args[:uuid] if args.key?(:uuid)
+        end
+      end
+      
       # Metadata describing the Batch operation.
       class BatchOperationMetadata
         include Google::Apis::Core::Hashable
@@ -289,7 +431,7 @@ module Google
         end
       end
       
-      # Associates members with a role.
+      # Associates members, or principals, with a role.
       class Binding
         include Google::Apis::Core::Hashable
       
@@ -312,7 +454,7 @@ module Google
         # @return [Google::Apis::DataprocV1::Expr]
         attr_accessor :condition
       
-        # Specifies the identities requesting access for a Cloud Platform resource.
+        # Specifies the principals requesting access for a Cloud Platform resource.
         # members can have the following values: allUsers: A special identifier that
         # represents anyone who is on the internet; with or without a Google account.
         # allAuthenticatedUsers: A special identifier that represents anyone who is
@@ -341,8 +483,8 @@ module Google
         # @return [Array<String>]
         attr_accessor :members
       
-        # Role that is assigned to members. For example, roles/viewer, roles/editor, or
-        # roles/owner.
+        # Role that is assigned to the list of members, or principals. For example,
+        # roles/viewer, roles/editor, or roles/owner.
         # Corresponds to the JSON property `role`
         # @return [String]
         attr_accessor :role
@@ -867,6 +1009,14 @@ module Google
         # @return [String]
         attr_accessor :boot_disk_type
       
+        # Optional. Interface type of local SSDs (default is "scsi"). Valid values: "
+        # scsi" (Small Computer System Interface), "nvme" (Non-Volatile Memory Express).
+        # See SSD Interface types (https://cloud.google.com/compute/docs/disks/local-ssd#
+        # performance).
+        # Corresponds to the JSON property `localSsdInterface`
+        # @return [String]
+        attr_accessor :local_ssd_interface
+      
         # Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not
         # attached, the boot disk is used to store runtime logs and HDFS (https://hadoop.
         # apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are
@@ -884,6 +1034,7 @@ module Google
         def update!(**args)
           @boot_disk_size_gb = args[:boot_disk_size_gb] if args.key?(:boot_disk_size_gb)
           @boot_disk_type = args[:boot_disk_type] if args.key?(:boot_disk_type)
+          @local_ssd_interface = args[:local_ssd_interface] if args.key?(:local_ssd_interface)
           @num_local_ssds = args[:num_local_ssds] if args.key?(:num_local_ssds)
         end
       end
@@ -953,6 +1104,74 @@ module Google
         end
       end
       
+      # Environment configuration for a workload.
+      class EnvironmentConfig
+        include Google::Apis::Core::Hashable
+      
+        # Execution configuration for a workload.
+        # Corresponds to the JSON property `executionConfig`
+        # @return [Google::Apis::DataprocV1::ExecutionConfig]
+        attr_accessor :execution_config
+      
+        # Auxiliary services configuration for a workload.
+        # Corresponds to the JSON property `peripheralsConfig`
+        # @return [Google::Apis::DataprocV1::PeripheralsConfig]
+        attr_accessor :peripherals_config
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @execution_config = args[:execution_config] if args.key?(:execution_config)
+          @peripherals_config = args[:peripherals_config] if args.key?(:peripherals_config)
+        end
+      end
+      
+      # Execution configuration for a workload.
+      class ExecutionConfig
+        include Google::Apis::Core::Hashable
+      
+        # Optional. The Cloud KMS key to use for encryption.
+        # Corresponds to the JSON property `kmsKey`
+        # @return [String]
+        attr_accessor :kms_key
+      
+        # Optional. Tags used for network traffic control.
+        # Corresponds to the JSON property `networkTags`
+        # @return [Array<String>]
+        attr_accessor :network_tags
+      
+        # Optional. Network URI to connect workload to.
+        # Corresponds to the JSON property `networkUri`
+        # @return [String]
+        attr_accessor :network_uri
+      
+        # Optional. Service account that used to execute workload.
+        # Corresponds to the JSON property `serviceAccount`
+        # @return [String]
+        attr_accessor :service_account
+      
+        # Optional. Subnetwork URI to connect workload to.
+        # Corresponds to the JSON property `subnetworkUri`
+        # @return [String]
+        attr_accessor :subnetwork_uri
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @kms_key = args[:kms_key] if args.key?(:kms_key)
+          @network_tags = args[:network_tags] if args.key?(:network_tags)
+          @network_uri = args[:network_uri] if args.key?(:network_uri)
+          @service_account = args[:service_account] if args.key?(:service_account)
+          @subnetwork_uri = args[:subnetwork_uri] if args.key?(:subnetwork_uri)
+        end
+      end
+      
       # Represents a textual expression in the Common Expression Language (CEL) syntax.
       # CEL is a C-like expression language. The syntax and semantics of CEL are
       # documented at https://github.com/google/cel-spec.Example (Comparison): title: "
@@ -1164,12 +1383,16 @@ module Google
       class GetPolicyOptions
         include Google::Apis::Core::Hashable
       
-        # Optional. The policy format version to be returned.Valid values are 0, 1, and
-        # 3. Requests specifying an invalid value will be rejected.Requests for policies
-        # with any conditional bindings must specify version 3. Policies without any
-        # conditional bindings may specify any valid value or leave the field unset.To
-        # learn which resources support conditions in their IAM policies, see the IAM
-        # documentation (https://cloud.google.com/iam/help/conditions/resource-policies).
+        # Optional. The maximum policy version that will be used to format the policy.
+        # Valid values are 0, 1, and 3. Requests specifying an invalid value will be
+        # rejected.Requests for policies with any conditional role bindings must specify
+        # version 3. Policies with no conditional role bindings may specify any valid
+        # value or leave the field unset.The policy in the response might use the policy
+        # version that you specified, or it might use a lower policy version. For
+        # example, if you specify version 3, but the policy has no conditional role
+        # bindings, the response uses version 1.To learn which resources support
+        # conditions in their IAM policies, see the IAM documentation (https://cloud.
+        # google.com/iam/help/conditions/resource-policies).
         # Corresponds to the JSON property `requestedPolicyVersion`
         # @return [Fixnum]
         attr_accessor :requested_policy_version
@@ -1876,14 +2099,19 @@ module Google
         # Optional. Maximum number of times per hour a driver may be restarted as a
         # result of driver exiting with non-zero code before job is reported failed.A
         # job may be reported as thrashing if driver exits with non-zero code 4 times
-        # within 10 minute window.Maximum value is 10.
+        # within 10 minute window.Maximum value is 10.Note: Currently, this restartable
+        # job option is not supported in Dataproc workflow template (https://cloud.
+        # google.com/dataproc/docs/concepts/workflows/using-workflows#
+        # adding_jobs_to_a_template) jobs.
         # Corresponds to the JSON property `maxFailuresPerHour`
         # @return [Fixnum]
         attr_accessor :max_failures_per_hour
       
         # Optional. Maximum number of times in total a driver may be restarted as a
         # result of driver exiting with non-zero code before job is reported failed.
-        # Maximum value is 240.
+        # Maximum value is 240.Note: Currently, this restartable job option is not
+        # supported in Dataproc workflow template (https://cloud.google.com/dataproc/
+        # docs/concepts/workflows/using-workflows#adding_jobs_to_a_template) jobs.
         # Corresponds to the JSON property `maxFailuresTotal`
         # @return [Fixnum]
         attr_accessor :max_failures_total
@@ -2133,6 +2361,32 @@ module Google
         end
       end
       
+      # A list of batch workloads.
+      class ListBatchesResponse
+        include Google::Apis::Core::Hashable
+      
+        # The batches from the specified collection.
+        # Corresponds to the JSON property `batches`
+        # @return [Array<Google::Apis::DataprocV1::Batch>]
+        attr_accessor :batches
+      
+        # A token, which can be sent as page_token to retrieve the next page. If this
+        # field is omitted, there are no subsequent pages.
+        # Corresponds to the JSON property `nextPageToken`
+        # @return [String]
+        attr_accessor :next_page_token
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @batches = args[:batches] if args.key?(:batches)
+          @next_page_token = args[:next_page_token] if args.key?(:next_page_token)
+        end
+      end
+      
       # The list of all clusters in a project.
       class ListClustersResponse
         include Google::Apis::Core::Hashable
@@ -2619,6 +2873,32 @@ module Google
         end
       end
       
+      # Auxiliary services configuration for a workload.
+      class PeripheralsConfig
+        include Google::Apis::Core::Hashable
+      
+        # Optional. Resource name of an existing Dataproc Metastore service.Example:
+        # projects/[project_id]/locations/[region]/services/[service_id]
+        # Corresponds to the JSON property `metastoreService`
+        # @return [String]
+        attr_accessor :metastore_service
+      
+        # Spark History Server configuration for the workload.
+        # Corresponds to the JSON property `sparkHistoryServerConfig`
+        # @return [Google::Apis::DataprocV1::SparkHistoryServerConfig]
+        attr_accessor :spark_history_server_config
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @metastore_service = args[:metastore_service] if args.key?(:metastore_service)
+          @spark_history_server_config = args[:spark_history_server_config] if args.key?(:spark_history_server_config)
+        end
+      end
+      
       # A Dataproc job for running Apache Pig (https://pig.apache.org/) queries on
       # YARN.
       class PigJob
@@ -2685,16 +2965,16 @@ module Google
       
       # An Identity and Access Management (IAM) policy, which specifies access
       # controls for Google Cloud resources.A Policy is a collection of bindings. A
-      # binding binds one or more members to a single role. Members can be user
-      # accounts, service accounts, Google groups, and domains (such as G Suite). A
-      # role is a named list of permissions; each role can be an IAM predefined role
-      # or a user-created custom role.For some types of Google Cloud resources, a
-      # binding can also specify a condition, which is a logical expression that
-      # allows access to a resource only if the expression evaluates to true. A
-      # condition can add constraints based on attributes of the request, the resource,
-      # or both. To learn which resources support conditions in their IAM policies,
-      # see the IAM documentation (https://cloud.google.com/iam/help/conditions/
-      # resource-policies).JSON example: ` "bindings": [ ` "role": "roles/
+      # binding binds one or more members, or principals, to a single role. Principals
+      # can be user accounts, service accounts, Google groups, and domains (such as G
+      # Suite). A role is a named list of permissions; each role can be an IAM
+      # predefined role or a user-created custom role.For some types of Google Cloud
+      # resources, a binding can also specify a condition, which is a logical
+      # expression that allows access to a resource only if the expression evaluates
+      # to true. A condition can add constraints based on attributes of the request,
+      # the resource, or both. To learn which resources support conditions in their
+      # IAM policies, see the IAM documentation (https://cloud.google.com/iam/help/
+      # conditions/resource-policies).JSON example: ` "bindings": [ ` "role": "roles/
       # resourcemanager.organizationAdmin", "members": [ "user:mike@example.com", "
       # group:admins@example.com", "domain:google.com", "serviceAccount:my-project-id@
       # appspot.gserviceaccount.com" ] `, ` "role": "roles/resourcemanager.
@@ -2713,13 +2993,14 @@ module Google
       class Policy
         include Google::Apis::Core::Hashable
       
-        # Associates a list of members to a role. Optionally, may specify a condition
-        # that determines how and when the bindings are applied. Each of the bindings
-        # must contain at least one member.The bindings in a Policy can refer to up to 1,
-        # 500 members; up to 250 of these members can be Google groups. Each occurrence
-        # of a member counts towards these limits. For example, if the bindings grant 50
-        # different roles to user:alice@example.com, and not to any other member, then
-        # you can add another 1,450 members to the bindings in the Policy.
+        # Associates a list of members, or principals, with a role. Optionally, may
+        # specify a condition that determines how and when the bindings are applied.
+        # Each of the bindings must contain at least one principal.The bindings in a
+        # Policy can refer to up to 1,500 principals; up to 250 of these principals can
+        # be Google groups. Each occurrence of a principal counts towards these limits.
+        # For example, if the bindings grant 50 different roles to user:alice@example.
+        # com, and not to any other principal, then you can add another 1,450 principals
+        # to the bindings in the Policy.
         # Corresponds to the JSON property `bindings`
         # @return [Array<Google::Apis::DataprocV1::Binding>]
         attr_accessor :bindings
@@ -2833,6 +3114,63 @@ module Google
         end
       end
       
+      # A configuration for running an Apache PySpark (https://spark.apache.org/docs/
+      # latest/api/python/getting_started/quickstart.html) batch workload.
+      class PySparkBatch
+        include Google::Apis::Core::Hashable
+      
+        # Optional. HCFS URIs of archives to be extracted into the working directory of
+        # each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.
+        # Corresponds to the JSON property `archiveUris`
+        # @return [Array<String>]
+        attr_accessor :archive_uris
+      
+        # Optional. The arguments to pass to the driver. Do not include arguments that
+        # can be set as batch properties, such as --conf, since a collision can occur
+        # that causes an incorrect batch submission.
+        # Corresponds to the JSON property `args`
+        # @return [Array<String>]
+        attr_accessor :args
+      
+        # Optional. HCFS URIs of files to be placed in the working directory of each
+        # executor.
+        # Corresponds to the JSON property `fileUris`
+        # @return [Array<String>]
+        attr_accessor :file_uris
+      
+        # Optional. HCFS URIs of jar files to add to the classpath of the Spark driver
+        # and tasks.
+        # Corresponds to the JSON property `jarFileUris`
+        # @return [Array<String>]
+        attr_accessor :jar_file_uris
+      
+        # Required. The HCFS URI of the main Python file to use as the Spark driver.
+        # Must be a .py file.
+        # Corresponds to the JSON property `mainPythonFileUri`
+        # @return [String]
+        attr_accessor :main_python_file_uri
+      
+        # Optional. HCFS file URIs of Python files to pass to the PySpark framework.
+        # Supported file types: .py, .egg, and .zip.
+        # Corresponds to the JSON property `pythonFileUris`
+        # @return [Array<String>]
+        attr_accessor :python_file_uris
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @archive_uris = args[:archive_uris] if args.key?(:archive_uris)
+          @args = args[:args] if args.key?(:args)
+          @file_uris = args[:file_uris] if args.key?(:file_uris)
+          @jar_file_uris = args[:jar_file_uris] if args.key?(:jar_file_uris)
+          @main_python_file_uri = args[:main_python_file_uri] if args.key?(:main_python_file_uri)
+          @python_file_uris = args[:python_file_uris] if args.key?(:python_file_uris)
+        end
+      end
+      
       # A Dataproc job for running Apache PySpark (https://spark.apache.org/docs/0.9.0/
       # python-programming-guide.html) applications on YARN.
       class PySparkJob
@@ -3012,6 +3350,72 @@ module Google
         end
       end
       
+      # Runtime configuration for a workload.
+      class RuntimeConfig
+        include Google::Apis::Core::Hashable
+      
+        # Optional. Optional custom container image for the job runtime environment. If
+        # not specified, a default container image will be used.
+        # Corresponds to the JSON property `containerImage`
+        # @return [String]
+        attr_accessor :container_image
+      
+        # Optional. A mapping of property names to values, which are used to configure
+        # workload execution.
+        # Corresponds to the JSON property `properties`
+        # @return [Hash<String,String>]
+        attr_accessor :properties
+      
+        # Optional. Version of the batch runtime.
+        # Corresponds to the JSON property `version`
+        # @return [String]
+        attr_accessor :version
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @container_image = args[:container_image] if args.key?(:container_image)
+          @properties = args[:properties] if args.key?(:properties)
+          @version = args[:version] if args.key?(:version)
+        end
+      end
+      
+      # Runtime information about workload execution.
+      class RuntimeInfo
+        include Google::Apis::Core::Hashable
+      
+        # Output only. A URI pointing to the location of the diagnostics tarball.
+        # Corresponds to the JSON property `diagnosticOutputUri`
+        # @return [String]
+        attr_accessor :diagnostic_output_uri
+      
+        # Output only. Map of remote access endpoints (such as web interfaces and APIs)
+        # to their URIs.
+        # Corresponds to the JSON property `endpoints`
+        # @return [Hash<String,String>]
+        attr_accessor :endpoints
+      
+        # Output only. A URI pointing to the location of the stdout and stderr of the
+        # workload.
+        # Corresponds to the JSON property `outputUri`
+        # @return [String]
+        attr_accessor :output_uri
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @diagnostic_output_uri = args[:diagnostic_output_uri] if args.key?(:diagnostic_output_uri)
+          @endpoints = args[:endpoints] if args.key?(:endpoints)
+          @output_uri = args[:output_uri] if args.key?(:output_uri)
+        end
+      end
+      
       # Security related configuration, including encryption, Kerberos, etc.
       class SecurityConfig
         include Google::Apis::Core::Hashable
@@ -3105,16 +3509,16 @@ module Google
       
         # An Identity and Access Management (IAM) policy, which specifies access
         # controls for Google Cloud resources.A Policy is a collection of bindings. A
-        # binding binds one or more members to a single role. Members can be user
-        # accounts, service accounts, Google groups, and domains (such as G Suite). A
-        # role is a named list of permissions; each role can be an IAM predefined role
-        # or a user-created custom role.For some types of Google Cloud resources, a
-        # binding can also specify a condition, which is a logical expression that
-        # allows access to a resource only if the expression evaluates to true. A
-        # condition can add constraints based on attributes of the request, the resource,
-        # or both. To learn which resources support conditions in their IAM policies,
-        # see the IAM documentation (https://cloud.google.com/iam/help/conditions/
-        # resource-policies).JSON example: ` "bindings": [ ` "role": "roles/
+        # binding binds one or more members, or principals, to a single role. Principals
+        # can be user accounts, service accounts, Google groups, and domains (such as G
+        # Suite). A role is a named list of permissions; each role can be an IAM
+        # predefined role or a user-created custom role.For some types of Google Cloud
+        # resources, a binding can also specify a condition, which is a logical
+        # expression that allows access to a resource only if the expression evaluates
+        # to true. A condition can add constraints based on attributes of the request,
+        # the resource, or both. To learn which resources support conditions in their
+        # IAM policies, see the IAM documentation (https://cloud.google.com/iam/help/
+        # conditions/resource-policies).JSON example: ` "bindings": [ ` "role": "roles/
         # resourcemanager.organizationAdmin", "members": [ "user:mike@example.com", "
         # group:admins@example.com", "domain:google.com", "serviceAccount:my-project-id@
         # appspot.gserviceaccount.com" ] `, ` "role": "roles/resourcemanager.
@@ -3222,6 +3626,83 @@ module Google
         end
       end
       
+      # A configuration for running an Apache Spark (http://spark.apache.org/) batch
+      # workload.
+      class SparkBatch
+        include Google::Apis::Core::Hashable
+      
+        # Optional. HCFS URIs of archives to be extracted into the working directory of
+        # each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.
+        # Corresponds to the JSON property `archiveUris`
+        # @return [Array<String>]
+        attr_accessor :archive_uris
+      
+        # Optional. The arguments to pass to the driver. Do not include arguments that
+        # can be set as batch properties, such as --conf, since a collision can occur
+        # that causes an incorrect batch submission.
+        # Corresponds to the JSON property `args`
+        # @return [Array<String>]
+        attr_accessor :args
+      
+        # Optional. HCFS URIs of files to be placed in the working directory of each
+        # executor.
+        # Corresponds to the JSON property `fileUris`
+        # @return [Array<String>]
+        attr_accessor :file_uris
+      
+        # Optional. HCFS URIs of jar files to add to the classpath of the Spark driver
+        # and tasks.
+        # Corresponds to the JSON property `jarFileUris`
+        # @return [Array<String>]
+        attr_accessor :jar_file_uris
+      
+        # Optional. The name of the driver main class. The jar file that contains the
+        # class must be in the classpath or specified in jar_file_uris.
+        # Corresponds to the JSON property `mainClass`
+        # @return [String]
+        attr_accessor :main_class
+      
+        # Optional. The HCFS URI of the jar file that contains the main class.
+        # Corresponds to the JSON property `mainJarFileUri`
+        # @return [String]
+        attr_accessor :main_jar_file_uri
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @archive_uris = args[:archive_uris] if args.key?(:archive_uris)
+          @args = args[:args] if args.key?(:args)
+          @file_uris = args[:file_uris] if args.key?(:file_uris)
+          @jar_file_uris = args[:jar_file_uris] if args.key?(:jar_file_uris)
+          @main_class = args[:main_class] if args.key?(:main_class)
+          @main_jar_file_uri = args[:main_jar_file_uri] if args.key?(:main_jar_file_uri)
+        end
+      end
+      
+      # Spark History Server configuration for the workload.
+      class SparkHistoryServerConfig
+        include Google::Apis::Core::Hashable
+      
+        # Optional. Resource name of an existing Dataproc Cluster to act as a Spark
+        # History Server for the workload.Example: projects/[project_id]/regions/[region]
+        # /clusters/[cluster_name]
+        # Corresponds to the JSON property `dataprocCluster`
+        # @return [String]
+        attr_accessor :dataproc_cluster
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @dataproc_cluster = args[:dataproc_cluster] if args.key?(:dataproc_cluster)
+        end
+      end
+      
       # A Dataproc job for running Apache Spark (http://spark.apache.org/)
       # applications on YARN.
       class SparkJob
@@ -3293,6 +3774,49 @@ module Google
         end
       end
       
+      # A configuration for running an Apache SparkR (https://spark.apache.org/docs/
+      # latest/sparkr.html) batch workload.
+      class SparkRBatch
+        include Google::Apis::Core::Hashable
+      
+        # Optional. HCFS URIs of archives to be extracted into the working directory of
+        # each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.
+        # Corresponds to the JSON property `archiveUris`
+        # @return [Array<String>]
+        attr_accessor :archive_uris
+      
+        # Optional. The arguments to pass to the Spark driver. Do not include arguments
+        # that can be set as batch properties, such as --conf, since a collision can
+        # occur that causes an incorrect batch submission.
+        # Corresponds to the JSON property `args`
+        # @return [Array<String>]
+        attr_accessor :args
+      
+        # Optional. HCFS URIs of files to be placed in the working directory of each
+        # executor.
+        # Corresponds to the JSON property `fileUris`
+        # @return [Array<String>]
+        attr_accessor :file_uris
+      
+        # Required. The HCFS URI of the main R file to use as the driver. Must be a .R
+        # or .r file.
+        # Corresponds to the JSON property `mainRFileUri`
+        # @return [String]
+        attr_accessor :main_r_file_uri
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @archive_uris = args[:archive_uris] if args.key?(:archive_uris)
+          @args = args[:args] if args.key?(:args)
+          @file_uris = args[:file_uris] if args.key?(:file_uris)
+          @main_r_file_uri = args[:main_r_file_uri] if args.key?(:main_r_file_uri)
+        end
+      end
+      
       # A Dataproc job for running Apache SparkR (https://spark.apache.org/docs/latest/
       # sparkr.html) applications on YARN.
       class SparkRJob
@@ -3351,6 +3875,40 @@ module Google
         end
       end
       
+      # A configuration for running Apache Spark SQL (http://spark.apache.org/sql/)
+      # queries as a batch workload.
+      class SparkSqlBatch
+        include Google::Apis::Core::Hashable
+      
+        # Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.
+        # Corresponds to the JSON property `jarFileUris`
+        # @return [Array<String>]
+        attr_accessor :jar_file_uris
+      
+        # Required. The HCFS URI of the script that contains Spark SQL queries to
+        # execute.
+        # Corresponds to the JSON property `queryFileUri`
+        # @return [String]
+        attr_accessor :query_file_uri
+      
+        # Optional. Mapping of query variable names to values (equivalent to the Spark
+        # SQL command: SET name="value";).
+        # Corresponds to the JSON property `queryVariables`
+        # @return [Hash<String,String>]
+        attr_accessor :query_variables
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @jar_file_uris = args[:jar_file_uris] if args.key?(:jar_file_uris)
+          @query_file_uri = args[:query_file_uri] if args.key?(:query_file_uri)
+          @query_variables = args[:query_variables] if args.key?(:query_variables)
+        end
+      end
+      
       # A Dataproc job for running Apache Spark SQL (http://spark.apache.org/sql/)
       # queries.
       class SparkSqlJob
@@ -3404,6 +3962,68 @@ module Google
         end
       end
       
+      # Basic autoscaling configurations for Spark Standalone.
+      class SparkStandaloneAutoscalingConfig
+        include Google::Apis::Core::Hashable
+      
+        # Required. Timeout for Spark graceful decommissioning of spark workers.
+        # Specifies the duration to wait for spark worker to complete spark
+        # decomissioning tasks before forcefully removing workers. Only applicable to
+        # downscaling operations.Bounds: 0s, 1d.
+        # Corresponds to the JSON property `gracefulDecommissionTimeout`
+        # @return [String]
+        attr_accessor :graceful_decommission_timeout
+      
+        # Required. Fraction of required executors to remove from Spark Serverless
+        # clusters. A scale-down factor of 1.0 will result in scaling down so that there
+        # are no more executors for the Spark Job.(more aggressive scaling). A scale-
+        # down factor closer to 0 will result in a smaller magnitude of scaling donw (
+        # less aggressive scaling).Bounds: 0.0, 1.0.
+        # Corresponds to the JSON property `scaleDownFactor`
+        # @return [Float]
+        attr_accessor :scale_down_factor
+      
+        # Optional. Minimum scale-down threshold as a fraction of total cluster size
+        # before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1
+        # means the autoscaler must recommend at least a 2 worker scale-down for the
+        # cluster to scale. A threshold of 0 means the autoscaler will scale down on any
+        # recommended change.Bounds: 0.0, 1.0. Default: 0.0.
+        # Corresponds to the JSON property `scaleDownMinWorkerFraction`
+        # @return [Float]
+        attr_accessor :scale_down_min_worker_fraction
+      
+        # Required. Fraction of required workers to add to Spark Standalone clusters. A
+        # scale-up factor of 1.0 will result in scaling up so that there are no more
+        # required workers for the Spark Job (more aggressive scaling). A scale-up
+        # factor closer to 0 will result in a smaller magnitude of scaling up (less
+        # aggressive scaling).Bounds: 0.0, 1.0.
+        # Corresponds to the JSON property `scaleUpFactor`
+        # @return [Float]
+        attr_accessor :scale_up_factor
+      
+        # Optional. Minimum scale-up threshold as a fraction of total cluster size
+        # before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1
+        # means the autoscaler must recommend at least a 2-worker scale-up for the
+        # cluster to scale. A threshold of 0 means the autoscaler will scale up on any
+        # recommended change.Bounds: 0.0, 1.0. Default: 0.0.
+        # Corresponds to the JSON property `scaleUpMinWorkerFraction`
+        # @return [Float]
+        attr_accessor :scale_up_min_worker_fraction
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @graceful_decommission_timeout = args[:graceful_decommission_timeout] if args.key?(:graceful_decommission_timeout)
+          @scale_down_factor = args[:scale_down_factor] if args.key?(:scale_down_factor)
+          @scale_down_min_worker_fraction = args[:scale_down_min_worker_fraction] if args.key?(:scale_down_min_worker_fraction)
+          @scale_up_factor = args[:scale_up_factor] if args.key?(:scale_up_factor)
+          @scale_up_min_worker_fraction = args[:scale_up_min_worker_fraction] if args.key?(:scale_up_min_worker_fraction)
+        end
+      end
+      
       # A request to start a cluster.
       class StartClusterRequest
         include Google::Apis::Core::Hashable
@@ -3438,6 +4058,37 @@ module Google
         end
       end
       
+      # Historical state information.
+      class StateHistory
+        include Google::Apis::Core::Hashable
+      
+        # Output only. The state of the batch at this point in history.
+        # Corresponds to the JSON property `state`
+        # @return [String]
+        attr_accessor :state
+      
+        # Output only. Details about the state at this point in history.
+        # Corresponds to the JSON property `stateMessage`
+        # @return [String]
+        attr_accessor :state_message
+      
+        # Output only. The time when the batch entered the historical state.
+        # Corresponds to the JSON property `stateStartTime`
+        # @return [String]
+        attr_accessor :state_start_time
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @state = args[:state] if args.key?(:state)
+          @state_message = args[:state_message] if args.key?(:state_message)
+          @state_start_time = args[:state_start_time] if args.key?(:state_start_time)
+        end
+      end
+      
       # The Status type defines a logical error model that is suitable for different
       # programming environments, including REST APIs and RPC APIs. It is used by gRPC
       # (https://github.com/grpc). Each Status message contains three pieces of data: