google-apis-dataproc_v1 0.17.0 → 0.21.0

data/lib/google/apis/dataproc_v1/classes.rb

@@ -245,6 +245,131 @@ module Google
         end
       end
       
+      # A representation of a batch workload in the service.
+      class Batch
+        include Google::Apis::Core::Hashable
+      
+        # Output only. The time when the batch was created.
+        # Corresponds to the JSON property `createTime`
+        # @return [String]
+        attr_accessor :create_time
+      
+        # Output only. The email address of the user who created the batch.
+        # Corresponds to the JSON property `creator`
+        # @return [String]
+        attr_accessor :creator
+      
+        # Environment configuration for a workload.
+        # Corresponds to the JSON property `environmentConfig`
+        # @return [Google::Apis::DataprocV1::EnvironmentConfig]
+        attr_accessor :environment_config
+      
+        # Optional. The labels to associate with this batch. Label keys must contain 1
+        # to 63 characters, and must conform to RFC 1035 (https://www.ietf.org/rfc/
+        # rfc1035.txt). Label values may be empty, but, if present, must contain 1 to 63
+        # characters, and must conform to RFC 1035 (https://www.ietf.org/rfc/rfc1035.txt)
+        # . No more than 32 labels can be associated with a batch.
+        # Corresponds to the JSON property `labels`
+        # @return [Hash<String,String>]
+        attr_accessor :labels
+      
+        # Output only. The resource name of the batch.
+        # Corresponds to the JSON property `name`
+        # @return [String]
+        attr_accessor :name
+      
+        # Output only. The resource name of the operation associated with this batch.
+        # Corresponds to the JSON property `operation`
+        # @return [String]
+        attr_accessor :operation
+      
+        # A configuration for running an Apache PySpark (https://spark.apache.org/docs/
+        # latest/api/python/getting_started/quickstart.html) batch workload.
+        # Corresponds to the JSON property `pysparkBatch`
+        # @return [Google::Apis::DataprocV1::PySparkBatch]
+        attr_accessor :pyspark_batch
+      
+        # Runtime configuration for a workload.
+        # Corresponds to the JSON property `runtimeConfig`
+        # @return [Google::Apis::DataprocV1::RuntimeConfig]
+        attr_accessor :runtime_config
+      
+        # Runtime information about workload execution.
+        # Corresponds to the JSON property `runtimeInfo`
+        # @return [Google::Apis::DataprocV1::RuntimeInfo]
+        attr_accessor :runtime_info
+      
+        # A configuration for running an Apache Spark (https://spark.apache.org/) batch
+        # workload.
+        # Corresponds to the JSON property `sparkBatch`
+        # @return [Google::Apis::DataprocV1::SparkBatch]
+        attr_accessor :spark_batch
+      
+        # A configuration for running an Apache SparkR (https://spark.apache.org/docs/
+        # latest/sparkr.html) batch workload.
+        # Corresponds to the JSON property `sparkRBatch`
+        # @return [Google::Apis::DataprocV1::SparkRBatch]
+        attr_accessor :spark_r_batch
+      
+        # A configuration for running Apache Spark SQL (https://spark.apache.org/sql/)
+        # queries as a batch workload.
+        # Corresponds to the JSON property `sparkSqlBatch`
+        # @return [Google::Apis::DataprocV1::SparkSqlBatch]
+        attr_accessor :spark_sql_batch
+      
+        # Output only. The state of the batch.
+        # Corresponds to the JSON property `state`
+        # @return [String]
+        attr_accessor :state
+      
+        # Output only. Historical state information for the batch.
+        # Corresponds to the JSON property `stateHistory`
+        # @return [Array<Google::Apis::DataprocV1::StateHistory>]
+        attr_accessor :state_history
+      
+        # Output only. Batch state details, such as a failure description if the state
+        # is FAILED.
+        # Corresponds to the JSON property `stateMessage`
+        # @return [String]
+        attr_accessor :state_message
+      
+        # Output only. The time when the batch entered a current state.
+        # Corresponds to the JSON property `stateTime`
+        # @return [String]
+        attr_accessor :state_time
+      
+        # Output only. A batch UUID (Unique Universal Identifier). The service generates
+        # this value when it creates the batch.
+        # Corresponds to the JSON property `uuid`
+        # @return [String]
+        attr_accessor :uuid
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @create_time = args[:create_time] if args.key?(:create_time)
+          @creator = args[:creator] if args.key?(:creator)
+          @environment_config = args[:environment_config] if args.key?(:environment_config)
+          @labels = args[:labels] if args.key?(:labels)
+          @name = args[:name] if args.key?(:name)
+          @operation = args[:operation] if args.key?(:operation)
+          @pyspark_batch = args[:pyspark_batch] if args.key?(:pyspark_batch)
+          @runtime_config = args[:runtime_config] if args.key?(:runtime_config)
+          @runtime_info = args[:runtime_info] if args.key?(:runtime_info)
+          @spark_batch = args[:spark_batch] if args.key?(:spark_batch)
+          @spark_r_batch = args[:spark_r_batch] if args.key?(:spark_r_batch)
+          @spark_sql_batch = args[:spark_sql_batch] if args.key?(:spark_sql_batch)
+          @state = args[:state] if args.key?(:state)
+          @state_history = args[:state_history] if args.key?(:state_history)
+          @state_message = args[:state_message] if args.key?(:state_message)
+          @state_time = args[:state_time] if args.key?(:state_time)
+          @uuid = args[:uuid] if args.key?(:uuid)
+        end
+      end
+      
       # Metadata describing the Batch operation.
       class BatchOperationMetadata
         include Google::Apis::Core::Hashable
@@ -481,6 +606,11 @@ module Google
         # @return [String]
         attr_accessor :config_bucket
       
+        # Contains dataproc metric config.
+        # Corresponds to the JSON property `dataprocMetricConfig`
+        # @return [Google::Apis::DataprocV1::DataprocMetricConfig]
+        attr_accessor :dataproc_metric_config
+      
         # Encryption settings for the cluster.
         # Corresponds to the JSON property `encryptionConfig`
         # @return [Google::Apis::DataprocV1::EncryptionConfig]
@@ -573,6 +703,7 @@ module Google
         def update!(**args)
           @autoscaling_config = args[:autoscaling_config] if args.key?(:autoscaling_config)
           @config_bucket = args[:config_bucket] if args.key?(:config_bucket)
+          @dataproc_metric_config = args[:dataproc_metric_config] if args.key?(:dataproc_metric_config)
           @encryption_config = args[:encryption_config] if args.key?(:encryption_config)
           @endpoint_config = args[:endpoint_config] if args.key?(:endpoint_config)
           @gce_cluster_config = args[:gce_cluster_config] if args.key?(:gce_cluster_config)
@@ -834,6 +965,25 @@ module Google
         end
       end
       
+      # Contains dataproc metric config.
+      class DataprocMetricConfig
+        include Google::Apis::Core::Hashable
+      
+        # Required. Metrics to be enabled.
+        # Corresponds to the JSON property `metrics`
+        # @return [Array<Google::Apis::DataprocV1::Metric>]
+        attr_accessor :metrics
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @metrics = args[:metrics] if args.key?(:metrics)
+        end
+      end
+      
       # A request to collect cluster diagnostic information.
       class DiagnoseClusterRequest
         include Google::Apis::Core::Hashable
@@ -884,6 +1034,14 @@ module Google
         # @return [String]
         attr_accessor :boot_disk_type
       
+        # Optional. Interface type of local SSDs (default is "scsi"). Valid values: "
+        # scsi" (Small Computer System Interface), "nvme" (Non-Volatile Memory Express).
+        # See local SSD performance (https://cloud.google.com/compute/docs/disks/local-
+        # ssd#performance).
+        # Corresponds to the JSON property `localSsdInterface`
+        # @return [String]
+        attr_accessor :local_ssd_interface
+      
         # Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not
         # attached, the boot disk is used to store runtime logs and HDFS (https://hadoop.
         # apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are
@@ -901,6 +1059,7 @@ module Google
         def update!(**args)
           @boot_disk_size_gb = args[:boot_disk_size_gb] if args.key?(:boot_disk_size_gb)
           @boot_disk_type = args[:boot_disk_type] if args.key?(:boot_disk_type)
+          @local_ssd_interface = args[:local_ssd_interface] if args.key?(:local_ssd_interface)
           @num_local_ssds = args[:num_local_ssds] if args.key?(:num_local_ssds)
         end
       end
@@ -970,6 +1129,74 @@ module Google
         end
       end
       
+      # Environment configuration for a workload.
+      class EnvironmentConfig
+        include Google::Apis::Core::Hashable
+      
+        # Execution configuration for a workload.
+        # Corresponds to the JSON property `executionConfig`
+        # @return [Google::Apis::DataprocV1::ExecutionConfig]
+        attr_accessor :execution_config
+      
+        # Auxiliary services configuration for a workload.
+        # Corresponds to the JSON property `peripheralsConfig`
+        # @return [Google::Apis::DataprocV1::PeripheralsConfig]
+        attr_accessor :peripherals_config
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @execution_config = args[:execution_config] if args.key?(:execution_config)
+          @peripherals_config = args[:peripherals_config] if args.key?(:peripherals_config)
+        end
+      end
+      
+      # Execution configuration for a workload.
+      class ExecutionConfig
+        include Google::Apis::Core::Hashable
+      
+        # Optional. The Cloud KMS key to use for encryption.
+        # Corresponds to the JSON property `kmsKey`
+        # @return [String]
+        attr_accessor :kms_key
+      
+        # Optional. Tags used for network traffic control.
+        # Corresponds to the JSON property `networkTags`
+        # @return [Array<String>]
+        attr_accessor :network_tags
+      
+        # Optional. Network URI to connect workload to.
+        # Corresponds to the JSON property `networkUri`
+        # @return [String]
+        attr_accessor :network_uri
+      
+        # Optional. Service account that used to execute workload.
+        # Corresponds to the JSON property `serviceAccount`
+        # @return [String]
+        attr_accessor :service_account
+      
+        # Optional. Subnetwork URI to connect workload to.
+        # Corresponds to the JSON property `subnetworkUri`
+        # @return [String]
+        attr_accessor :subnetwork_uri
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @kms_key = args[:kms_key] if args.key?(:kms_key)
+          @network_tags = args[:network_tags] if args.key?(:network_tags)
+          @network_uri = args[:network_uri] if args.key?(:network_uri)
+          @service_account = args[:service_account] if args.key?(:service_account)
+          @subnetwork_uri = args[:subnetwork_uri] if args.key?(:subnetwork_uri)
+        end
+      end
+      
       # Represents a textual expression in the Common Expression Language (CEL) syntax.
       # CEL is a C-like expression language. The syntax and semantics of CEL are
       # documented at https://github.com/google/cel-spec.Example (Comparison): title: "
@@ -1729,7 +1956,7 @@ module Google
         # @return [Google::Apis::DataprocV1::JobScheduling]
         attr_accessor :scheduling
       
-        # A Dataproc job for running Apache Spark (http://spark.apache.org/)
+        # A Dataproc job for running Apache Spark (https://spark.apache.org/)
         # applications on YARN.
         # Corresponds to the JSON property `sparkJob`
         # @return [Google::Apis::DataprocV1::SparkJob]
@@ -1741,7 +1968,7 @@ module Google
         # @return [Google::Apis::DataprocV1::SparkRJob]
         attr_accessor :spark_r_job
       
-        # A Dataproc job for running Apache Spark SQL (http://spark.apache.org/sql/)
+        # A Dataproc job for running Apache Spark SQL (https://spark.apache.org/sql/)
         # queries.
         # Corresponds to the JSON property `sparkSqlJob`
         # @return [Google::Apis::DataprocV1::SparkSqlJob]
@@ -1897,14 +2124,19 @@ module Google
         # Optional. Maximum number of times per hour a driver may be restarted as a
         # result of driver exiting with non-zero code before job is reported failed.A
         # job may be reported as thrashing if driver exits with non-zero code 4 times
-        # within 10 minute window.Maximum value is 10.
+        # within 10 minute window.Maximum value is 10.Note: Currently, this restartable
+        # job option is not supported in Dataproc workflow template (https://cloud.
+        # google.com/dataproc/docs/concepts/workflows/using-workflows#
+        # adding_jobs_to_a_template) jobs.
         # Corresponds to the JSON property `maxFailuresPerHour`
         # @return [Fixnum]
         attr_accessor :max_failures_per_hour
       
         # Optional. Maximum number of times in total a driver may be restarted as a
         # result of driver exiting with non-zero code before job is reported failed.
-        # Maximum value is 240.
+        # Maximum value is 240.Note: Currently, this restartable job option is not
+        # supported in Dataproc workflow template (https://cloud.google.com/dataproc/
+        # docs/concepts/workflows/using-workflows#adding_jobs_to_a_template) jobs.
         # Corresponds to the JSON property `maxFailuresTotal`
         # @return [Fixnum]
         attr_accessor :max_failures_total
@@ -2154,6 +2386,32 @@ module Google
         end
       end
       
+      # A list of batch workloads.
+      class ListBatchesResponse
+        include Google::Apis::Core::Hashable
+      
+        # The batches from the specified collection.
+        # Corresponds to the JSON property `batches`
+        # @return [Array<Google::Apis::DataprocV1::Batch>]
+        attr_accessor :batches
+      
+        # A token, which can be sent as page_token to retrieve the next page. If this
+        # field is omitted, there are no subsequent pages.
+        # Corresponds to the JSON property `nextPageToken`
+        # @return [String]
+        attr_accessor :next_page_token
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @batches = args[:batches] if args.key?(:batches)
+          @next_page_token = args[:next_page_token] if args.key?(:next_page_token)
+        end
+      end
+      
       # The list of all clusters in a project.
       class ListClustersResponse
         include Google::Apis::Core::Hashable
@@ -2365,6 +2623,33 @@ module Google
         end
       end
       
+      # Metric source to enable along with any optional metrics for this source that
+      # override the dataproc defaults
+      class Metric
+        include Google::Apis::Core::Hashable
+      
+        # Optional. Optional Metrics to override the dataproc default metrics configured
+        # for the metric source
+        # Corresponds to the JSON property `metricOverrides`
+        # @return [Array<String>]
+        attr_accessor :metric_overrides
+      
+        # Required. MetricSource that should be enabled
+        # Corresponds to the JSON property `metricSource`
+        # @return [String]
+        attr_accessor :metric_source
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @metric_overrides = args[:metric_overrides] if args.key?(:metric_overrides)
+          @metric_source = args[:metric_source] if args.key?(:metric_source)
+        end
+      end
+      
       # A full, namespace-isolated deployment target for an existing GKE cluster.
       class NamespacedGkeDeploymentTarget
         include Google::Apis::Core::Hashable
@@ -2566,7 +2851,7 @@ module Google
         # @return [Google::Apis::DataprocV1::JobScheduling]
         attr_accessor :scheduling
       
-        # A Dataproc job for running Apache Spark (http://spark.apache.org/)
+        # A Dataproc job for running Apache Spark (https://spark.apache.org/)
         # applications on YARN.
         # Corresponds to the JSON property `sparkJob`
         # @return [Google::Apis::DataprocV1::SparkJob]
@@ -2578,7 +2863,7 @@ module Google
         # @return [Google::Apis::DataprocV1::SparkRJob]
         attr_accessor :spark_r_job
       
-        # A Dataproc job for running Apache Spark SQL (http://spark.apache.org/sql/)
+        # A Dataproc job for running Apache Spark SQL (https://spark.apache.org/sql/)
         # queries.
         # Corresponds to the JSON property `sparkSqlJob`
         # @return [Google::Apis::DataprocV1::SparkSqlJob]
@@ -2640,6 +2925,32 @@ module Google
         end
       end
       
+      # Auxiliary services configuration for a workload.
+      class PeripheralsConfig
+        include Google::Apis::Core::Hashable
+      
+        # Optional. Resource name of an existing Dataproc Metastore service.Example:
+        # projects/[project_id]/locations/[region]/services/[service_id]
+        # Corresponds to the JSON property `metastoreService`
+        # @return [String]
+        attr_accessor :metastore_service
+      
+        # Spark History Server configuration for the workload.
+        # Corresponds to the JSON property `sparkHistoryServerConfig`
+        # @return [Google::Apis::DataprocV1::SparkHistoryServerConfig]
+        attr_accessor :spark_history_server_config
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @metastore_service = args[:metastore_service] if args.key?(:metastore_service)
+          @spark_history_server_config = args[:spark_history_server_config] if args.key?(:spark_history_server_config)
+        end
+      end
+      
       # A Dataproc job for running Apache Pig (https://pig.apache.org/) queries on
       # YARN.
       class PigJob
@@ -2855,6 +3166,63 @@ module Google
         end
       end
       
+      # A configuration for running an Apache PySpark (https://spark.apache.org/docs/
+      # latest/api/python/getting_started/quickstart.html) batch workload.
+      class PySparkBatch
+        include Google::Apis::Core::Hashable
+      
+        # Optional. HCFS URIs of archives to be extracted into the working directory of
+        # each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.
+        # Corresponds to the JSON property `archiveUris`
+        # @return [Array<String>]
+        attr_accessor :archive_uris
+      
+        # Optional. The arguments to pass to the driver. Do not include arguments that
+        # can be set as batch properties, such as --conf, since a collision can occur
+        # that causes an incorrect batch submission.
+        # Corresponds to the JSON property `args`
+        # @return [Array<String>]
+        attr_accessor :args
+      
+        # Optional. HCFS URIs of files to be placed in the working directory of each
+        # executor.
+        # Corresponds to the JSON property `fileUris`
+        # @return [Array<String>]
+        attr_accessor :file_uris
+      
+        # Optional. HCFS URIs of jar files to add to the classpath of the Spark driver
+        # and tasks.
+        # Corresponds to the JSON property `jarFileUris`
+        # @return [Array<String>]
+        attr_accessor :jar_file_uris
+      
+        # Required. The HCFS URI of the main Python file to use as the Spark driver.
+        # Must be a .py file.
+        # Corresponds to the JSON property `mainPythonFileUri`
+        # @return [String]
+        attr_accessor :main_python_file_uri
+      
+        # Optional. HCFS file URIs of Python files to pass to the PySpark framework.
+        # Supported file types: .py, .egg, and .zip.
+        # Corresponds to the JSON property `pythonFileUris`
+        # @return [Array<String>]
+        attr_accessor :python_file_uris
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @archive_uris = args[:archive_uris] if args.key?(:archive_uris)
+          @args = args[:args] if args.key?(:args)
+          @file_uris = args[:file_uris] if args.key?(:file_uris)
+          @jar_file_uris = args[:jar_file_uris] if args.key?(:jar_file_uris)
+          @main_python_file_uri = args[:main_python_file_uri] if args.key?(:main_python_file_uri)
+          @python_file_uris = args[:python_file_uris] if args.key?(:python_file_uris)
+        end
+      end
+      
       # A Dataproc job for running Apache PySpark (https://spark.apache.org/docs/0.9.0/
       # python-programming-guide.html) applications on YARN.
       class PySparkJob
@@ -3034,6 +3402,72 @@ module Google
         end
       end
       
+      # Runtime configuration for a workload.
+      class RuntimeConfig
+        include Google::Apis::Core::Hashable
+      
+        # Optional. Optional custom container image for the job runtime environment. If
+        # not specified, a default container image will be used.
+        # Corresponds to the JSON property `containerImage`
+        # @return [String]
+        attr_accessor :container_image
+      
+        # Optional. A mapping of property names to values, which are used to configure
+        # workload execution.
+        # Corresponds to the JSON property `properties`
+        # @return [Hash<String,String>]
+        attr_accessor :properties
+      
+        # Optional. Version of the batch runtime.
+        # Corresponds to the JSON property `version`
+        # @return [String]
+        attr_accessor :version
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @container_image = args[:container_image] if args.key?(:container_image)
+          @properties = args[:properties] if args.key?(:properties)
+          @version = args[:version] if args.key?(:version)
+        end
+      end
+      
+      # Runtime information about workload execution.
+      class RuntimeInfo
+        include Google::Apis::Core::Hashable
+      
+        # Output only. A URI pointing to the location of the diagnostics tarball.
+        # Corresponds to the JSON property `diagnosticOutputUri`
+        # @return [String]
+        attr_accessor :diagnostic_output_uri
+      
+        # Output only. Map of remote access endpoints (such as web interfaces and APIs)
+        # to their URIs.
+        # Corresponds to the JSON property `endpoints`
+        # @return [Hash<String,String>]
+        attr_accessor :endpoints
+      
+        # Output only. A URI pointing to the location of the stdout and stderr of the
+        # workload.
+        # Corresponds to the JSON property `outputUri`
+        # @return [String]
+        attr_accessor :output_uri
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @diagnostic_output_uri = args[:diagnostic_output_uri] if args.key?(:diagnostic_output_uri)
+          @endpoints = args[:endpoints] if args.key?(:endpoints)
+          @output_uri = args[:output_uri] if args.key?(:output_uri)
+        end
+      end
+      
       # Security related configuration, including encryption, Kerberos, etc.
       class SecurityConfig
         include Google::Apis::Core::Hashable
@@ -3244,7 +3678,84 @@ module Google
         end
       end
       
-      # A Dataproc job for running Apache Spark (http://spark.apache.org/)
+      # A configuration for running an Apache Spark (https://spark.apache.org/) batch
+      # workload.
+      class SparkBatch
+        include Google::Apis::Core::Hashable
+      
+        # Optional. HCFS URIs of archives to be extracted into the working directory of
+        # each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.
+        # Corresponds to the JSON property `archiveUris`
+        # @return [Array<String>]
+        attr_accessor :archive_uris
+      
+        # Optional. The arguments to pass to the driver. Do not include arguments that
+        # can be set as batch properties, such as --conf, since a collision can occur
+        # that causes an incorrect batch submission.
+        # Corresponds to the JSON property `args`
+        # @return [Array<String>]
+        attr_accessor :args
+      
+        # Optional. HCFS URIs of files to be placed in the working directory of each
+        # executor.
+        # Corresponds to the JSON property `fileUris`
+        # @return [Array<String>]
+        attr_accessor :file_uris
+      
+        # Optional. HCFS URIs of jar files to add to the classpath of the Spark driver
+        # and tasks.
+        # Corresponds to the JSON property `jarFileUris`
+        # @return [Array<String>]
+        attr_accessor :jar_file_uris
+      
+        # Optional. The name of the driver main class. The jar file that contains the
+        # class must be in the classpath or specified in jar_file_uris.
+        # Corresponds to the JSON property `mainClass`
+        # @return [String]
+        attr_accessor :main_class
+      
+        # Optional. The HCFS URI of the jar file that contains the main class.
+        # Corresponds to the JSON property `mainJarFileUri`
+        # @return [String]
+        attr_accessor :main_jar_file_uri
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @archive_uris = args[:archive_uris] if args.key?(:archive_uris)
+          @args = args[:args] if args.key?(:args)
+          @file_uris = args[:file_uris] if args.key?(:file_uris)
+          @jar_file_uris = args[:jar_file_uris] if args.key?(:jar_file_uris)
+          @main_class = args[:main_class] if args.key?(:main_class)
+          @main_jar_file_uri = args[:main_jar_file_uri] if args.key?(:main_jar_file_uri)
+        end
+      end
+      
+      # Spark History Server configuration for the workload.
+      class SparkHistoryServerConfig
+        include Google::Apis::Core::Hashable
+      
+        # Optional. Resource name of an existing Dataproc Cluster to act as a Spark
+        # History Server for the workload.Example: projects/[project_id]/regions/[region]
+        # /clusters/[cluster_name]
+        # Corresponds to the JSON property `dataprocCluster`
+        # @return [String]
+        attr_accessor :dataproc_cluster
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @dataproc_cluster = args[:dataproc_cluster] if args.key?(:dataproc_cluster)
+        end
+      end
+      
+      # A Dataproc job for running Apache Spark (https://spark.apache.org/)
       # applications on YARN.
       class SparkJob
         include Google::Apis::Core::Hashable
@@ -3315,6 +3826,49 @@ module Google
         end
       end
       
+      # A configuration for running an Apache SparkR (https://spark.apache.org/docs/
+      # latest/sparkr.html) batch workload.
+      class SparkRBatch
+        include Google::Apis::Core::Hashable
+      
+        # Optional. HCFS URIs of archives to be extracted into the working directory of
+        # each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.
+        # Corresponds to the JSON property `archiveUris`
+        # @return [Array<String>]
+        attr_accessor :archive_uris
+      
+        # Optional. The arguments to pass to the Spark driver. Do not include arguments
+        # that can be set as batch properties, such as --conf, since a collision can
+        # occur that causes an incorrect batch submission.
+        # Corresponds to the JSON property `args`
+        # @return [Array<String>]
+        attr_accessor :args
+      
+        # Optional. HCFS URIs of files to be placed in the working directory of each
+        # executor.
+        # Corresponds to the JSON property `fileUris`
+        # @return [Array<String>]
+        attr_accessor :file_uris
+      
+        # Required. The HCFS URI of the main R file to use as the driver. Must be a .R
+        # or .r file.
+        # Corresponds to the JSON property `mainRFileUri`
+        # @return [String]
+        attr_accessor :main_r_file_uri
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @archive_uris = args[:archive_uris] if args.key?(:archive_uris)
+          @args = args[:args] if args.key?(:args)
+          @file_uris = args[:file_uris] if args.key?(:file_uris)
+          @main_r_file_uri = args[:main_r_file_uri] if args.key?(:main_r_file_uri)
+        end
+      end
+      
       # A Dataproc job for running Apache SparkR (https://spark.apache.org/docs/latest/
       # sparkr.html) applications on YARN.
       class SparkRJob
@@ -3373,7 +3927,41 @@ module Google
         end
       end
       
-      # A Dataproc job for running Apache Spark SQL (http://spark.apache.org/sql/)
+      # A configuration for running Apache Spark SQL (https://spark.apache.org/sql/)
+      # queries as a batch workload.
+      class SparkSqlBatch
+        include Google::Apis::Core::Hashable
+      
+        # Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.
+        # Corresponds to the JSON property `jarFileUris`
+        # @return [Array<String>]
+        attr_accessor :jar_file_uris
+      
+        # Required. The HCFS URI of the script that contains Spark SQL queries to
+        # execute.
+        # Corresponds to the JSON property `queryFileUri`
+        # @return [String]
+        attr_accessor :query_file_uri
+      
+        # Optional. Mapping of query variable names to values (equivalent to the Spark
+        # SQL command: SET name="value";).
+        # Corresponds to the JSON property `queryVariables`
+        # @return [Hash<String,String>]
+        attr_accessor :query_variables
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @jar_file_uris = args[:jar_file_uris] if args.key?(:jar_file_uris)
+          @query_file_uri = args[:query_file_uri] if args.key?(:query_file_uri)
+          @query_variables = args[:query_variables] if args.key?(:query_variables)
+        end
+      end
+      
+      # A Dataproc job for running Apache Spark SQL (https://spark.apache.org/sql/)
       # queries.
       class SparkSqlJob
         include Google::Apis::Core::Hashable
@@ -3522,6 +4110,37 @@ module Google
         end
       end
       
+      # Historical state information.
+      class StateHistory
+        include Google::Apis::Core::Hashable
+      
+        # Output only. The state of the batch at this point in history.
+        # Corresponds to the JSON property `state`
+        # @return [String]
+        attr_accessor :state
+      
+        # Output only. Details about the state at this point in history.
+        # Corresponds to the JSON property `stateMessage`
+        # @return [String]
+        attr_accessor :state_message
+      
+        # Output only. The time when the batch entered the historical state.
+        # Corresponds to the JSON property `stateStartTime`
+        # @return [String]
+        attr_accessor :state_start_time
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @state = args[:state] if args.key?(:state)
+          @state_message = args[:state_message] if args.key?(:state_message)
+          @state_start_time = args[:state_start_time] if args.key?(:state_start_time)
+        end
+      end
+      
       # The Status type defines a logical error model that is suitable for different
       # programming environments, including REST APIs and RPC APIs. It is used by gRPC
       # (https://github.com/grpc). Each Status message contains three pieces of data: