google-cloud-dataproc-v1 0.6.3 → 0.9.0

data/proto_docs/google/cloud/dataproc/v1/clusters.rb

@@ -22,7 +22,7 @@ module Google
     module Dataproc
       module V1
         # Describes the identifying information, config, and status of
-        # a cluster of Compute Engine instances.
+        # a Dataproc cluster
         # @!attribute [rw] project_id
         #   @return [::String]
         #     Required. The Google Cloud Platform project ID that the cluster belongs to.
@@ -32,8 +32,18 @@ module Google
         #     unique. Names of deleted clusters can be reused.
         # @!attribute [rw] config
         #   @return [::Google::Cloud::Dataproc::V1::ClusterConfig]
-        #     Required. The cluster config. Note that Dataproc may set
-        #     default values, and values may change when clusters are updated.
+        #     Optional. The cluster config for a cluster of Compute Engine Instances.
+        #     Note that Dataproc may set default values, and values may change
+        #     when clusters are updated.
+        # @!attribute [rw] virtual_cluster_config
+        #   @return [::Google::Cloud::Dataproc::V1::VirtualClusterConfig]
+        #     Optional. The virtual cluster config, used when creating a Dataproc cluster that
+        #     does not directly control the underlying compute resources, for example,
+        #     when creating a [Dataproc-on-GKE
+        #     cluster](https://cloud.google.com/dataproc/docs/concepts/jobs/dataproc-gke#create-a-dataproc-on-gke-cluster).
+        #     Note that Dataproc may set default values, and values may change when
+        #     clusters are updated. Exactly one of config or virtualClusterConfig must be
+        #     specified.
         # @!attribute [rw] labels
         #   @return [::Google::Protobuf::Map{::String => ::String}]
         #     Optional. The labels to associate with this cluster.
@@ -83,10 +93,10 @@ module Google
         #     ASIA, or EU) for your cluster's staging bucket according to the
         #     Compute Engine zone where your cluster is deployed, and then create
         #     and manage this project-level, per-location bucket (see
-        #     [Dataproc staging
-        #     bucket](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
-        #     **This field requires a Cloud Storage bucket name, not a URI to a Cloud
-        #     Storage bucket.**
+        #     [Dataproc staging and temp
+        #     buckets](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
+        #     **This field requires a Cloud Storage bucket name, not a `gs://...` URI to
+        #     a Cloud Storage bucket.**
         # @!attribute [rw] temp_bucket
         #   @return [::String]
         #     Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data,
@@ -97,9 +107,11 @@ module Google
         #     Compute Engine zone where your cluster is deployed, and then create
         #     and manage this project-level, per-location bucket. The default bucket has
         #     a TTL of 90 days, but you can use any TTL (or none) if you specify a
-        #     bucket.
-        #     **This field requires a Cloud Storage bucket name, not a URI to a Cloud
-        #     Storage bucket.**
+        #     bucket (see
+        #     [Dataproc staging and temp
+        #     buckets](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
+        #     **This field requires a Cloud Storage bucket name, not a `gs://...` URI to
+        #     a Cloud Storage bucket.**
         # @!attribute [rw] gce_cluster_config
         #   @return [::Google::Cloud::Dataproc::V1::GceClusterConfig]
         #     Optional. The shared Compute Engine config settings for
@@ -107,18 +119,18 @@ module Google
         # @!attribute [rw] master_config
         #   @return [::Google::Cloud::Dataproc::V1::InstanceGroupConfig]
         #     Optional. The Compute Engine config settings for
-        #     the master instance in a cluster.
+        #     the cluster's master instance.
         # @!attribute [rw] worker_config
         #   @return [::Google::Cloud::Dataproc::V1::InstanceGroupConfig]
         #     Optional. The Compute Engine config settings for
-        #     worker instances in a cluster.
+        #     the cluster's worker instances.
         # @!attribute [rw] secondary_worker_config
         #   @return [::Google::Cloud::Dataproc::V1::InstanceGroupConfig]
         #     Optional. The Compute Engine config settings for
-        #     additional worker instances in a cluster.
+        #     a cluster's secondary worker instances
         # @!attribute [rw] software_config
         #   @return [::Google::Cloud::Dataproc::V1::SoftwareConfig]
-        #     Optional. The config settings for software inside the cluster.
+        #     Optional. The config settings for cluster software.
         # @!attribute [rw] initialization_actions
         #   @return [::Array<::Google::Cloud::Dataproc::V1::NodeInitializationAction>]
         #     Optional. Commands to execute on each node after config is
@@ -153,37 +165,63 @@ module Google
         # @!attribute [rw] metastore_config
         #   @return [::Google::Cloud::Dataproc::V1::MetastoreConfig]
         #     Optional. Metastore configuration.
-        # @!attribute [rw] gke_cluster_config
-        #   @return [::Google::Cloud::Dataproc::V1::GkeClusterConfig]
-        #     Optional. BETA. The Kubernetes Engine config for Dataproc clusters deployed to
-        #     Kubernetes. Setting this is considered mutually exclusive with Compute
-        #     Engine-based options such as `gce_cluster_config`, `master_config`,
-        #     `worker_config`, `secondary_worker_config`, and `autoscaling_config`.
         class ClusterConfig
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
         end
 
-        # The GKE config for this cluster.
-        # @!attribute [rw] namespaced_gke_deployment_target
-        #   @return [::Google::Cloud::Dataproc::V1::GkeClusterConfig::NamespacedGkeDeploymentTarget]
-        #     Optional. A target for the deployment.
-        class GkeClusterConfig
+        # Dataproc cluster config for a cluster that does not directly control the
+        # underlying compute resources, such as a [Dataproc-on-GKE
+        # cluster](https://cloud.google.com/dataproc/docs/concepts/jobs/dataproc-gke#create-a-dataproc-on-gke-cluster).
+        # @!attribute [rw] staging_bucket
+        #   @return [::String]
+        #     Optional. A Storage bucket used to stage job
+        #     dependencies, config files, and job driver console output.
+        #     If you do not specify a staging bucket, Cloud
+        #     Dataproc will determine a Cloud Storage location (US,
+        #     ASIA, or EU) for your cluster's staging bucket according to the
+        #     Compute Engine zone where your cluster is deployed, and then create
+        #     and manage this project-level, per-location bucket (see
+        #     [Dataproc staging and temp
+        #     buckets](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
+        #     **This field requires a Cloud Storage bucket name, not a `gs://...` URI to
+        #     a Cloud Storage bucket.**
+        # @!attribute [rw] temp_bucket
+        #   @return [::String]
+        #     Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data,
+        #     such as Spark and MapReduce history files.
+        #     If you do not specify a temp bucket,
+        #     Dataproc will determine a Cloud Storage location (US,
+        #     ASIA, or EU) for your cluster's temp bucket according to the
+        #     Compute Engine zone where your cluster is deployed, and then create
+        #     and manage this project-level, per-location bucket. The default bucket has
+        #     a TTL of 90 days, but you can use any TTL (or none) if you specify a
+        #     bucket (see
+        #     [Dataproc staging and temp
+        #     buckets](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
+        #     **This field requires a Cloud Storage bucket name, not a `gs://...` URI to
+        #     a Cloud Storage bucket.**
+        # @!attribute [rw] kubernetes_cluster_config
+        #   @return [::Google::Cloud::Dataproc::V1::KubernetesClusterConfig]
+        #     Required. The configuration for running the Dataproc cluster on Kubernetes.
+        # @!attribute [rw] auxiliary_services_config
+        #   @return [::Google::Cloud::Dataproc::V1::AuxiliaryServicesConfig]
+        #     Optional. Configuration of auxiliary services used by this cluster.
+        class VirtualClusterConfig
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
+        end
 
-          # A full, namespace-isolated deployment target for an existing GKE cluster.
-          # @!attribute [rw] target_gke_cluster
-          #   @return [::String]
-          #     Optional. The target GKE cluster to deploy to.
-          #     Format: 'projects/\\{project}/locations/\\{location}/clusters/\\{cluster_id}'
-          # @!attribute [rw] cluster_namespace
-          #   @return [::String]
-          #     Optional. A namespace within the GKE cluster to deploy into.
-          class NamespacedGkeDeploymentTarget
-            include ::Google::Protobuf::MessageExts
-            extend ::Google::Protobuf::MessageExts::ClassMethods
-          end
+        # Auxiliary services configuration for a Cluster.
+        # @!attribute [rw] metastore_config
+        #   @return [::Google::Cloud::Dataproc::V1::MetastoreConfig]
+        #     Optional. The Hive Metastore configuration for this workload.
+        # @!attribute [rw] spark_history_server_config
+        #   @return [::Google::Cloud::Dataproc::V1::SparkHistoryServerConfig]
+        #     Optional. The Spark History Server configuration for the workload.
+        class AuxiliaryServicesConfig
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
         end
 
         # Endpoint config for this cluster
@@ -334,6 +372,10 @@ module Google
         #   @return [::Google::Cloud::Dataproc::V1::ShieldedInstanceConfig]
         #     Optional. Shielded Instance Config for clusters using [Compute Engine Shielded
         #     VMs](https://cloud.google.com/security/shielded-cloud/shielded-vm).
+        # @!attribute [rw] confidential_instance_config
+        #   @return [::Google::Cloud::Dataproc::V1::ConfidentialInstanceConfig]
+        #     Optional. Confidential Instance Config for clusters using [Confidential
+        #     VMs](https://cloud.google.com/compute/confidential-vm/docs).
         class GceClusterConfig
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -406,6 +448,16 @@ module Google
           extend ::Google::Protobuf::MessageExts::ClassMethods
         end
 
+        # Confidential Instance Config for clusters using [Confidential
+        # VMs](https://cloud.google.com/compute/confidential-vm/docs)
+        # @!attribute [rw] enable_confidential_compute
+        #   @return [::Boolean]
+        #     Optional. Defines whether the instance should have confidential compute enabled.
+        class ConfidentialInstanceConfig
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+        end
+
         # The config settings for Compute Engine resources in
         # an instance group, such as a master or worker group.
         # @!attribute [rw] num_instances
@@ -572,6 +624,13 @@ module Google
         #     If one or more SSDs are attached, this runtime bulk
         #     data is spread across them, and the boot disk contains only basic
         #     config and installed binaries.
+        # @!attribute [rw] local_ssd_interface
+        #   @return [::String]
+        #     Optional. Interface type of local SSDs (default is "scsi").
+        #     Valid values: "scsi" (Small Computer System Interface),
+        #     "nvme" (Non-Volatile Memory Express).
+        #     See [local SSD
+        #     performance](https://cloud.google.com/compute/docs/disks/local-ssd#performance).
         class DiskConfig
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -624,11 +683,19 @@ module Google
             CREATING = 1
 
             # The cluster is currently running and healthy. It is ready for use.
+            #
+            # **Note:** The cluster state changes from "creating" to "running" status
+            # after the master node(s), first two primary worker nodes (and the last
+            # primary worker node if primary workers > 2) are running.
             RUNNING = 2
 
             # The cluster encountered an error. It is not ready for use.
             ERROR = 3
 
+            # The cluster has encountered an error while being updated. Jobs can
+            # be submitted to the cluster, but the cluster cannot be updated.
+            ERROR_DUE_TO_UPDATE = 9
+
             # The cluster is being deleted. It cannot be used.
             DELETING = 4
 
@@ -905,7 +972,7 @@ module Google
         #     Required. The cluster to create.
         # @!attribute [rw] request_id
         #   @return [::String]
-        #     Optional. A unique id used to identify the request. If the server receives two
+        #     Optional. A unique ID used to identify the request. If the server receives two
         #     [CreateClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.CreateClusterRequest)s
         #     with the same id, then the second request will be ignored and the
         #     first {::Google::Longrunning::Operation google.longrunning.Operation} created and stored in the backend
@@ -914,8 +981,11 @@ module Google
         #     It is recommended to always set this value to a
         #     [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
         #
-        #     The id must contain only letters (a-z, A-Z), numbers (0-9),
+        #     The ID must contain only letters (a-z, A-Z), numbers (0-9),
         #     underscores (_), and hyphens (-). The maximum length is 40 characters.
+        # @!attribute [rw] action_on_failed_primary_workers
+        #   @return [::Google::Cloud::Dataproc::V1::FailureAction]
+        #     Optional. Failure action when primary worker creation fails.
         class CreateClusterRequest
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -1001,7 +1071,7 @@ module Google
         #      </table>
         # @!attribute [rw] request_id
         #   @return [::String]
-        #     Optional. A unique id used to identify the request. If the server
+        #     Optional. A unique ID used to identify the request. If the server
         #     receives two
         #     [UpdateClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.UpdateClusterRequest)s
         #     with the same id, then the second request will be ignored and the
@@ -1011,7 +1081,7 @@ module Google
         #     It is recommended to always set this value to a
         #     [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
         #
-        #     The id must contain only letters (a-z, A-Z), numbers (0-9),
+        #     The ID must contain only letters (a-z, A-Z), numbers (0-9),
         #     underscores (_), and hyphens (-). The maximum length is 40 characters.
         class UpdateClusterRequest
           include ::Google::Protobuf::MessageExts
@@ -1035,7 +1105,7 @@ module Google
         #     (with error NOT_FOUND) if a cluster with the specified UUID does not exist.
         # @!attribute [rw] request_id
         #   @return [::String]
-        #     Optional. A unique id used to identify the request. If the server
+        #     Optional. A unique ID used to identify the request. If the server
         #     receives two
         #     [StopClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.StopClusterRequest)s
         #     with the same id, then the second request will be ignored and the
@@ -1045,7 +1115,7 @@ module Google
         #     Recommendation: Set this value to a
         #     [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
         #
-        #     The id must contain only letters (a-z, A-Z), numbers (0-9),
+        #     The ID must contain only letters (a-z, A-Z), numbers (0-9),
         #     underscores (_), and hyphens (-). The maximum length is 40 characters.
         class StopClusterRequest
           include ::Google::Protobuf::MessageExts
@@ -1069,7 +1139,7 @@ module Google
         #     (with error NOT_FOUND) if a cluster with the specified UUID does not exist.
         # @!attribute [rw] request_id
         #   @return [::String]
-        #     Optional. A unique id used to identify the request. If the server
+        #     Optional. A unique ID used to identify the request. If the server
         #     receives two
         #     [StartClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.StartClusterRequest)s
         #     with the same id, then the second request will be ignored and the
@@ -1079,7 +1149,7 @@ module Google
         #     Recommendation: Set this value to a
         #     [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
         #
-        #     The id must contain only letters (a-z, A-Z), numbers (0-9),
+        #     The ID must contain only letters (a-z, A-Z), numbers (0-9),
         #     underscores (_), and hyphens (-). The maximum length is 40 characters.
         class StartClusterRequest
           include ::Google::Protobuf::MessageExts
@@ -1103,7 +1173,7 @@ module Google
         #     (with error NOT_FOUND) if cluster with specified UUID does not exist.
         # @!attribute [rw] request_id
         #   @return [::String]
-        #     Optional. A unique id used to identify the request. If the server
+        #     Optional. A unique ID used to identify the request. If the server
         #     receives two
         #     [DeleteClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.DeleteClusterRequest)s
         #     with the same id, then the second request will be ignored and the
@@ -1113,7 +1183,7 @@ module Google
         #     It is recommended to always set this value to a
         #     [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
         #
-        #     The id must contain only letters (a-z, A-Z), numbers (0-9),
+        #     The ID must contain only letters (a-z, A-Z), numbers (0-9),
         #     underscores (_), and hyphens (-). The maximum length is 40 characters.
         class DeleteClusterRequest
           include ::Google::Protobuf::MessageExts

data/proto_docs/google/cloud/dataproc/v1/jobs.rb

@@ -42,7 +42,7 @@ module Google
           end
 
           # The Log4j level for job execution. When running an
-          # [Apache Hive](http://hive.apache.org/) job, Cloud
+          # [Apache Hive](https://hive.apache.org/) job, Cloud
           # Dataproc configures the Hive client to an equivalent verbosity level.
           module Level
             # Level is unspecified. Use default level for log4j.
@@ -507,8 +507,7 @@ module Google
         #     the job is submitted.
         # @!attribute [rw] cluster_labels
         #   @return [::Google::Protobuf::Map{::String => ::String}]
-        #     Optional. Cluster labels to identify a cluster where the job will be
-        #     submitted.
+        #     Optional. Cluster labels to identify a cluster where the job will be submitted.
         class JobPlacement
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -609,8 +608,8 @@ module Google
         # Encapsulates the full scoping used to reference a job.
         # @!attribute [rw] project_id
         #   @return [::String]
-        #     Optional. The ID of the Google Cloud Platform project that the job belongs
-        #     to. If specified, must match the request project ID.
+        #     Optional. The ID of the Google Cloud Platform project that the job belongs to. If
+        #     specified, must match the request project ID.
         # @!attribute [rw] job_id
         #   @return [::String]
         #     Optional. The job ID, which must be unique within the project.
@@ -757,8 +756,8 @@ module Google
         #     may be reused over time.
         # @!attribute [r] done
         #   @return [::Boolean]
-        #     Output only. Indicates whether the job is completed. If the value is
-        #     `false`, the job is still in progress. If `true`, the job is completed, and
+        #     Output only. Indicates whether the job is completed. If the value is `false`,
+        #     the job is still in progress. If `true`, the job is completed, and
         #     `status.state` field will indicate if it was successful, failed,
         #     or cancelled.
         class Job
@@ -786,11 +785,23 @@ module Google
         #     4 times within 10 minute window.
         #
         #     Maximum value is 10.
+        #
+        #     **Note:** Currently, this restartable job option is
+        #     not supported in Dataproc
+        #     [workflow
+        #     template](https://cloud.google.com/dataproc/docs/concepts/workflows/using-workflows#adding_jobs_to_a_template)
+        #     jobs.
         # @!attribute [rw] max_failures_total
         #   @return [::Integer]
-        #     Optional. Maximum number of times in total a driver may be restarted as a
-        #     result of driver exiting with non-zero code before job is reported failed.
+        #     Optional. Maximum number of times in total a driver may be restarted as a result of
+        #     driver exiting with non-zero code before job is reported failed.
         #     Maximum value is 240.
+        #
+        #     **Note:** Currently, this restartable job option is
+        #     not supported in Dataproc
+        #     [workflow
+        #     template](https://cloud.google.com/dataproc/docs/concepts/workflows/using-workflows#adding_jobs_to_a_template)
+        #     jobs.
         class JobScheduling
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods

data/proto_docs/google/cloud/dataproc/v1/operations.rb

@@ -21,6 +21,54 @@ module Google
   module Cloud
     module Dataproc
       module V1
+        # Metadata describing the Batch operation.
+        # @!attribute [rw] batch
+        #   @return [::String]
+        #     Name of the batch for the operation.
+        # @!attribute [rw] batch_uuid
+        #   @return [::String]
+        #     Batch UUID for the operation.
+        # @!attribute [rw] create_time
+        #   @return [::Google::Protobuf::Timestamp]
+        #     The time when the operation was created.
+        # @!attribute [rw] done_time
+        #   @return [::Google::Protobuf::Timestamp]
+        #     The time when the operation finished.
+        # @!attribute [rw] operation_type
+        #   @return [::Google::Cloud::Dataproc::V1::BatchOperationMetadata::BatchOperationType]
+        #     The operation type.
+        # @!attribute [rw] description
+        #   @return [::String]
+        #     Short description of the operation.
+        # @!attribute [rw] labels
+        #   @return [::Google::Protobuf::Map{::String => ::String}]
+        #     Labels associated with the operation.
+        # @!attribute [rw] warnings
+        #   @return [::Array<::String>]
+        #     Warnings encountered during operation execution.
+        class BatchOperationMetadata
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+
+          # @!attribute [rw] key
+          #   @return [::String]
+          # @!attribute [rw] value
+          #   @return [::String]
+          class LabelsEntry
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+
+          # Operation type for Batch resources
+          module BatchOperationType
+            # Batch operation type is unknown.
+            BATCH_OPERATION_TYPE_UNSPECIFIED = 0
+
+            # Batch operation type.
+            BATCH = 1
+          end
+        end
+
         # The status of the operation.
         # @!attribute [r] state
         #   @return [::Google::Cloud::Dataproc::V1::ClusterOperationStatus::State]