google-cloud-dataproc-v1 0.14.0 → 0.16.0

data/proto_docs/google/cloud/dataproc/v1/batches.rb

@@ -30,8 +30,8 @@ module Google
         #     Required. The batch to create.
         # @!attribute [rw] batch_id
         #   @return [::String]
-        #     Optional. The ID to use for the batch, which will become the final component of
-        #     the batch's resource name.
+        #     Optional. The ID to use for the batch, which will become the final
+        #     component of the batch's resource name.
         #
         #     This value must be 4-63 characters. Valid characters are `/[a-z][0-9]-/`.
         # @!attribute [rw] request_id
@@ -56,7 +56,9 @@ module Google
         # A request to get the resource representation for a batch workload.
         # @!attribute [rw] name
         #   @return [::String]
-        #     Required. The name of the batch to retrieve.
+        #     Required. The fully qualified name of the batch to retrieve
+        #     in the format
+        #     "projects/PROJECT_ID/locations/DATAPROC_REGION/batches/BATCH_ID"
         class GetBatchRequest
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -75,6 +77,28 @@ module Google
         #   @return [::String]
         #     Optional. A page token received from a previous `ListBatches` call.
         #     Provide this token to retrieve the subsequent page.
+        # @!attribute [rw] filter
+        #   @return [::String]
+        #     Optional. A filter for the batches to return in the response.
+        #
+        #     A filter is a logical expression constraining the values of various fields
+        #     in each batch resource. Filters are case sensitive, and may contain
+        #     multiple clauses combined with logical operators (AND/OR).
+        #     Supported fields are `batch_id`, `batch_uuid`, `state`, and `create_time`.
+        #
+        #     e.g. `state = RUNNING and create_time < "2023-01-01T00:00:00Z"`
+        #     filters for batches in state RUNNING that were created before 2023-01-01
+        #
+        #     See https://google.aip.dev/assets/misc/ebnf-filtering.txt for a detailed
+        #     description of the filter syntax and a list of supported comparisons.
+        # @!attribute [rw] order_by
+        #   @return [::String]
+        #     Optional. Field(s) on which to sort the list of batches.
+        #
+        #     Currently the only supported sort orders are unspecified (empty) and
+        #     `create_time desc` to sort by most recently created batches first.
+        #
+        #     See https://google.aip.dev/132#ordering for more details.
         class ListBatchesRequest
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -96,7 +120,9 @@ module Google
         # A request to delete a batch workload.
         # @!attribute [rw] name
         #   @return [::String]
-        #     Required. The name of the batch resource to delete.
+        #     Required. The fully qualified name of the batch to retrieve
+        #     in the format
+        #     "projects/PROJECT_ID/locations/DATAPROC_REGION/batches/BATCH_ID"
         class DeleteBatchRequest
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -221,8 +247,8 @@ module Google
         # batch workload.
         # @!attribute [rw] main_python_file_uri
         #   @return [::String]
-        #     Required. The HCFS URI of the main Python file to use as the Spark driver. Must
-        #     be a .py file.
+        #     Required. The HCFS URI of the main Python file to use as the Spark driver.
+        #     Must be a .py file.
         # @!attribute [rw] args
         #   @return [::Array<::String>]
         #     Optional. The arguments to pass to the driver. Do not include arguments
@@ -250,15 +276,15 @@ module Google
           extend ::Google::Protobuf::MessageExts::ClassMethods
         end
 
-        # A configuration for running an [Apache Spark](http://spark.apache.org/)
+        # A configuration for running an [Apache Spark](https://spark.apache.org/)
         # batch workload.
         # @!attribute [rw] main_jar_file_uri
         #   @return [::String]
         #     Optional. The HCFS URI of the jar file that contains the main class.
         # @!attribute [rw] main_class
         #   @return [::String]
-        #     Optional. The name of the driver main class. The jar file that contains the class
-        #     must be in the classpath or specified in `jar_file_uris`.
+        #     Optional. The name of the driver main class. The jar file that contains
+        #     the class must be in the classpath or specified in `jar_file_uris`.
         # @!attribute [rw] args
         #   @return [::Array<::String>]
         #     Optional. The arguments to pass to the driver. Do not include arguments
@@ -291,9 +317,9 @@ module Google
         #     Must be a `.R` or `.r` file.
         # @!attribute [rw] args
         #   @return [::Array<::String>]
-        #     Optional. The arguments to pass to the Spark driver. Do not include arguments
-        #     that can be set as batch properties, such as `--conf`, since a collision
-        #     can occur that causes an incorrect batch submission.
+        #     Optional. The arguments to pass to the Spark driver. Do not include
+        #     arguments that can be set as batch properties, such as `--conf`, since a
+        #     collision can occur that causes an incorrect batch submission.
         # @!attribute [rw] file_uris
         #   @return [::Array<::String>]
         #     Optional. HCFS URIs of files to be placed in the working directory of
@@ -309,10 +335,12 @@ module Google
         end
 
         # A configuration for running
-        # [Apache Spark SQL](http://spark.apache.org/sql/) queries as a batch workload.
+        # [Apache Spark SQL](https://spark.apache.org/sql/) queries as a batch
+        # workload.
         # @!attribute [rw] query_file_uri
         #   @return [::String]
-        #     Required. The HCFS URI of the script that contains Spark SQL queries to execute.
+        #     Required. The HCFS URI of the script that contains Spark SQL queries to
+        #     execute.
         # @!attribute [rw] query_variables
         #   @return [::Google::Protobuf::Map{::String => ::String}]
         #     Optional. Mapping of query variable names to values (equivalent to the

data/proto_docs/google/cloud/dataproc/v1/clusters.rb

@@ -37,12 +37,14 @@ module Google
         #     Optional. The cluster config for a cluster of Compute Engine Instances.
         #     Note that Dataproc may set default values, and values may change
         #     when clusters are updated.
+        #
+        #     Exactly one of ClusterConfig or VirtualClusterConfig must be specified.
         # @!attribute [rw] virtual_cluster_config
         #   @return [::Google::Cloud::Dataproc::V1::VirtualClusterConfig]
         #     Optional. The virtual cluster config is used when creating a Dataproc
         #     cluster that does not directly control the underlying compute resources,
         #     for example, when creating a [Dataproc-on-GKE
-        #     cluster](https://cloud.google.com/dataproc/docs/guides/dpgke/dataproc-gke).
+        #     cluster](https://cloud.google.com/dataproc/docs/guides/dpgke/dataproc-gke-overview).
         #     Dataproc may set default values, and values may change when
         #     clusters are updated. Exactly one of
         #     {::Google::Cloud::Dataproc::V1::Cluster#config config} or
@@ -180,7 +182,7 @@ module Google
 
         # The Dataproc cluster config for a cluster that does not directly control the
         # underlying compute resources, such as a [Dataproc-on-GKE
-        # cluster](https://cloud.google.com/dataproc/docs/guides/dpgke/dataproc-gke).
+        # cluster](https://cloud.google.com/dataproc/docs/guides/dpgke/dataproc-gke-overview).
         # @!attribute [rw] staging_bucket
         #   @return [::String]
         #     Optional. A Cloud Storage bucket used to stage job
@@ -272,17 +274,15 @@ module Google
         # instances, applicable to all instances in the cluster.
         # @!attribute [rw] zone_uri
         #   @return [::String]
-        #     Optional. The zone where the Compute Engine cluster will be located.
-        #     On a create request, it is required in the "global" region. If omitted
-        #     in a non-global Dataproc region, the service will pick a zone in the
-        #     corresponding Compute Engine region. On a get request, zone will
-        #     always be present.
+        #     Optional. The Compute Engine zone where the Dataproc cluster will be
+        #     located. If omitted, the service will pick a zone in the cluster's Compute
+        #     Engine region. On a get request, zone will always be present.
         #
         #     A full URL, partial URI, or short name are valid. Examples:
         #
         #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/[zone]`
         #     * `projects/[project_id]/zones/[zone]`
-        #     * `us-central1-f`
+        #     * `[zone]`
         # @!attribute [rw] network_uri
         #   @return [::String]
         #     Optional. The Compute Engine network to be used for machine
@@ -294,8 +294,8 @@ module Google
         #
         #     A full URL, partial URI, or short name are valid. Examples:
         #
-        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/global/default`
-        #     * `projects/[project_id]/regions/global/default`
+        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/global/networks/default`
+        #     * `projects/[project_id]/global/networks/default`
         #     * `default`
         # @!attribute [rw] subnetwork_uri
         #   @return [::String]
@@ -304,8 +304,8 @@ module Google
         #
         #     A full URL, partial URI, or short name are valid. Examples:
         #
-        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/us-east1/subnetworks/sub0`
-        #     * `projects/[project_id]/regions/us-east1/subnetworks/sub0`
+        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/[region]/subnetworks/sub0`
+        #     * `projects/[project_id]/regions/[region]/subnetworks/sub0`
         #     * `sub0`
         # @!attribute [rw] internal_ip_only
         #   @return [::Boolean]
@@ -422,8 +422,8 @@ module Google
         #
         #     A full URL, partial URI, or node group name are valid. Examples:
         #
-        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/us-central1-a/nodeGroups/node-group-1`
-        #     * `projects/[project_id]/zones/us-central1-a/nodeGroups/node-group-1`
+        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/[zone]/nodeGroups/node-group-1`
+        #     * `projects/[project_id]/zones/[zone]/nodeGroups/node-group-1`
         #     * `node-group-1`
         class NodeGroupAffinity
           include ::Google::Protobuf::MessageExts
@@ -479,14 +479,14 @@ module Google
         #
         #     Image examples:
         #
-        #     * `https://www.googleapis.com/compute/beta/projects/[project_id]/global/images/[image-id]`
+        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/global/images/[image-id]`
         #     * `projects/[project_id]/global/images/[image-id]`
         #     * `image-id`
         #
         #     Image family examples. Dataproc will use the most recent
         #     image from the family:
         #
-        #     * `https://www.googleapis.com/compute/beta/projects/[project_id]/global/images/family/[custom-image-family-name]`
+        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/global/images/family/[custom-image-family-name]`
         #     * `projects/[project_id]/global/images/family/[custom-image-family-name]`
         #
         #     If the URI is unspecified, it will be inferred from
@@ -497,8 +497,8 @@ module Google
         #
         #     A full URL, partial URI, or short name are valid. Examples:
         #
-        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2`
-        #     * `projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2`
+        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/[zone]/machineTypes/n1-standard-2`
+        #     * `projects/[project_id]/zones/[zone]/machineTypes/n1-standard-2`
         #     * `n1-standard-2`
         #
         #     **Auto Zone Exception**: If you are using the Dataproc
@@ -559,6 +559,16 @@ module Google
             # (https://cloud.google.com/dataproc/docs/concepts/compute/secondary-vms)
             # groups.
             PREEMPTIBLE = 2
+
+            # Instances are [Spot VMs]
+            # (https://cloud.google.com/compute/docs/instances/spot).
+            #
+            # This option is allowed only for [secondary worker]
+            # (https://cloud.google.com/dataproc/docs/concepts/compute/secondary-vms)
+            # groups. Spot VMs are the latest version of [preemptible VMs]
+            # (https://cloud.google.com/compute/docs/instances/preemptible), and
+            # provide additional features.
+            SPOT = 3
           end
         end
 
@@ -583,12 +593,12 @@ module Google
         #     Full URL, partial URI, or short name of the accelerator type resource to
         #     expose to this instance. See
         #     [Compute Engine
-        #     AcceleratorTypes](https://cloud.google.com/compute/docs/reference/beta/acceleratorTypes).
+        #     AcceleratorTypes](https://cloud.google.com/compute/docs/reference/v1/acceleratorTypes).
         #
         #     Examples:
         #
-        #     * `https://www.googleapis.com/compute/beta/projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80`
-        #     * `projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80`
+        #     * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/[zone]/acceleratorTypes/nvidia-tesla-k80`
+        #     * `projects/[project_id]/zones/[zone]/acceleratorTypes/nvidia-tesla-k80`
         #     * `nvidia-tesla-k80`
         #
         #     **Auto Zone Exception**: If you are using the Dataproc
@@ -623,6 +633,9 @@ module Google
         #     If one or more SSDs are attached, this runtime bulk
         #     data is spread across them, and the boot disk contains only basic
         #     config and installed binaries.
+        #
+        #     Note: Local SSD options may vary by machine type and number of vCPUs
+        #     selected.
         # @!attribute [rw] local_ssd_interface
         #   @return [::String]
         #     Optional. Interface type of local SSDs (default is "scsi").
@@ -988,6 +1001,39 @@ module Google
           extend ::Google::Protobuf::MessageExts::ClassMethods
         end
 
+        # Contains cluster daemon metrics, such as HDFS and YARN stats.
+        #
+        # **Beta Feature**: This report is available for testing purposes only. It may
+        # be changed before final release.
+        # @!attribute [rw] hdfs_metrics
+        #   @return [::Google::Protobuf::Map{::String => ::Integer}]
+        #     The HDFS metrics.
+        # @!attribute [rw] yarn_metrics
+        #   @return [::Google::Protobuf::Map{::String => ::Integer}]
+        #     YARN metrics.
+        class ClusterMetrics
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+
+          # @!attribute [rw] key
+          #   @return [::String]
+          # @!attribute [rw] value
+          #   @return [::Integer]
+          class HdfsMetricsEntry
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+
+          # @!attribute [rw] key
+          #   @return [::String]
+          # @!attribute [rw] value
+          #   @return [::Integer]
+          class YarnMetricsEntry
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+        end
+
         # Dataproc metric config.
         # @!attribute [rw] metrics
         #   @return [::Array<::Google::Cloud::Dataproc::V1::DataprocMetricConfig::Metric>]
@@ -1066,39 +1112,9 @@ module Google
 
             # Hiveserver2 metric source.
             HIVESERVER2 = 6
-          end
-        end
-
-        # Contains cluster daemon metrics, such as HDFS and YARN stats.
-        #
-        # **Beta Feature**: This report is available for testing purposes only. It may
-        # be changed before final release.
-        # @!attribute [rw] hdfs_metrics
-        #   @return [::Google::Protobuf::Map{::String => ::Integer}]
-        #     The HDFS metrics.
-        # @!attribute [rw] yarn_metrics
-        #   @return [::Google::Protobuf::Map{::String => ::Integer}]
-        #     The YARN metrics.
-        class ClusterMetrics
-          include ::Google::Protobuf::MessageExts
-          extend ::Google::Protobuf::MessageExts::ClassMethods
 
-          # @!attribute [rw] key
-          #   @return [::String]
-          # @!attribute [rw] value
-          #   @return [::Integer]
-          class HdfsMetricsEntry
-            include ::Google::Protobuf::MessageExts
-            extend ::Google::Protobuf::MessageExts::ClassMethods
-          end
-
-          # @!attribute [rw] key
-          #   @return [::String]
-          # @!attribute [rw] value
-          #   @return [::Integer]
-          class YarnMetricsEntry
-            include ::Google::Protobuf::MessageExts
-            extend ::Google::Protobuf::MessageExts::ClassMethods
+            # hivemetastore metric source
+            HIVEMETASTORE = 7
           end
         end
 
@@ -1151,7 +1167,7 @@ module Google
         #     Required. The changes to the cluster.
         # @!attribute [rw] graceful_decommission_timeout
         #   @return [::Google::Protobuf::Duration]
-        #     Optional. Timeout for graceful YARN decomissioning. Graceful
+        #     Optional. Timeout for graceful YARN decommissioning. Graceful
         #     decommissioning allows removing nodes from the cluster without
         #     interrupting jobs in progress. Timeout specifies how long to wait for jobs
         #     in progress to finish before forcefully removing nodes (and potentially

data/proto_docs/google/cloud/dataproc/v1/jobs.rb

@@ -497,6 +497,51 @@ module Google
           end
         end
 
+        # A Dataproc job for running [Trino](https://trino.io/) queries.
+        # **IMPORTANT**: The [Dataproc Trino Optional
+        # Component](https://cloud.google.com/dataproc/docs/concepts/components/trino)
+        # must be enabled when the cluster is created to submit a Trino job to the
+        # cluster.
+        # @!attribute [rw] query_file_uri
+        #   @return [::String]
+        #     The HCFS URI of the script that contains SQL queries.
+        # @!attribute [rw] query_list
+        #   @return [::Google::Cloud::Dataproc::V1::QueryList]
+        #     A list of queries.
+        # @!attribute [rw] continue_on_failure
+        #   @return [::Boolean]
+        #     Optional. Whether to continue executing queries if a query fails.
+        #     The default value is `false`. Setting to `true` can be useful when
+        #     executing independent parallel queries.
+        # @!attribute [rw] output_format
+        #   @return [::String]
+        #     Optional. The format in which query output will be displayed. See the
+        #     Trino documentation for supported output formats
+        # @!attribute [rw] client_tags
+        #   @return [::Array<::String>]
+        #     Optional. Trino client tags to attach to this query
+        # @!attribute [rw] properties
+        #   @return [::Google::Protobuf::Map{::String => ::String}]
+        #     Optional. A mapping of property names to values. Used to set Trino
+        #     [session properties](https://trino.io/docs/current/sql/set-session.html)
+        #     Equivalent to using the --session flag in the Trino CLI
+        # @!attribute [rw] logging_config
+        #   @return [::Google::Cloud::Dataproc::V1::LoggingConfig]
+        #     Optional. The runtime log config for job execution.
+        class TrinoJob
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+
+          # @!attribute [rw] key
+          #   @return [::String]
+          # @!attribute [rw] value
+          #   @return [::String]
+          class PropertiesEntry
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+        end
+
         # Dataproc job config.
         # @!attribute [rw] cluster_name
         #   @return [::String]
@@ -715,6 +760,9 @@ module Google
         # @!attribute [rw] presto_job
         #   @return [::Google::Cloud::Dataproc::V1::PrestoJob]
         #     Optional. Job is a Presto job.
+        # @!attribute [rw] trino_job
+        #   @return [::Google::Cloud::Dataproc::V1::TrinoJob]
+        #     Optional. Job is a Trino job.
         # @!attribute [r] status
         #   @return [::Google::Cloud::Dataproc::V1::JobStatus]
         #     Output only. The job status. Additional application-specific

data/proto_docs/google/cloud/dataproc/v1/node_groups.rb

@@ -82,7 +82,7 @@ module Google
         #     underscores (_), and hyphens (-). The maximum length is 40 characters.
         # @!attribute [rw] graceful_decommission_timeout
         #   @return [::Google::Protobuf::Duration]
-        #     Optional. Timeout for graceful YARN decommissioning. [Graceful
+        #     Optional. Timeout for graceful YARN decomissioning. [Graceful
         #     decommissioning]
         #     (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/scaling-clusters#graceful_decommissioning)
         #     allows the removal of nodes from the Compute Engine node group

data/proto_docs/google/cloud/dataproc/v1/operations.rb

@@ -127,6 +127,9 @@ module Google
         # @!attribute [r] warnings
         #   @return [::Array<::String>]
         #     Output only. Errors encountered during operation execution.
+        # @!attribute [r] child_operation_ids
+        #   @return [::Array<::String>]
+        #     Output only. Child operation ids
         class ClusterOperationMetadata
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods