google-api-client 0.40.0 → 0.41.2

data/generated/google/apis/dataproc_v1.rb

@@ -25,7 +25,7 @@ module Google
     # @see https://cloud.google.com/dataproc/
     module DataprocV1
       VERSION = 'V1'
-      REVISION = '20200501'
+      REVISION = '20200619'
 
       # View and manage your data across Google Cloud Platform services
       AUTH_CLOUD_PLATFORM = 'https://www.googleapis.com/auth/cloud-platform'

data/generated/google/apis/dataproc_v1/classes.rb

@@ -175,11 +175,12 @@ module Google
         # @return [String]
         attr_accessor :graceful_decommission_timeout
       
-        # Required. Fraction of average pending memory in the last cooldown period for
-        # which to remove workers. A scale-down factor of 1 will result in scaling down
-        # so that there is no available memory remaining after the update (more
+        # Required. Fraction of average YARN pending memory in the last cooldown period
+        # for which to remove workers. A scale-down factor of 1 will result in scaling
+        # down so that there is no available memory remaining after the update (more
         # aggressive scaling). A scale-down factor of 0 disables removing workers, which
-        # can be beneficial for autoscaling a single job.Bounds: 0.0, 1.0.
+        # can be beneficial for autoscaling a single job. See How autoscaling works for
+        # more information.Bounds: 0.0, 1.0.
         # Corresponds to the JSON property `scaleDownFactor`
         # @return [Float]
         attr_accessor :scale_down_factor
@@ -193,11 +194,12 @@ module Google
         # @return [Float]
         attr_accessor :scale_down_min_worker_fraction
       
-        # Required. Fraction of average pending memory in the last cooldown period for
-        # which to add workers. A scale-up factor of 1.0 will result in scaling up so
-        # that there is no pending memory remaining after the update (more aggressive
+        # Required. Fraction of average YARN pending memory in the last cooldown period
+        # for which to add workers. A scale-up factor of 1.0 will result in scaling up
+        # so that there is no pending memory remaining after the update (more aggressive
         # scaling). A scale-up factor closer to 0 will result in a smaller magnitude of
-        # scaling up (less aggressive scaling).Bounds: 0.0, 1.0.
+        # scaling up (less aggressive scaling). See How autoscaling works for more
+        # information.Bounds: 0.0, 1.0.
         # Corresponds to the JSON property `scaleUpFactor`
         # @return [Float]
         attr_accessor :scale_up_factor
@@ -414,6 +416,11 @@ module Google
         # @return [Google::Apis::DataprocV1::EncryptionConfig]
         attr_accessor :encryption_config
       
+        # Endpoint config for this cluster
+        # Corresponds to the JSON property `endpointConfig`
+        # @return [Google::Apis::DataprocV1::EndpointConfig]
+        attr_accessor :endpoint_config
+      
         # Common config settings for resources of Compute Engine cluster instances,
         # applicable to all instances in the cluster.
         # Corresponds to the JSON property `gceClusterConfig`
@@ -462,6 +469,17 @@ module Google
         # @return [Google::Apis::DataprocV1::SoftwareConfig]
         attr_accessor :software_config
       
+        # Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data,
+        # such as Spark and MapReduce history files. If you do not specify a temp
+        # bucket, Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for
+        # your cluster's temp bucket according to the Compute Engine zone where your
+        # cluster is deployed, and then create and manage this project-level, per-
+        # location bucket. The default bucket has a TTL of 90 days, but you can use any
+        # TTL (or none) if you specify a bucket.
+        # Corresponds to the JSON property `tempBucket`
+        # @return [String]
+        attr_accessor :temp_bucket
+      
         # The config settings for Compute Engine resources in an instance group, such as
         # a master or worker group.
         # Corresponds to the JSON property `workerConfig`
@@ -477,6 +495,7 @@ module Google
           @autoscaling_config = args[:autoscaling_config] if args.key?(:autoscaling_config)
           @config_bucket = args[:config_bucket] if args.key?(:config_bucket)
           @encryption_config = args[:encryption_config] if args.key?(:encryption_config)
+          @endpoint_config = args[:endpoint_config] if args.key?(:endpoint_config)
           @gce_cluster_config = args[:gce_cluster_config] if args.key?(:gce_cluster_config)
           @initialization_actions = args[:initialization_actions] if args.key?(:initialization_actions)
           @lifecycle_config = args[:lifecycle_config] if args.key?(:lifecycle_config)
@@ -484,6 +503,7 @@ module Google
           @secondary_worker_config = args[:secondary_worker_config] if args.key?(:secondary_worker_config)
           @security_config = args[:security_config] if args.key?(:security_config)
           @software_config = args[:software_config] if args.key?(:software_config)
+          @temp_bucket = args[:temp_bucket] if args.key?(:temp_bucket)
           @worker_config = args[:worker_config] if args.key?(:worker_config)
         end
       end
@@ -820,6 +840,34 @@ module Google
         end
       end
       
+      # Endpoint config for this cluster
+      class EndpointConfig
+        include Google::Apis::Core::Hashable
+      
+        # Optional. If true, enable http access to specific ports on the cluster from
+        # external sources. Defaults to false.
+        # Corresponds to the JSON property `enableHttpPortAccess`
+        # @return [Boolean]
+        attr_accessor :enable_http_port_access
+        alias_method :enable_http_port_access?, :enable_http_port_access
+      
+        # Output only. The map of port descriptions to URLs. Will only be populated if
+        # enable_http_port_access is true.
+        # Corresponds to the JSON property `httpPorts`
+        # @return [Hash<String,String>]
+        attr_accessor :http_ports
+      
+        def initialize(**args)
+           update!(**args)
+        end
+      
+        # Update properties of this object
+        def update!(**args)
+          @enable_http_port_access = args[:enable_http_port_access] if args.key?(:enable_http_port_access)
+          @http_ports = args[:http_ports] if args.key?(:http_ports)
+        end
+      end
+      
       # Represents a textual expression in the Common Expression Language (CEL) syntax.
       # CEL is a C-like expression language. The syntax and semantics of CEL are
       # documented at https://github.com/google/cel-spec.Example (Comparison):
@@ -1593,7 +1641,8 @@ module Google
         # @return [String]
         attr_accessor :job_id
       
-        # Required. The ID of the Google Cloud Platform project that the job belongs to.
+        # Optional. The ID of the Google Cloud Platform project that the job belongs to.
+        # If specified, must match the request project ID.
         # Corresponds to the JSON property `projectId`
         # @return [String]
         attr_accessor :project_id
@@ -2528,8 +2577,8 @@ module Google
       class PySparkJob
         include Google::Apis::Core::Hashable
       
-        # Optional. HCFS URIs of archives to be extracted in the working directory of .
-        # jar, .tar, .tar.gz, .tgz, and .zip.
+        # Optional. HCFS URIs of archives to be extracted into the working directory of
+        # each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.
         # Corresponds to the JSON property `archiveUris`
         # @return [Array<String>]
         attr_accessor :archive_uris
@@ -2541,8 +2590,8 @@ module Google
         # @return [Array<String>]
         attr_accessor :args
       
-        # Optional. HCFS URIs of files to be copied to the working directory of Python
-        # drivers and distributed tasks. Useful for naively parallel tasks.
+        # Optional. HCFS URIs of files to be placed in the working directory of each
+        # executor. Useful for naively parallel tasks.
         # Corresponds to the JSON property `fileUris`
         # @return [Array<String>]
         attr_accessor :file_uris
@@ -2825,9 +2874,8 @@ module Google
       class SparkJob
         include Google::Apis::Core::Hashable
       
-        # Optional. HCFS URIs of archives to be extracted in the working directory of
-        # Spark drivers and tasks. Supported file types: .jar, .tar, .tar.gz, .tgz, and .
-        # zip.
+        # Optional. HCFS URIs of archives to be extracted into the working directory of
+        # each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.
         # Corresponds to the JSON property `archiveUris`
         # @return [Array<String>]
         attr_accessor :archive_uris
@@ -2839,8 +2887,8 @@ module Google
         # @return [Array<String>]
         attr_accessor :args
       
-        # Optional. HCFS URIs of files to be copied to the working directory of Spark
-        # drivers and distributed tasks. Useful for naively parallel tasks.
+        # Optional. HCFS URIs of files to be placed in the working directory of each
+        # executor. Useful for naively parallel tasks.
         # Corresponds to the JSON property `fileUris`
         # @return [Array<String>]
         attr_accessor :file_uris
@@ -2897,9 +2945,8 @@ module Google
       class SparkRJob
         include Google::Apis::Core::Hashable
       
-        # Optional. HCFS URIs of archives to be extracted in the working directory of
-        # Spark drivers and tasks. Supported file types: .jar, .tar, .tar.gz, .tgz, and .
-        # zip.
+        # Optional. HCFS URIs of archives to be extracted into the working directory of
+        # each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.
         # Corresponds to the JSON property `archiveUris`
         # @return [Array<String>]
         attr_accessor :archive_uris
@@ -2911,8 +2958,8 @@ module Google
         # @return [Array<String>]
         attr_accessor :args
       
-        # Optional. HCFS URIs of files to be copied to the working directory of R
-        # drivers and distributed tasks. Useful for naively parallel tasks.
+        # Optional. HCFS URIs of files to be placed in the working directory of each
+        # executor. Useful for naively parallel tasks.
         # Corresponds to the JSON property `fileUris`
         # @return [Array<String>]
         attr_accessor :file_uris
@@ -3398,7 +3445,7 @@ module Google
         # @return [String]
         attr_accessor :name
       
-        # Optional. emplate parameters whose values are substituted into the template.
+        # Optional. Template parameters whose values are substituted into the template.
         # Values for parameters must be provided when the template is instantiated.
         # Corresponds to the JSON property `parameters`
         # @return [Array<Google::Apis::DataprocV1::TemplateParameter>]

data/generated/google/apis/dataproc_v1/representations.rb

@@ -142,6 +142,12 @@ module Google
         include Google::Apis::Core::JsonObjectSupport
       end
       
+      class EndpointConfig
+        class Representation < Google::Apis::Core::JsonRepresentation; end
+      
+        include Google::Apis::Core::JsonObjectSupport
+      end
+      
       class Expr
         class Representation < Google::Apis::Core::JsonRepresentation; end
       
@@ -557,6 +563,8 @@ module Google
           property :config_bucket, as: 'configBucket'
           property :encryption_config, as: 'encryptionConfig', class: Google::Apis::DataprocV1::EncryptionConfig, decorator: Google::Apis::DataprocV1::EncryptionConfig::Representation
       
+          property :endpoint_config, as: 'endpointConfig', class: Google::Apis::DataprocV1::EndpointConfig, decorator: Google::Apis::DataprocV1::EndpointConfig::Representation
+      
           property :gce_cluster_config, as: 'gceClusterConfig', class: Google::Apis::DataprocV1::GceClusterConfig, decorator: Google::Apis::DataprocV1::GceClusterConfig::Representation
       
           collection :initialization_actions, as: 'initializationActions', class: Google::Apis::DataprocV1::NodeInitializationAction, decorator: Google::Apis::DataprocV1::NodeInitializationAction::Representation
@@ -571,6 +579,7 @@ module Google
       
           property :software_config, as: 'softwareConfig', class: Google::Apis::DataprocV1::SoftwareConfig, decorator: Google::Apis::DataprocV1::SoftwareConfig::Representation
       
+          property :temp_bucket, as: 'tempBucket'
           property :worker_config, as: 'workerConfig', class: Google::Apis::DataprocV1::InstanceGroupConfig, decorator: Google::Apis::DataprocV1::InstanceGroupConfig::Representation
       
         end
@@ -672,6 +681,14 @@ module Google
         end
       end
       
+      class EndpointConfig
+        # @private
+        class Representation < Google::Apis::Core::JsonRepresentation
+          property :enable_http_port_access, as: 'enableHttpPortAccess'
+          hash :http_ports, as: 'httpPorts'
+        end
+      end
+      
       class Expr
         # @private
         class Representation < Google::Apis::Core::JsonRepresentation

data/generated/google/apis/dataproc_v1beta2.rb

@@ -25,7 +25,7 @@ module Google
     # @see https://cloud.google.com/dataproc/
     module DataprocV1beta2
       VERSION = 'V1beta2'
-      REVISION = '20200511'
+      REVISION = '20200619'
 
       # View and manage your data across Google Cloud Platform services
       AUTH_CLOUD_PLATFORM = 'https://www.googleapis.com/auth/cloud-platform'

data/generated/google/apis/dataproc_v1beta2/classes.rb

@@ -175,11 +175,12 @@ module Google
         # @return [String]
         attr_accessor :graceful_decommission_timeout
       
-        # Required. Fraction of average pending memory in the last cooldown period for
-        # which to remove workers. A scale-down factor of 1 will result in scaling down
-        # so that there is no available memory remaining after the update (more
+        # Required. Fraction of average YARN pending memory in the last cooldown period
+        # for which to remove workers. A scale-down factor of 1 will result in scaling
+        # down so that there is no available memory remaining after the update (more
         # aggressive scaling). A scale-down factor of 0 disables removing workers, which
-        # can be beneficial for autoscaling a single job.Bounds: 0.0, 1.0.
+        # can be beneficial for autoscaling a single job. See How autoscaling works for
+        # more information.Bounds: 0.0, 1.0.
         # Corresponds to the JSON property `scaleDownFactor`
         # @return [Float]
         attr_accessor :scale_down_factor
@@ -193,11 +194,12 @@ module Google
         # @return [Float]
         attr_accessor :scale_down_min_worker_fraction
       
-        # Required. Fraction of average pending memory in the last cooldown period for
-        # which to add workers. A scale-up factor of 1.0 will result in scaling up so
-        # that there is no pending memory remaining after the update (more aggressive
+        # Required. Fraction of average YARN pending memory in the last cooldown period
+        # for which to add workers. A scale-up factor of 1.0 will result in scaling up
+        # so that there is no pending memory remaining after the update (more aggressive
         # scaling). A scale-up factor closer to 0 will result in a smaller magnitude of
-        # scaling up (less aggressive scaling).Bounds: 0.0, 1.0.
+        # scaling up (less aggressive scaling). See How autoscaling works for more
+        # information.Bounds: 0.0, 1.0.
         # Corresponds to the JSON property `scaleUpFactor`
         # @return [Float]
         attr_accessor :scale_up_factor
@@ -472,6 +474,17 @@ module Google
         # @return [Google::Apis::DataprocV1beta2::SoftwareConfig]
         attr_accessor :software_config
       
+        # Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data,
+        # such as Spark and MapReduce history files. If you do not specify a temp
+        # bucket, Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for
+        # your cluster's temp bucket according to the Compute Engine zone where your
+        # cluster is deployed, and then create and manage this project-level, per-
+        # location bucket. The default bucket has a TTL of 90 days, but you can use any
+        # TTL (or none) if you specify a bucket.
+        # Corresponds to the JSON property `tempBucket`
+        # @return [String]
+        attr_accessor :temp_bucket
+      
         # The config settings for Compute Engine resources in an instance group, such as
         # a master or worker group.
         # Corresponds to the JSON property `workerConfig`
@@ -496,6 +509,7 @@ module Google
           @secondary_worker_config = args[:secondary_worker_config] if args.key?(:secondary_worker_config)
           @security_config = args[:security_config] if args.key?(:security_config)
           @software_config = args[:software_config] if args.key?(:software_config)
+          @temp_bucket = args[:temp_bucket] if args.key?(:temp_bucket)
           @worker_config = args[:worker_config] if args.key?(:worker_config)
         end
       end
@@ -1668,7 +1682,8 @@ module Google
         # @return [String]
         attr_accessor :job_id
       
-        # Required. The ID of the Google Cloud Platform project that the job belongs to.
+        # Optional. The ID of the Google Cloud Platform project that the job belongs to.
+        # If specified, must match the request project ID.
         # Corresponds to the JSON property `projectId`
         # @return [String]
         attr_accessor :project_id
@@ -2632,8 +2647,8 @@ module Google
       class PySparkJob
         include Google::Apis::Core::Hashable
       
-        # Optional. HCFS URIs of archives to be extracted in the working directory of .
-        # jar, .tar, .tar.gz, .tgz, and .zip.
+        # Optional. HCFS URIs of archives to be extracted into the working directory of
+        # each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.
         # Corresponds to the JSON property `archiveUris`
         # @return [Array<String>]
         attr_accessor :archive_uris
@@ -2645,8 +2660,8 @@ module Google
         # @return [Array<String>]
         attr_accessor :args
       
-        # Optional. HCFS URIs of files to be copied to the working directory of Python
-        # drivers and distributed tasks. Useful for naively parallel tasks.
+        # Optional. HCFS URIs of files to be placed in the working directory of each
+        # executor. Useful for naively parallel tasks.
         # Corresponds to the JSON property `fileUris`
         # @return [Array<String>]
         attr_accessor :file_uris
@@ -2932,9 +2947,8 @@ module Google
       class SparkJob
         include Google::Apis::Core::Hashable
       
-        # Optional. HCFS URIs of archives to be extracted in the working directory of
-        # Spark drivers and tasks. Supported file types: .jar, .tar, .tar.gz, .tgz, and .
-        # zip.
+        # Optional. HCFS URIs of archives to be extracted into the working directory of
+        # each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.
         # Corresponds to the JSON property `archiveUris`
         # @return [Array<String>]
         attr_accessor :archive_uris
@@ -2946,8 +2960,8 @@ module Google
         # @return [Array<String>]
         attr_accessor :args
       
-        # Optional. HCFS URIs of files to be copied to the working directory of Spark
-        # drivers and distributed tasks. Useful for naively parallel tasks.
+        # Optional. HCFS URIs of files to be placed in the working directory of each
+        # executor. Useful for naively parallel tasks.
         # Corresponds to the JSON property `fileUris`
         # @return [Array<String>]
         attr_accessor :file_uris
@@ -3004,9 +3018,8 @@ module Google
       class SparkRJob
         include Google::Apis::Core::Hashable
       
-        # Optional. HCFS URIs of archives to be extracted in the working directory of
-        # Spark drivers and tasks. Supported file types: .jar, .tar, .tar.gz, .tgz, and .
-        # zip.
+        # Optional. HCFS URIs of archives to be extracted into the working directory of
+        # each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.
         # Corresponds to the JSON property `archiveUris`
         # @return [Array<String>]
         attr_accessor :archive_uris
@@ -3018,8 +3031,8 @@ module Google
         # @return [Array<String>]
         attr_accessor :args
       
-        # Optional. HCFS URIs of files to be copied to the working directory of R
-        # drivers and distributed tasks. Useful for naively parallel tasks.
+        # Optional. HCFS URIs of files to be placed in the working directory of each
+        # executor. Useful for naively parallel tasks.
         # Corresponds to the JSON property `fileUris`
         # @return [Array<String>]
         attr_accessor :file_uris
@@ -3418,6 +3431,25 @@ module Google
         # @return [Google::Apis::DataprocV1beta2::ClusterOperation]
         attr_accessor :create_cluster
       
+        # Output only. DAG end time, only set for workflows with dag_timeout when DAG
+        # ends.
+        # Corresponds to the JSON property `dagEndTime`
+        # @return [String]
+        attr_accessor :dag_end_time
+      
+        # Output only. DAG start time, only set for workflows with dag_timeout when DAG
+        # begins.
+        # Corresponds to the JSON property `dagStartTime`
+        # @return [String]
+        attr_accessor :dag_start_time
+      
+        # Output only. The timeout duration for the DAG of jobs. Minimum timeout
+        # duration is 10 minutes and maximum is 24 hours, expressed as a google.protobuf.
+        # Duration. For example, "1800" = 1800 seconds/30 minutes duration.
+        # Corresponds to the JSON property `dagTimeout`
+        # @return [String]
+        attr_accessor :dag_timeout
+      
         # The cluster operation triggered by a workflow.
         # Corresponds to the JSON property `deleteCluster`
         # @return [Google::Apis::DataprocV1beta2::ClusterOperation]
@@ -3474,6 +3506,9 @@ module Google
           @cluster_name = args[:cluster_name] if args.key?(:cluster_name)
           @cluster_uuid = args[:cluster_uuid] if args.key?(:cluster_uuid)
           @create_cluster = args[:create_cluster] if args.key?(:create_cluster)
+          @dag_end_time = args[:dag_end_time] if args.key?(:dag_end_time)
+          @dag_start_time = args[:dag_start_time] if args.key?(:dag_start_time)
+          @dag_timeout = args[:dag_timeout] if args.key?(:dag_timeout)
           @delete_cluster = args[:delete_cluster] if args.key?(:delete_cluster)
           @end_time = args[:end_time] if args.key?(:end_time)
           @graph = args[:graph] if args.key?(:graph)
@@ -3537,6 +3572,12 @@ module Google
         # @return [String]
         attr_accessor :create_time
       
+        # Optional. Timeout for DAG of jobs. The timer begins when the first job is
+        # submitted. Minimum duration of 10 minutes, max of 24 hours.
+        # Corresponds to the JSON property `dagTimeout`
+        # @return [String]
+        attr_accessor :dag_timeout
+      
         # Required. The template id.The id must contain only letters (a-z, A-Z), numbers
         # (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore
         # or hyphen. Must consist of between 3 and 50 characters..
@@ -3607,6 +3648,7 @@ module Google
         # Update properties of this object
         def update!(**args)
           @create_time = args[:create_time] if args.key?(:create_time)
+          @dag_timeout = args[:dag_timeout] if args.key?(:dag_timeout)
           @id = args[:id] if args.key?(:id)
           @jobs = args[:jobs] if args.key?(:jobs)
           @labels = args[:labels] if args.key?(:labels)

data/generated/google/apis/dataproc_v1beta2/representations.rb

@@ -605,6 +605,7 @@ module Google
       
           property :software_config, as: 'softwareConfig', class: Google::Apis::DataprocV1beta2::SoftwareConfig, decorator: Google::Apis::DataprocV1beta2::SoftwareConfig::Representation
       
+          property :temp_bucket, as: 'tempBucket'
           property :worker_config, as: 'workerConfig', class: Google::Apis::DataprocV1beta2::InstanceGroupConfig, decorator: Google::Apis::DataprocV1beta2::InstanceGroupConfig::Representation
       
         end
@@ -1305,6 +1306,9 @@ module Google
           property :cluster_uuid, as: 'clusterUuid'
           property :create_cluster, as: 'createCluster', class: Google::Apis::DataprocV1beta2::ClusterOperation, decorator: Google::Apis::DataprocV1beta2::ClusterOperation::Representation
       
+          property :dag_end_time, as: 'dagEndTime'
+          property :dag_start_time, as: 'dagStartTime'
+          property :dag_timeout, as: 'dagTimeout'
           property :delete_cluster, as: 'deleteCluster', class: Google::Apis::DataprocV1beta2::ClusterOperation, decorator: Google::Apis::DataprocV1beta2::ClusterOperation::Representation
       
           property :end_time, as: 'endTime'
@@ -1333,6 +1337,7 @@ module Google
         # @private
         class Representation < Google::Apis::Core::JsonRepresentation
           property :create_time, as: 'createTime'
+          property :dag_timeout, as: 'dagTimeout'
           property :id, as: 'id'
           collection :jobs, as: 'jobs', class: Google::Apis::DataprocV1beta2::OrderedJob, decorator: Google::Apis::DataprocV1beta2::OrderedJob::Representation