google-cloud-dataproc 0.2.2 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/google/cloud/dataproc.rb +56 -3
- data/lib/google/cloud/dataproc/v1.rb +62 -3
- data/lib/google/cloud/dataproc/v1/cluster_controller_client.rb +65 -5
- data/lib/google/cloud/dataproc/v1/cluster_controller_client_config.json +10 -7
- data/lib/google/cloud/dataproc/v1/clusters_pb.rb +11 -0
- data/lib/google/cloud/dataproc/v1/clusters_services_pb.rb +3 -2
- data/lib/google/cloud/dataproc/v1/credentials.rb +1 -1
- data/lib/google/cloud/dataproc/v1/doc/google/cloud/dataproc/v1/clusters.rb +128 -45
- data/lib/google/cloud/dataproc/v1/doc/google/cloud/dataproc/v1/jobs.rb +32 -15
- data/lib/google/cloud/dataproc/v1/doc/google/cloud/dataproc/v1/workflow_templates.rb +519 -0
- data/lib/google/cloud/dataproc/v1/doc/google/longrunning/operations.rb +1 -43
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/any.rb +1 -1
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/duration.rb +1 -1
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/empty.rb +1 -1
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/field_mask.rb +1 -1
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/timestamp.rb +1 -1
- data/lib/google/cloud/dataproc/v1/doc/google/rpc/status.rb +1 -1
- data/lib/google/cloud/dataproc/v1/job_controller_client.rb +16 -2
- data/lib/google/cloud/dataproc/v1/job_controller_client_config.json +9 -6
- data/lib/google/cloud/dataproc/v1/jobs_pb.rb +2 -0
- data/lib/google/cloud/dataproc/v1/jobs_services_pb.rb +2 -1
- data/lib/google/cloud/dataproc/v1/operations_pb.rb +0 -2
- data/lib/google/cloud/dataproc/v1/workflow_template_service_client.rb +690 -0
- data/lib/google/cloud/dataproc/v1/workflow_template_service_client_config.json +64 -0
- data/lib/google/cloud/dataproc/v1/workflow_templates_pb.rb +181 -0
- data/lib/google/cloud/dataproc/v1/workflow_templates_services_pb.rb +102 -0
- data/lib/google/cloud/dataproc/v1beta2.rb +281 -0
- data/lib/google/cloud/dataproc/v1beta2/cluster_controller_client.rb +799 -0
- data/lib/google/cloud/dataproc/v1beta2/cluster_controller_client_config.json +59 -0
- data/lib/google/cloud/dataproc/v1beta2/clusters_pb.rb +189 -0
- data/lib/google/cloud/dataproc/v1beta2/clusters_services_pb.rb +60 -0
- data/lib/google/cloud/dataproc/v1beta2/credentials.rb +41 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/clusters.rb +666 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/jobs.rb +691 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/workflow_templates.rb +524 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/longrunning/operations.rb +51 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/any.rb +130 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/duration.rb +91 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/empty.rb +29 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/field_mask.rb +230 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/timestamp.rb +109 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/rpc/status.rb +84 -0
- data/lib/google/cloud/dataproc/v1beta2/job_controller_client.rb +581 -0
- data/lib/google/cloud/dataproc/v1beta2/job_controller_client_config.json +59 -0
- data/lib/google/cloud/dataproc/v1beta2/jobs_pb.rb +248 -0
- data/lib/google/cloud/dataproc/v1beta2/jobs_services_pb.rb +61 -0
- data/lib/google/cloud/dataproc/v1beta2/operations_pb.rb +44 -0
- data/lib/google/cloud/dataproc/v1beta2/shared_pb.rb +18 -0
- data/lib/google/cloud/dataproc/v1beta2/workflow_template_service_client.rb +698 -0
- data/lib/google/cloud/dataproc/v1beta2/workflow_template_service_client_config.json +64 -0
- data/lib/google/cloud/dataproc/v1beta2/workflow_templates_pb.rb +183 -0
- data/lib/google/cloud/dataproc/v1beta2/workflow_templates_services_pb.rb +102 -0
- metadata +36 -5
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright
|
1
|
+
# Copyright 2019 Google LLC
|
2
2
|
#
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
4
|
# you may not use this file except in compliance with the License.
|
@@ -18,7 +18,7 @@ module Google
|
|
18
18
|
module Dataproc
|
19
19
|
module V1
|
20
20
|
# Describes the identifying information, config, and status of
|
21
|
-
# a cluster of
|
21
|
+
# a cluster of Compute Engine instances.
|
22
22
|
# @!attribute [rw] project_id
|
23
23
|
# @return [String]
|
24
24
|
# Required. The Google Cloud Platform project ID that the cluster belongs to.
|
@@ -40,13 +40,13 @@ module Google
|
|
40
40
|
# No more than 32 labels can be associated with a cluster.
|
41
41
|
# @!attribute [rw] status
|
42
42
|
# @return [Google::Cloud::Dataproc::V1::ClusterStatus]
|
43
|
-
# Output
|
43
|
+
# Output only. Cluster status.
|
44
44
|
# @!attribute [rw] status_history
|
45
45
|
# @return [Array<Google::Cloud::Dataproc::V1::ClusterStatus>]
|
46
|
-
# Output
|
46
|
+
# Output only. The previous cluster status.
|
47
47
|
# @!attribute [rw] cluster_uuid
|
48
48
|
# @return [String]
|
49
|
-
# Output
|
49
|
+
# Output only. A cluster UUID (Unique Universal Identifier). Cloud Dataproc
|
50
50
|
# generates this value when it creates the cluster.
|
51
51
|
# @!attribute [rw] metrics
|
52
52
|
# @return [Google::Cloud::Dataproc::V1::ClusterMetrics]
|
@@ -59,7 +59,7 @@ module Google
|
|
59
59
|
# The cluster config.
|
60
60
|
# @!attribute [rw] config_bucket
|
61
61
|
# @return [String]
|
62
|
-
# Optional. A
|
62
|
+
# Optional. A Cloud Storage staging bucket used for sharing generated
|
63
63
|
# SSH keys and config. If you do not specify a staging bucket, Cloud
|
64
64
|
# Dataproc will determine an appropriate Cloud Storage location (US,
|
65
65
|
# ASIA, or EU) for your cluster's staging bucket according to the Google
|
@@ -67,19 +67,19 @@ module Google
|
|
67
67
|
# and manage this project-level, per-location bucket for you.
|
68
68
|
# @!attribute [rw] gce_cluster_config
|
69
69
|
# @return [Google::Cloud::Dataproc::V1::GceClusterConfig]
|
70
|
-
# Required. The shared
|
70
|
+
# Required. The shared Compute Engine config settings for
|
71
71
|
# all instances in a cluster.
|
72
72
|
# @!attribute [rw] master_config
|
73
73
|
# @return [Google::Cloud::Dataproc::V1::InstanceGroupConfig]
|
74
|
-
# Optional. The
|
74
|
+
# Optional. The Compute Engine config settings for
|
75
75
|
# the master instance in a cluster.
|
76
76
|
# @!attribute [rw] worker_config
|
77
77
|
# @return [Google::Cloud::Dataproc::V1::InstanceGroupConfig]
|
78
|
-
# Optional. The
|
78
|
+
# Optional. The Compute Engine config settings for
|
79
79
|
# worker instances in a cluster.
|
80
80
|
# @!attribute [rw] secondary_worker_config
|
81
81
|
# @return [Google::Cloud::Dataproc::V1::InstanceGroupConfig]
|
82
|
-
# Optional. The
|
82
|
+
# Optional. The Compute Engine config settings for
|
83
83
|
# additional worker instances in a cluster.
|
84
84
|
# @!attribute [rw] software_config
|
85
85
|
# @return [Google::Cloud::Dataproc::V1::SoftwareConfig]
|
@@ -97,13 +97,23 @@ module Google
|
|
97
97
|
# else
|
98
98
|
# ... worker specific actions ...
|
99
99
|
# fi
|
100
|
+
# @!attribute [rw] encryption_config
|
101
|
+
# @return [Google::Cloud::Dataproc::V1::EncryptionConfig]
|
102
|
+
# Optional. Encryption settings for the cluster.
|
100
103
|
class ClusterConfig; end
|
101
104
|
|
102
|
-
#
|
105
|
+
# Encryption settings for the cluster.
|
106
|
+
# @!attribute [rw] gce_pd_kms_key_name
|
107
|
+
# @return [String]
|
108
|
+
# Optional. The Cloud KMS key name to use for PD disk encryption for all
|
109
|
+
# instances in the cluster.
|
110
|
+
class EncryptionConfig; end
|
111
|
+
|
112
|
+
# Common config settings for resources of Compute Engine cluster
|
103
113
|
# instances, applicable to all instances in the cluster.
|
104
114
|
# @!attribute [rw] zone_uri
|
105
115
|
# @return [String]
|
106
|
-
# Optional. The zone where the
|
116
|
+
# Optional. The zone where the Compute Engine cluster will be located.
|
107
117
|
# On a create request, it is required in the "global" region. If omitted
|
108
118
|
# in a non-global Cloud Dataproc region, the service will pick a zone in the
|
109
119
|
# corresponding Compute Engine region. On a get request, zone will
|
@@ -116,7 +126,7 @@ module Google
|
|
116
126
|
# * `us-central1-f`
|
117
127
|
# @!attribute [rw] network_uri
|
118
128
|
# @return [String]
|
119
|
-
# Optional. The
|
129
|
+
# Optional. The Compute Engine network to be used for machine
|
120
130
|
# communications. Cannot be specified with subnetwork_uri. If neither
|
121
131
|
# `network_uri` nor `subnetwork_uri` is specified, the "default" network of
|
122
132
|
# the project is used, if it exists. Cannot be a "Custom Subnet Network" (see
|
@@ -129,7 +139,7 @@ module Google
|
|
129
139
|
# * `default`
|
130
140
|
# @!attribute [rw] subnetwork_uri
|
131
141
|
# @return [String]
|
132
|
-
# Optional. The
|
142
|
+
# Optional. The Compute Engine subnetwork to be used for machine
|
133
143
|
# communications. Cannot be specified with network_uri.
|
134
144
|
#
|
135
145
|
# A full URL, partial URI, or short name are valid. Examples:
|
@@ -148,8 +158,8 @@ module Google
|
|
148
158
|
# @!attribute [rw] service_account
|
149
159
|
# @return [String]
|
150
160
|
# Optional. The service account of the instances. Defaults to the default
|
151
|
-
#
|
152
|
-
# permissions equivalent to the
|
161
|
+
# Compute Engine service account. Custom service accounts need
|
162
|
+
# permissions equivalent to the following IAM roles:
|
153
163
|
#
|
154
164
|
# * roles/logging.logWriter
|
155
165
|
# * roles/storage.objectAdmin
|
@@ -159,7 +169,7 @@ module Google
|
|
159
169
|
# Example: `[account_id]@[project_id].iam.gserviceaccount.com`
|
160
170
|
# @!attribute [rw] service_account_scopes
|
161
171
|
# @return [Array<String>]
|
162
|
-
# Optional. The URIs of service account scopes to be included in
|
172
|
+
# Optional. The URIs of service account scopes to be included in
|
163
173
|
# Compute Engine instances. The following base set of scopes is always
|
164
174
|
# included:
|
165
175
|
#
|
@@ -175,15 +185,15 @@ module Google
|
|
175
185
|
# * https://www.googleapis.com/auth/devstorage.full_control
|
176
186
|
# @!attribute [rw] tags
|
177
187
|
# @return [Array<String>]
|
178
|
-
# The
|
188
|
+
# The Compute Engine tags to add to all instances (see
|
179
189
|
# [Tagging instances](https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).
|
180
190
|
# @!attribute [rw] metadata
|
181
191
|
# @return [Hash{String => String}]
|
182
|
-
# The
|
192
|
+
# The Compute Engine metadata entries to add to all instances (see
|
183
193
|
# [Project and instance metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).
|
184
194
|
class GceClusterConfig; end
|
185
195
|
|
186
|
-
# Optional. The config settings for
|
196
|
+
# Optional. The config settings for Compute Engine resources in
|
187
197
|
# an instance group, such as a master or worker group.
|
188
198
|
# @!attribute [rw] num_instances
|
189
199
|
# @return [Integer]
|
@@ -191,22 +201,27 @@ module Google
|
|
191
201
|
# For master instance groups, must be set to 1.
|
192
202
|
# @!attribute [rw] instance_names
|
193
203
|
# @return [Array<String>]
|
194
|
-
#
|
195
|
-
# `cluster_name`, `num_instances`, and the instance group
|
196
|
-
# (recommended practice is to let Cloud Dataproc derive the name).
|
204
|
+
# Output only. The list of instance names. Cloud Dataproc derives the names
|
205
|
+
# from `cluster_name`, `num_instances`, and the instance group.
|
197
206
|
# @!attribute [rw] image_uri
|
198
207
|
# @return [String]
|
199
|
-
#
|
200
|
-
# instances.
|
208
|
+
# Optional. The Compute Engine image resource used for cluster
|
209
|
+
# instances. It can be specified or may be inferred from
|
210
|
+
# `SoftwareConfig.image_version`.
|
201
211
|
# @!attribute [rw] machine_type_uri
|
202
212
|
# @return [String]
|
203
|
-
# Optional. The
|
213
|
+
# Optional. The Compute Engine machine type used for cluster instances.
|
204
214
|
#
|
205
215
|
# A full URL, partial URI, or short name are valid. Examples:
|
206
216
|
#
|
207
217
|
# * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2`
|
208
218
|
# * `projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2`
|
209
219
|
# * `n1-standard-2`
|
220
|
+
#
|
221
|
+
# **Auto Zone Exception**: If you are using the Cloud Dataproc
|
222
|
+
# [Auto Zone Placement](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
|
223
|
+
# feature, you must use the short name of the machine type
|
224
|
+
# resource, for example, `n1-standard-2`.
|
210
225
|
# @!attribute [rw] disk_config
|
211
226
|
# @return [Google::Cloud::Dataproc::V1::DiskConfig]
|
212
227
|
# Optional. Disk option config settings.
|
@@ -215,12 +230,12 @@ module Google
|
|
215
230
|
# Optional. Specifies that this instance group contains preemptible instances.
|
216
231
|
# @!attribute [rw] managed_group_config
|
217
232
|
# @return [Google::Cloud::Dataproc::V1::ManagedGroupConfig]
|
218
|
-
# Output
|
233
|
+
# Output only. The config for Compute Engine Instance Group
|
219
234
|
# Manager that manages this group.
|
220
235
|
# This is only used for preemptible instance groups.
|
221
236
|
# @!attribute [rw] accelerators
|
222
237
|
# @return [Array<Google::Cloud::Dataproc::V1::AcceleratorConfig>]
|
223
|
-
# Optional. The
|
238
|
+
# Optional. The Compute Engine accelerator configuration for these
|
224
239
|
# instances.
|
225
240
|
#
|
226
241
|
# **Beta Feature**: This feature is still under development. It may be
|
@@ -230,31 +245,42 @@ module Google
|
|
230
245
|
# Specifies the resources used to actively manage an instance group.
|
231
246
|
# @!attribute [rw] instance_template_name
|
232
247
|
# @return [String]
|
233
|
-
# Output
|
248
|
+
# Output only. The name of the Instance Template used for the Managed
|
234
249
|
# Instance Group.
|
235
250
|
# @!attribute [rw] instance_group_manager_name
|
236
251
|
# @return [String]
|
237
|
-
# Output
|
252
|
+
# Output only. The name of the Instance Group Manager for this group.
|
238
253
|
class ManagedGroupConfig; end
|
239
254
|
|
240
255
|
# Specifies the type and number of accelerator cards attached to the instances
|
241
|
-
# of an instance
|
256
|
+
# of an instance. See [GPUs on Compute Engine](https://cloud.google.com/compute/docs/gpus/).
|
242
257
|
# @!attribute [rw] accelerator_type_uri
|
243
258
|
# @return [String]
|
244
259
|
# Full URL, partial URI, or short name of the accelerator type resource to
|
245
|
-
# expose to this instance. See
|
246
|
-
# /compute/docs/reference/beta/acceleratorTypes)
|
260
|
+
# expose to this instance. See
|
261
|
+
# [Compute Engine AcceleratorTypes](https://cloud.google.com/compute/docs/reference/beta/acceleratorTypes).
|
262
|
+
#
|
263
|
+
# Examples:
|
247
264
|
#
|
248
|
-
# Examples
|
249
265
|
# * `https://www.googleapis.com/compute/beta/projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80`
|
250
266
|
# * `projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80`
|
251
267
|
# * `nvidia-tesla-k80`
|
268
|
+
#
|
269
|
+
# **Auto Zone Exception**: If you are using the Cloud Dataproc
|
270
|
+
# [Auto Zone Placement](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
|
271
|
+
# feature, you must use the short name of the accelerator type
|
272
|
+
# resource, for example, `nvidia-tesla-k80`.
|
252
273
|
# @!attribute [rw] accelerator_count
|
253
274
|
# @return [Integer]
|
254
275
|
# The number of the accelerator cards of this type exposed to this instance.
|
255
276
|
class AcceleratorConfig; end
|
256
277
|
|
257
278
|
# Specifies the config of disk options for a group of VM instances.
|
279
|
+
# @!attribute [rw] boot_disk_type
|
280
|
+
# @return [String]
|
281
|
+
# Optional. Type of the boot disk (default is "pd-standard").
|
282
|
+
# Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or
|
283
|
+
# "pd-standard" (Persistent Disk Hard Disk Drive).
|
258
284
|
# @!attribute [rw] boot_disk_size_gb
|
259
285
|
# @return [Integer]
|
260
286
|
# Optional. Size in GB of the boot disk (default is 500GB).
|
@@ -272,7 +298,7 @@ module Google
|
|
272
298
|
# timeout period for executable completion.
|
273
299
|
# @!attribute [rw] executable_file
|
274
300
|
# @return [String]
|
275
|
-
# Required.
|
301
|
+
# Required. Cloud Storage URI of executable file.
|
276
302
|
# @!attribute [rw] execution_timeout
|
277
303
|
# @return [Google::Protobuf::Duration]
|
278
304
|
# Optional. Amount of time executable has to complete. Default is
|
@@ -284,16 +310,16 @@ module Google
|
|
284
310
|
# The status of a cluster and its instances.
|
285
311
|
# @!attribute [rw] state
|
286
312
|
# @return [Google::Cloud::Dataproc::V1::ClusterStatus::State]
|
287
|
-
# Output
|
313
|
+
# Output only. The cluster's state.
|
288
314
|
# @!attribute [rw] detail
|
289
315
|
# @return [String]
|
290
|
-
# Output
|
316
|
+
# Output only. Optional details of cluster's state.
|
291
317
|
# @!attribute [rw] state_start_time
|
292
318
|
# @return [Google::Protobuf::Timestamp]
|
293
|
-
# Output
|
319
|
+
# Output only. Time when this state was entered.
|
294
320
|
# @!attribute [rw] substate
|
295
321
|
# @return [Google::Cloud::Dataproc::V1::ClusterStatus::Substate]
|
296
|
-
# Output
|
322
|
+
# Output only. Additional state information that includes
|
297
323
|
# status reported by the agent.
|
298
324
|
class ClusterStatus
|
299
325
|
# The cluster state.
|
@@ -317,7 +343,9 @@ module Google
|
|
317
343
|
UPDATING = 5
|
318
344
|
end
|
319
345
|
|
346
|
+
# The cluster substate.
|
320
347
|
module Substate
|
348
|
+
# The cluster substate is unknown.
|
321
349
|
UNSPECIFIED = 0
|
322
350
|
|
323
351
|
# The cluster is known to be in an unhealthy state
|
@@ -338,9 +366,11 @@ module Google
|
|
338
366
|
# Specifies the selection and config of software inside the cluster.
|
339
367
|
# @!attribute [rw] image_version
|
340
368
|
# @return [String]
|
341
|
-
# Optional. The version of software inside the cluster. It must
|
342
|
-
#
|
343
|
-
#
|
369
|
+
# Optional. The version of software inside the cluster. It must be one of the supported
|
370
|
+
# [Cloud Dataproc Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported_cloud_dataproc_versions),
|
371
|
+
# such as "1.2" (including a subminor version, such as "1.2.29"), or the
|
372
|
+
# ["preview" version](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
|
373
|
+
# If unspecified, it defaults to the latest version.
|
344
374
|
# @!attribute [rw] properties
|
345
375
|
# @return [Hash{String => String}]
|
346
376
|
# Optional. The properties to set on daemon config files.
|
@@ -386,6 +416,19 @@ module Google
|
|
386
416
|
# @!attribute [rw] cluster
|
387
417
|
# @return [Google::Cloud::Dataproc::V1::Cluster]
|
388
418
|
# Required. The cluster to create.
|
419
|
+
# @!attribute [rw] request_id
|
420
|
+
# @return [String]
|
421
|
+
# Optional. A unique id used to identify the request. If the server
|
422
|
+
# receives two {Google::Cloud::Dataproc::V1::CreateClusterRequest CreateClusterRequest} requests with the same
|
423
|
+
# id, then the second request will be ignored and the
|
424
|
+
# first {Google::Longrunning::Operation} created and stored in the backend
|
425
|
+
# is returned.
|
426
|
+
#
|
427
|
+
# It is recommended to always set this value to a
|
428
|
+
# [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
|
429
|
+
#
|
430
|
+
# The id must contain only letters (a-z, A-Z), numbers (0-9),
|
431
|
+
# underscores (_), and hyphens (-). The maximum length is 40 characters.
|
389
432
|
class CreateClusterRequest; end
|
390
433
|
|
391
434
|
# A request to update a cluster.
|
@@ -402,6 +445,16 @@ module Google
|
|
402
445
|
# @!attribute [rw] cluster
|
403
446
|
# @return [Google::Cloud::Dataproc::V1::Cluster]
|
404
447
|
# Required. The changes to the cluster.
|
448
|
+
# @!attribute [rw] graceful_decommission_timeout
|
449
|
+
# @return [Google::Protobuf::Duration]
|
450
|
+
# Optional. Timeout for graceful YARN decomissioning. Graceful
|
451
|
+
# decommissioning allows removing nodes from the cluster without
|
452
|
+
# interrupting jobs in progress. Timeout specifies how long to wait for jobs
|
453
|
+
# in progress to finish before forcefully removing nodes (and potentially
|
454
|
+
# interrupting jobs). Default timeout is 0 (for forceful decommission), and
|
455
|
+
# the maximum allowed timeout is 1 day.
|
456
|
+
#
|
457
|
+
# Only supported on Dataproc image versions 1.2 and higher.
|
405
458
|
# @!attribute [rw] update_mask
|
406
459
|
# @return [Google::Protobuf::FieldMask]
|
407
460
|
# Required. Specifies the path, relative to `Cluster`, of
|
@@ -451,6 +504,19 @@ module Google
|
|
451
504
|
# </tr>
|
452
505
|
# </tbody>
|
453
506
|
# </table>
|
507
|
+
# @!attribute [rw] request_id
|
508
|
+
# @return [String]
|
509
|
+
# Optional. A unique id used to identify the request. If the server
|
510
|
+
# receives two {Google::Cloud::Dataproc::V1::UpdateClusterRequest UpdateClusterRequest} requests with the same
|
511
|
+
# id, then the second request will be ignored and the
|
512
|
+
# first {Google::Longrunning::Operation} created and stored in the
|
513
|
+
# backend is returned.
|
514
|
+
#
|
515
|
+
# It is recommended to always set this value to a
|
516
|
+
# [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
|
517
|
+
#
|
518
|
+
# The id must contain only letters (a-z, A-Z), numbers (0-9),
|
519
|
+
# underscores (_), and hyphens (-). The maximum length is 40 characters.
|
454
520
|
class UpdateClusterRequest; end
|
455
521
|
|
456
522
|
# A request to delete a cluster.
|
@@ -464,6 +530,23 @@ module Google
|
|
464
530
|
# @!attribute [rw] cluster_name
|
465
531
|
# @return [String]
|
466
532
|
# Required. The cluster name.
|
533
|
+
# @!attribute [rw] cluster_uuid
|
534
|
+
# @return [String]
|
535
|
+
# Optional. Specifying the `cluster_uuid` means the RPC should fail
|
536
|
+
# (with error NOT_FOUND) if cluster with specified UUID does not exist.
|
537
|
+
# @!attribute [rw] request_id
|
538
|
+
# @return [String]
|
539
|
+
# Optional. A unique id used to identify the request. If the server
|
540
|
+
# receives two {Google::Cloud::Dataproc::V1::DeleteClusterRequest DeleteClusterRequest} requests with the same
|
541
|
+
# id, then the second request will be ignored and the
|
542
|
+
# first {Google::Longrunning::Operation} created and stored in the
|
543
|
+
# backend is returned.
|
544
|
+
#
|
545
|
+
# It is recommended to always set this value to a
|
546
|
+
# [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
|
547
|
+
#
|
548
|
+
# The id must contain only letters (a-z, A-Z), numbers (0-9),
|
549
|
+
# underscores (_), and hyphens (-). The maximum length is 40 characters.
|
467
550
|
class DeleteClusterRequest; end
|
468
551
|
|
469
552
|
# Request to get the resource representation for a cluster in a project.
|
@@ -519,10 +602,10 @@ module Google
|
|
519
602
|
# The list of all clusters in a project.
|
520
603
|
# @!attribute [rw] clusters
|
521
604
|
# @return [Array<Google::Cloud::Dataproc::V1::Cluster>]
|
522
|
-
# Output
|
605
|
+
# Output only. The clusters in the project.
|
523
606
|
# @!attribute [rw] next_page_token
|
524
607
|
# @return [String]
|
525
|
-
# Output
|
608
|
+
# Output only. This token is included in the response if there are more
|
526
609
|
# results to fetch. To fetch additional results, provide this value as the
|
527
610
|
# `page_token` in a subsequent `ListClustersRequest`.
|
528
611
|
class ListClustersResponse; end
|
@@ -543,7 +626,7 @@ module Google
|
|
543
626
|
# The location of diagnostic output.
|
544
627
|
# @!attribute [rw] output_uri
|
545
628
|
# @return [String]
|
546
|
-
# Output
|
629
|
+
# Output only. The Cloud Storage URI of the diagnostic output.
|
547
630
|
# The output report is a plain text file with a summary of collected
|
548
631
|
# diagnostics.
|
549
632
|
class DiagnoseClusterResults; end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright
|
1
|
+
# Copyright 2019 Google LLC
|
2
2
|
#
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
4
|
# you may not use this file except in compliance with the License.
|
@@ -292,24 +292,24 @@ module Google
|
|
292
292
|
# Required. The name of the cluster where the job will be submitted.
|
293
293
|
# @!attribute [rw] cluster_uuid
|
294
294
|
# @return [String]
|
295
|
-
# Output
|
295
|
+
# Output only. A cluster UUID generated by the Cloud Dataproc service when
|
296
296
|
# the job is submitted.
|
297
297
|
class JobPlacement; end
|
298
298
|
|
299
299
|
# Cloud Dataproc job status.
|
300
300
|
# @!attribute [rw] state
|
301
301
|
# @return [Google::Cloud::Dataproc::V1::JobStatus::State]
|
302
|
-
# Output
|
302
|
+
# Output only. A state message specifying the overall job state.
|
303
303
|
# @!attribute [rw] details
|
304
304
|
# @return [String]
|
305
|
-
# Output
|
305
|
+
# Output only. Optional job state details, such as an error
|
306
306
|
# description if the state is <code>ERROR</code>.
|
307
307
|
# @!attribute [rw] state_start_time
|
308
308
|
# @return [Google::Protobuf::Timestamp]
|
309
|
-
# Output
|
309
|
+
# Output only. The time when this state was entered.
|
310
310
|
# @!attribute [rw] substate
|
311
311
|
# @return [Google::Cloud::Dataproc::V1::JobStatus::Substate]
|
312
|
-
# Output
|
312
|
+
# Output only. Additional state information, which includes
|
313
313
|
# status reported by the agent.
|
314
314
|
class JobStatus
|
315
315
|
# The job state.
|
@@ -350,7 +350,9 @@ module Google
|
|
350
350
|
ATTEMPT_FAILURE = 9
|
351
351
|
end
|
352
352
|
|
353
|
+
# The job substate.
|
353
354
|
module Substate
|
355
|
+
# The job substate is unknown.
|
354
356
|
UNSPECIFIED = 0
|
355
357
|
|
356
358
|
# The Job is submitted to the agent.
|
@@ -472,25 +474,25 @@ module Google
|
|
472
474
|
# Job is a SparkSql job.
|
473
475
|
# @!attribute [rw] status
|
474
476
|
# @return [Google::Cloud::Dataproc::V1::JobStatus]
|
475
|
-
# Output
|
477
|
+
# Output only. The job status. Additional application-specific
|
476
478
|
# status information may be contained in the <code>type_job</code>
|
477
479
|
# and <code>yarn_applications</code> fields.
|
478
480
|
# @!attribute [rw] status_history
|
479
481
|
# @return [Array<Google::Cloud::Dataproc::V1::JobStatus>]
|
480
|
-
# Output
|
482
|
+
# Output only. The previous job status.
|
481
483
|
# @!attribute [rw] yarn_applications
|
482
484
|
# @return [Array<Google::Cloud::Dataproc::V1::YarnApplication>]
|
483
|
-
# Output
|
485
|
+
# Output only. The collection of YARN applications spun up by this job.
|
484
486
|
#
|
485
487
|
# **Beta** Feature: This report is available for testing purposes only. It may
|
486
488
|
# be changed before final release.
|
487
489
|
# @!attribute [rw] driver_output_resource_uri
|
488
490
|
# @return [String]
|
489
|
-
# Output
|
491
|
+
# Output only. A URI pointing to the location of the stdout of the job's
|
490
492
|
# driver program.
|
491
493
|
# @!attribute [rw] driver_control_files_uri
|
492
494
|
# @return [String]
|
493
|
-
# Output
|
495
|
+
# Output only. If present, the location of miscellaneous control files
|
494
496
|
# which may be used as part of job setup and handling. If not present,
|
495
497
|
# control files may be placed in the same location as `driver_output_uri`.
|
496
498
|
# @!attribute [rw] labels
|
@@ -504,12 +506,14 @@ module Google
|
|
504
506
|
# @!attribute [rw] scheduling
|
505
507
|
# @return [Google::Cloud::Dataproc::V1::JobScheduling]
|
506
508
|
# Optional. Job scheduling configuration.
|
509
|
+
# @!attribute [rw] job_uuid
|
510
|
+
# @return [String]
|
511
|
+
# Output only. A UUID that uniquely identifies a job within the project
|
512
|
+
# over time. This is in contrast to a user-settable reference.job_id that
|
513
|
+
# may be reused over time.
|
507
514
|
class Job; end
|
508
515
|
|
509
516
|
# Job scheduling options.
|
510
|
-
#
|
511
|
-
# **Beta Feature**: These options are available for testing purposes only.
|
512
|
-
# They may be changed before final release.
|
513
517
|
# @!attribute [rw] max_failures_per_hour
|
514
518
|
# @return [Integer]
|
515
519
|
# Optional. Maximum number of times per hour a driver may be restarted as
|
@@ -533,6 +537,19 @@ module Google
|
|
533
537
|
# @!attribute [rw] job
|
534
538
|
# @return [Google::Cloud::Dataproc::V1::Job]
|
535
539
|
# Required. The job resource.
|
540
|
+
# @!attribute [rw] request_id
|
541
|
+
# @return [String]
|
542
|
+
# Optional. A unique id used to identify the request. If the server
|
543
|
+
# receives two {Google::Cloud::Dataproc::V1::SubmitJobRequest SubmitJobRequest} requests with the same
|
544
|
+
# id, then the second request will be ignored and the
|
545
|
+
# first {Google::Cloud::Dataproc::V1::Job Job} created and stored in the backend
|
546
|
+
# is returned.
|
547
|
+
#
|
548
|
+
# It is recommended to always set this value to a
|
549
|
+
# [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
|
550
|
+
#
|
551
|
+
# The id must contain only letters (a-z, A-Z), numbers (0-9),
|
552
|
+
# underscores (_), and hyphens (-). The maximum length is 40 characters.
|
536
553
|
class SubmitJobRequest; end
|
537
554
|
|
538
555
|
# A request to get the resource representation for a job in a project.
|
@@ -631,7 +648,7 @@ module Google
|
|
631
648
|
# A list of jobs in a project.
|
632
649
|
# @!attribute [rw] jobs
|
633
650
|
# @return [Array<Google::Cloud::Dataproc::V1::Job>]
|
634
|
-
# Output
|
651
|
+
# Output only. Jobs list.
|
635
652
|
# @!attribute [rw] next_page_token
|
636
653
|
# @return [String]
|
637
654
|
# Optional. This token is included in the response if there are more results
|