google-cloud-dataproc 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/google/cloud/dataproc.rb +56 -3
- data/lib/google/cloud/dataproc/v1.rb +62 -3
- data/lib/google/cloud/dataproc/v1/cluster_controller_client.rb +65 -5
- data/lib/google/cloud/dataproc/v1/cluster_controller_client_config.json +10 -7
- data/lib/google/cloud/dataproc/v1/clusters_pb.rb +11 -0
- data/lib/google/cloud/dataproc/v1/clusters_services_pb.rb +3 -2
- data/lib/google/cloud/dataproc/v1/credentials.rb +1 -1
- data/lib/google/cloud/dataproc/v1/doc/google/cloud/dataproc/v1/clusters.rb +128 -45
- data/lib/google/cloud/dataproc/v1/doc/google/cloud/dataproc/v1/jobs.rb +32 -15
- data/lib/google/cloud/dataproc/v1/doc/google/cloud/dataproc/v1/workflow_templates.rb +519 -0
- data/lib/google/cloud/dataproc/v1/doc/google/longrunning/operations.rb +1 -43
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/any.rb +1 -1
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/duration.rb +1 -1
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/empty.rb +1 -1
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/field_mask.rb +1 -1
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/timestamp.rb +1 -1
- data/lib/google/cloud/dataproc/v1/doc/google/rpc/status.rb +1 -1
- data/lib/google/cloud/dataproc/v1/job_controller_client.rb +16 -2
- data/lib/google/cloud/dataproc/v1/job_controller_client_config.json +9 -6
- data/lib/google/cloud/dataproc/v1/jobs_pb.rb +2 -0
- data/lib/google/cloud/dataproc/v1/jobs_services_pb.rb +2 -1
- data/lib/google/cloud/dataproc/v1/operations_pb.rb +0 -2
- data/lib/google/cloud/dataproc/v1/workflow_template_service_client.rb +690 -0
- data/lib/google/cloud/dataproc/v1/workflow_template_service_client_config.json +64 -0
- data/lib/google/cloud/dataproc/v1/workflow_templates_pb.rb +181 -0
- data/lib/google/cloud/dataproc/v1/workflow_templates_services_pb.rb +102 -0
- data/lib/google/cloud/dataproc/v1beta2.rb +281 -0
- data/lib/google/cloud/dataproc/v1beta2/cluster_controller_client.rb +799 -0
- data/lib/google/cloud/dataproc/v1beta2/cluster_controller_client_config.json +59 -0
- data/lib/google/cloud/dataproc/v1beta2/clusters_pb.rb +189 -0
- data/lib/google/cloud/dataproc/v1beta2/clusters_services_pb.rb +60 -0
- data/lib/google/cloud/dataproc/v1beta2/credentials.rb +41 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/clusters.rb +666 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/jobs.rb +691 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/workflow_templates.rb +524 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/longrunning/operations.rb +51 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/any.rb +130 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/duration.rb +91 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/empty.rb +29 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/field_mask.rb +230 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/timestamp.rb +109 -0
- data/lib/google/cloud/dataproc/v1beta2/doc/google/rpc/status.rb +84 -0
- data/lib/google/cloud/dataproc/v1beta2/job_controller_client.rb +581 -0
- data/lib/google/cloud/dataproc/v1beta2/job_controller_client_config.json +59 -0
- data/lib/google/cloud/dataproc/v1beta2/jobs_pb.rb +248 -0
- data/lib/google/cloud/dataproc/v1beta2/jobs_services_pb.rb +61 -0
- data/lib/google/cloud/dataproc/v1beta2/operations_pb.rb +44 -0
- data/lib/google/cloud/dataproc/v1beta2/shared_pb.rb +18 -0
- data/lib/google/cloud/dataproc/v1beta2/workflow_template_service_client.rb +698 -0
- data/lib/google/cloud/dataproc/v1beta2/workflow_template_service_client_config.json +64 -0
- data/lib/google/cloud/dataproc/v1beta2/workflow_templates_pb.rb +183 -0
- data/lib/google/cloud/dataproc/v1beta2/workflow_templates_services_pb.rb +102 -0
- metadata +36 -5
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2019 Google LLC
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -18,7 +18,7 @@ module Google
|
|
|
18
18
|
module Dataproc
|
|
19
19
|
module V1
|
|
20
20
|
# Describes the identifying information, config, and status of
|
|
21
|
-
# a cluster of
|
|
21
|
+
# a cluster of Compute Engine instances.
|
|
22
22
|
# @!attribute [rw] project_id
|
|
23
23
|
# @return [String]
|
|
24
24
|
# Required. The Google Cloud Platform project ID that the cluster belongs to.
|
|
@@ -40,13 +40,13 @@ module Google
|
|
|
40
40
|
# No more than 32 labels can be associated with a cluster.
|
|
41
41
|
# @!attribute [rw] status
|
|
42
42
|
# @return [Google::Cloud::Dataproc::V1::ClusterStatus]
|
|
43
|
-
# Output
|
|
43
|
+
# Output only. Cluster status.
|
|
44
44
|
# @!attribute [rw] status_history
|
|
45
45
|
# @return [Array<Google::Cloud::Dataproc::V1::ClusterStatus>]
|
|
46
|
-
# Output
|
|
46
|
+
# Output only. The previous cluster status.
|
|
47
47
|
# @!attribute [rw] cluster_uuid
|
|
48
48
|
# @return [String]
|
|
49
|
-
# Output
|
|
49
|
+
# Output only. A cluster UUID (Unique Universal Identifier). Cloud Dataproc
|
|
50
50
|
# generates this value when it creates the cluster.
|
|
51
51
|
# @!attribute [rw] metrics
|
|
52
52
|
# @return [Google::Cloud::Dataproc::V1::ClusterMetrics]
|
|
@@ -59,7 +59,7 @@ module Google
|
|
|
59
59
|
# The cluster config.
|
|
60
60
|
# @!attribute [rw] config_bucket
|
|
61
61
|
# @return [String]
|
|
62
|
-
# Optional. A
|
|
62
|
+
# Optional. A Cloud Storage staging bucket used for sharing generated
|
|
63
63
|
# SSH keys and config. If you do not specify a staging bucket, Cloud
|
|
64
64
|
# Dataproc will determine an appropriate Cloud Storage location (US,
|
|
65
65
|
# ASIA, or EU) for your cluster's staging bucket according to the Google
|
|
@@ -67,19 +67,19 @@ module Google
|
|
|
67
67
|
# and manage this project-level, per-location bucket for you.
|
|
68
68
|
# @!attribute [rw] gce_cluster_config
|
|
69
69
|
# @return [Google::Cloud::Dataproc::V1::GceClusterConfig]
|
|
70
|
-
# Required. The shared
|
|
70
|
+
# Required. The shared Compute Engine config settings for
|
|
71
71
|
# all instances in a cluster.
|
|
72
72
|
# @!attribute [rw] master_config
|
|
73
73
|
# @return [Google::Cloud::Dataproc::V1::InstanceGroupConfig]
|
|
74
|
-
# Optional. The
|
|
74
|
+
# Optional. The Compute Engine config settings for
|
|
75
75
|
# the master instance in a cluster.
|
|
76
76
|
# @!attribute [rw] worker_config
|
|
77
77
|
# @return [Google::Cloud::Dataproc::V1::InstanceGroupConfig]
|
|
78
|
-
# Optional. The
|
|
78
|
+
# Optional. The Compute Engine config settings for
|
|
79
79
|
# worker instances in a cluster.
|
|
80
80
|
# @!attribute [rw] secondary_worker_config
|
|
81
81
|
# @return [Google::Cloud::Dataproc::V1::InstanceGroupConfig]
|
|
82
|
-
# Optional. The
|
|
82
|
+
# Optional. The Compute Engine config settings for
|
|
83
83
|
# additional worker instances in a cluster.
|
|
84
84
|
# @!attribute [rw] software_config
|
|
85
85
|
# @return [Google::Cloud::Dataproc::V1::SoftwareConfig]
|
|
@@ -97,13 +97,23 @@ module Google
|
|
|
97
97
|
# else
|
|
98
98
|
# ... worker specific actions ...
|
|
99
99
|
# fi
|
|
100
|
+
# @!attribute [rw] encryption_config
|
|
101
|
+
# @return [Google::Cloud::Dataproc::V1::EncryptionConfig]
|
|
102
|
+
# Optional. Encryption settings for the cluster.
|
|
100
103
|
class ClusterConfig; end
|
|
101
104
|
|
|
102
|
-
#
|
|
105
|
+
# Encryption settings for the cluster.
|
|
106
|
+
# @!attribute [rw] gce_pd_kms_key_name
|
|
107
|
+
# @return [String]
|
|
108
|
+
# Optional. The Cloud KMS key name to use for PD disk encryption for all
|
|
109
|
+
# instances in the cluster.
|
|
110
|
+
class EncryptionConfig; end
|
|
111
|
+
|
|
112
|
+
# Common config settings for resources of Compute Engine cluster
|
|
103
113
|
# instances, applicable to all instances in the cluster.
|
|
104
114
|
# @!attribute [rw] zone_uri
|
|
105
115
|
# @return [String]
|
|
106
|
-
# Optional. The zone where the
|
|
116
|
+
# Optional. The zone where the Compute Engine cluster will be located.
|
|
107
117
|
# On a create request, it is required in the "global" region. If omitted
|
|
108
118
|
# in a non-global Cloud Dataproc region, the service will pick a zone in the
|
|
109
119
|
# corresponding Compute Engine region. On a get request, zone will
|
|
@@ -116,7 +126,7 @@ module Google
|
|
|
116
126
|
# * `us-central1-f`
|
|
117
127
|
# @!attribute [rw] network_uri
|
|
118
128
|
# @return [String]
|
|
119
|
-
# Optional. The
|
|
129
|
+
# Optional. The Compute Engine network to be used for machine
|
|
120
130
|
# communications. Cannot be specified with subnetwork_uri. If neither
|
|
121
131
|
# `network_uri` nor `subnetwork_uri` is specified, the "default" network of
|
|
122
132
|
# the project is used, if it exists. Cannot be a "Custom Subnet Network" (see
|
|
@@ -129,7 +139,7 @@ module Google
|
|
|
129
139
|
# * `default`
|
|
130
140
|
# @!attribute [rw] subnetwork_uri
|
|
131
141
|
# @return [String]
|
|
132
|
-
# Optional. The
|
|
142
|
+
# Optional. The Compute Engine subnetwork to be used for machine
|
|
133
143
|
# communications. Cannot be specified with network_uri.
|
|
134
144
|
#
|
|
135
145
|
# A full URL, partial URI, or short name are valid. Examples:
|
|
@@ -148,8 +158,8 @@ module Google
|
|
|
148
158
|
# @!attribute [rw] service_account
|
|
149
159
|
# @return [String]
|
|
150
160
|
# Optional. The service account of the instances. Defaults to the default
|
|
151
|
-
#
|
|
152
|
-
# permissions equivalent to the
|
|
161
|
+
# Compute Engine service account. Custom service accounts need
|
|
162
|
+
# permissions equivalent to the following IAM roles:
|
|
153
163
|
#
|
|
154
164
|
# * roles/logging.logWriter
|
|
155
165
|
# * roles/storage.objectAdmin
|
|
@@ -159,7 +169,7 @@ module Google
|
|
|
159
169
|
# Example: `[account_id]@[project_id].iam.gserviceaccount.com`
|
|
160
170
|
# @!attribute [rw] service_account_scopes
|
|
161
171
|
# @return [Array<String>]
|
|
162
|
-
# Optional. The URIs of service account scopes to be included in
|
|
172
|
+
# Optional. The URIs of service account scopes to be included in
|
|
163
173
|
# Compute Engine instances. The following base set of scopes is always
|
|
164
174
|
# included:
|
|
165
175
|
#
|
|
@@ -175,15 +185,15 @@ module Google
|
|
|
175
185
|
# * https://www.googleapis.com/auth/devstorage.full_control
|
|
176
186
|
# @!attribute [rw] tags
|
|
177
187
|
# @return [Array<String>]
|
|
178
|
-
# The
|
|
188
|
+
# The Compute Engine tags to add to all instances (see
|
|
179
189
|
# [Tagging instances](https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).
|
|
180
190
|
# @!attribute [rw] metadata
|
|
181
191
|
# @return [Hash{String => String}]
|
|
182
|
-
# The
|
|
192
|
+
# The Compute Engine metadata entries to add to all instances (see
|
|
183
193
|
# [Project and instance metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).
|
|
184
194
|
class GceClusterConfig; end
|
|
185
195
|
|
|
186
|
-
# Optional. The config settings for
|
|
196
|
+
# Optional. The config settings for Compute Engine resources in
|
|
187
197
|
# an instance group, such as a master or worker group.
|
|
188
198
|
# @!attribute [rw] num_instances
|
|
189
199
|
# @return [Integer]
|
|
@@ -191,22 +201,27 @@ module Google
|
|
|
191
201
|
# For master instance groups, must be set to 1.
|
|
192
202
|
# @!attribute [rw] instance_names
|
|
193
203
|
# @return [Array<String>]
|
|
194
|
-
#
|
|
195
|
-
# `cluster_name`, `num_instances`, and the instance group
|
|
196
|
-
# (recommended practice is to let Cloud Dataproc derive the name).
|
|
204
|
+
# Output only. The list of instance names. Cloud Dataproc derives the names
|
|
205
|
+
# from `cluster_name`, `num_instances`, and the instance group.
|
|
197
206
|
# @!attribute [rw] image_uri
|
|
198
207
|
# @return [String]
|
|
199
|
-
#
|
|
200
|
-
# instances.
|
|
208
|
+
# Optional. The Compute Engine image resource used for cluster
|
|
209
|
+
# instances. It can be specified or may be inferred from
|
|
210
|
+
# `SoftwareConfig.image_version`.
|
|
201
211
|
# @!attribute [rw] machine_type_uri
|
|
202
212
|
# @return [String]
|
|
203
|
-
# Optional. The
|
|
213
|
+
# Optional. The Compute Engine machine type used for cluster instances.
|
|
204
214
|
#
|
|
205
215
|
# A full URL, partial URI, or short name are valid. Examples:
|
|
206
216
|
#
|
|
207
217
|
# * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2`
|
|
208
218
|
# * `projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2`
|
|
209
219
|
# * `n1-standard-2`
|
|
220
|
+
#
|
|
221
|
+
# **Auto Zone Exception**: If you are using the Cloud Dataproc
|
|
222
|
+
# [Auto Zone Placement](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
|
|
223
|
+
# feature, you must use the short name of the machine type
|
|
224
|
+
# resource, for example, `n1-standard-2`.
|
|
210
225
|
# @!attribute [rw] disk_config
|
|
211
226
|
# @return [Google::Cloud::Dataproc::V1::DiskConfig]
|
|
212
227
|
# Optional. Disk option config settings.
|
|
@@ -215,12 +230,12 @@ module Google
|
|
|
215
230
|
# Optional. Specifies that this instance group contains preemptible instances.
|
|
216
231
|
# @!attribute [rw] managed_group_config
|
|
217
232
|
# @return [Google::Cloud::Dataproc::V1::ManagedGroupConfig]
|
|
218
|
-
# Output
|
|
233
|
+
# Output only. The config for Compute Engine Instance Group
|
|
219
234
|
# Manager that manages this group.
|
|
220
235
|
# This is only used for preemptible instance groups.
|
|
221
236
|
# @!attribute [rw] accelerators
|
|
222
237
|
# @return [Array<Google::Cloud::Dataproc::V1::AcceleratorConfig>]
|
|
223
|
-
# Optional. The
|
|
238
|
+
# Optional. The Compute Engine accelerator configuration for these
|
|
224
239
|
# instances.
|
|
225
240
|
#
|
|
226
241
|
# **Beta Feature**: This feature is still under development. It may be
|
|
@@ -230,31 +245,42 @@ module Google
|
|
|
230
245
|
# Specifies the resources used to actively manage an instance group.
|
|
231
246
|
# @!attribute [rw] instance_template_name
|
|
232
247
|
# @return [String]
|
|
233
|
-
# Output
|
|
248
|
+
# Output only. The name of the Instance Template used for the Managed
|
|
234
249
|
# Instance Group.
|
|
235
250
|
# @!attribute [rw] instance_group_manager_name
|
|
236
251
|
# @return [String]
|
|
237
|
-
# Output
|
|
252
|
+
# Output only. The name of the Instance Group Manager for this group.
|
|
238
253
|
class ManagedGroupConfig; end
|
|
239
254
|
|
|
240
255
|
# Specifies the type and number of accelerator cards attached to the instances
|
|
241
|
-
# of an instance
|
|
256
|
+
# of an instance. See [GPUs on Compute Engine](https://cloud.google.com/compute/docs/gpus/).
|
|
242
257
|
# @!attribute [rw] accelerator_type_uri
|
|
243
258
|
# @return [String]
|
|
244
259
|
# Full URL, partial URI, or short name of the accelerator type resource to
|
|
245
|
-
# expose to this instance. See
|
|
246
|
-
# /compute/docs/reference/beta/acceleratorTypes)
|
|
260
|
+
# expose to this instance. See
|
|
261
|
+
# [Compute Engine AcceleratorTypes](https://cloud.google.com/compute/docs/reference/beta/acceleratorTypes).
|
|
262
|
+
#
|
|
263
|
+
# Examples:
|
|
247
264
|
#
|
|
248
|
-
# Examples
|
|
249
265
|
# * `https://www.googleapis.com/compute/beta/projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80`
|
|
250
266
|
# * `projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80`
|
|
251
267
|
# * `nvidia-tesla-k80`
|
|
268
|
+
#
|
|
269
|
+
# **Auto Zone Exception**: If you are using the Cloud Dataproc
|
|
270
|
+
# [Auto Zone Placement](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
|
|
271
|
+
# feature, you must use the short name of the accelerator type
|
|
272
|
+
# resource, for example, `nvidia-tesla-k80`.
|
|
252
273
|
# @!attribute [rw] accelerator_count
|
|
253
274
|
# @return [Integer]
|
|
254
275
|
# The number of the accelerator cards of this type exposed to this instance.
|
|
255
276
|
class AcceleratorConfig; end
|
|
256
277
|
|
|
257
278
|
# Specifies the config of disk options for a group of VM instances.
|
|
279
|
+
# @!attribute [rw] boot_disk_type
|
|
280
|
+
# @return [String]
|
|
281
|
+
# Optional. Type of the boot disk (default is "pd-standard").
|
|
282
|
+
# Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or
|
|
283
|
+
# "pd-standard" (Persistent Disk Hard Disk Drive).
|
|
258
284
|
# @!attribute [rw] boot_disk_size_gb
|
|
259
285
|
# @return [Integer]
|
|
260
286
|
# Optional. Size in GB of the boot disk (default is 500GB).
|
|
@@ -272,7 +298,7 @@ module Google
|
|
|
272
298
|
# timeout period for executable completion.
|
|
273
299
|
# @!attribute [rw] executable_file
|
|
274
300
|
# @return [String]
|
|
275
|
-
# Required.
|
|
301
|
+
# Required. Cloud Storage URI of executable file.
|
|
276
302
|
# @!attribute [rw] execution_timeout
|
|
277
303
|
# @return [Google::Protobuf::Duration]
|
|
278
304
|
# Optional. Amount of time executable has to complete. Default is
|
|
@@ -284,16 +310,16 @@ module Google
|
|
|
284
310
|
# The status of a cluster and its instances.
|
|
285
311
|
# @!attribute [rw] state
|
|
286
312
|
# @return [Google::Cloud::Dataproc::V1::ClusterStatus::State]
|
|
287
|
-
# Output
|
|
313
|
+
# Output only. The cluster's state.
|
|
288
314
|
# @!attribute [rw] detail
|
|
289
315
|
# @return [String]
|
|
290
|
-
# Output
|
|
316
|
+
# Output only. Optional details of cluster's state.
|
|
291
317
|
# @!attribute [rw] state_start_time
|
|
292
318
|
# @return [Google::Protobuf::Timestamp]
|
|
293
|
-
# Output
|
|
319
|
+
# Output only. Time when this state was entered.
|
|
294
320
|
# @!attribute [rw] substate
|
|
295
321
|
# @return [Google::Cloud::Dataproc::V1::ClusterStatus::Substate]
|
|
296
|
-
# Output
|
|
322
|
+
# Output only. Additional state information that includes
|
|
297
323
|
# status reported by the agent.
|
|
298
324
|
class ClusterStatus
|
|
299
325
|
# The cluster state.
|
|
@@ -317,7 +343,9 @@ module Google
|
|
|
317
343
|
UPDATING = 5
|
|
318
344
|
end
|
|
319
345
|
|
|
346
|
+
# The cluster substate.
|
|
320
347
|
module Substate
|
|
348
|
+
# The cluster substate is unknown.
|
|
321
349
|
UNSPECIFIED = 0
|
|
322
350
|
|
|
323
351
|
# The cluster is known to be in an unhealthy state
|
|
@@ -338,9 +366,11 @@ module Google
|
|
|
338
366
|
# Specifies the selection and config of software inside the cluster.
|
|
339
367
|
# @!attribute [rw] image_version
|
|
340
368
|
# @return [String]
|
|
341
|
-
# Optional. The version of software inside the cluster. It must
|
|
342
|
-
#
|
|
343
|
-
#
|
|
369
|
+
# Optional. The version of software inside the cluster. It must be one of the supported
|
|
370
|
+
# [Cloud Dataproc Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported_cloud_dataproc_versions),
|
|
371
|
+
# such as "1.2" (including a subminor version, such as "1.2.29"), or the
|
|
372
|
+
# ["preview" version](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
|
|
373
|
+
# If unspecified, it defaults to the latest version.
|
|
344
374
|
# @!attribute [rw] properties
|
|
345
375
|
# @return [Hash{String => String}]
|
|
346
376
|
# Optional. The properties to set on daemon config files.
|
|
@@ -386,6 +416,19 @@ module Google
|
|
|
386
416
|
# @!attribute [rw] cluster
|
|
387
417
|
# @return [Google::Cloud::Dataproc::V1::Cluster]
|
|
388
418
|
# Required. The cluster to create.
|
|
419
|
+
# @!attribute [rw] request_id
|
|
420
|
+
# @return [String]
|
|
421
|
+
# Optional. A unique id used to identify the request. If the server
|
|
422
|
+
# receives two {Google::Cloud::Dataproc::V1::CreateClusterRequest CreateClusterRequest} requests with the same
|
|
423
|
+
# id, then the second request will be ignored and the
|
|
424
|
+
# first {Google::Longrunning::Operation} created and stored in the backend
|
|
425
|
+
# is returned.
|
|
426
|
+
#
|
|
427
|
+
# It is recommended to always set this value to a
|
|
428
|
+
# [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
|
|
429
|
+
#
|
|
430
|
+
# The id must contain only letters (a-z, A-Z), numbers (0-9),
|
|
431
|
+
# underscores (_), and hyphens (-). The maximum length is 40 characters.
|
|
389
432
|
class CreateClusterRequest; end
|
|
390
433
|
|
|
391
434
|
# A request to update a cluster.
|
|
@@ -402,6 +445,16 @@ module Google
|
|
|
402
445
|
# @!attribute [rw] cluster
|
|
403
446
|
# @return [Google::Cloud::Dataproc::V1::Cluster]
|
|
404
447
|
# Required. The changes to the cluster.
|
|
448
|
+
# @!attribute [rw] graceful_decommission_timeout
|
|
449
|
+
# @return [Google::Protobuf::Duration]
|
|
450
|
+
# Optional. Timeout for graceful YARN decomissioning. Graceful
|
|
451
|
+
# decommissioning allows removing nodes from the cluster without
|
|
452
|
+
# interrupting jobs in progress. Timeout specifies how long to wait for jobs
|
|
453
|
+
# in progress to finish before forcefully removing nodes (and potentially
|
|
454
|
+
# interrupting jobs). Default timeout is 0 (for forceful decommission), and
|
|
455
|
+
# the maximum allowed timeout is 1 day.
|
|
456
|
+
#
|
|
457
|
+
# Only supported on Dataproc image versions 1.2 and higher.
|
|
405
458
|
# @!attribute [rw] update_mask
|
|
406
459
|
# @return [Google::Protobuf::FieldMask]
|
|
407
460
|
# Required. Specifies the path, relative to `Cluster`, of
|
|
@@ -451,6 +504,19 @@ module Google
|
|
|
451
504
|
# </tr>
|
|
452
505
|
# </tbody>
|
|
453
506
|
# </table>
|
|
507
|
+
# @!attribute [rw] request_id
|
|
508
|
+
# @return [String]
|
|
509
|
+
# Optional. A unique id used to identify the request. If the server
|
|
510
|
+
# receives two {Google::Cloud::Dataproc::V1::UpdateClusterRequest UpdateClusterRequest} requests with the same
|
|
511
|
+
# id, then the second request will be ignored and the
|
|
512
|
+
# first {Google::Longrunning::Operation} created and stored in the
|
|
513
|
+
# backend is returned.
|
|
514
|
+
#
|
|
515
|
+
# It is recommended to always set this value to a
|
|
516
|
+
# [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
|
|
517
|
+
#
|
|
518
|
+
# The id must contain only letters (a-z, A-Z), numbers (0-9),
|
|
519
|
+
# underscores (_), and hyphens (-). The maximum length is 40 characters.
|
|
454
520
|
class UpdateClusterRequest; end
|
|
455
521
|
|
|
456
522
|
# A request to delete a cluster.
|
|
@@ -464,6 +530,23 @@ module Google
|
|
|
464
530
|
# @!attribute [rw] cluster_name
|
|
465
531
|
# @return [String]
|
|
466
532
|
# Required. The cluster name.
|
|
533
|
+
# @!attribute [rw] cluster_uuid
|
|
534
|
+
# @return [String]
|
|
535
|
+
# Optional. Specifying the `cluster_uuid` means the RPC should fail
|
|
536
|
+
# (with error NOT_FOUND) if cluster with specified UUID does not exist.
|
|
537
|
+
# @!attribute [rw] request_id
|
|
538
|
+
# @return [String]
|
|
539
|
+
# Optional. A unique id used to identify the request. If the server
|
|
540
|
+
# receives two {Google::Cloud::Dataproc::V1::DeleteClusterRequest DeleteClusterRequest} requests with the same
|
|
541
|
+
# id, then the second request will be ignored and the
|
|
542
|
+
# first {Google::Longrunning::Operation} created and stored in the
|
|
543
|
+
# backend is returned.
|
|
544
|
+
#
|
|
545
|
+
# It is recommended to always set this value to a
|
|
546
|
+
# [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
|
|
547
|
+
#
|
|
548
|
+
# The id must contain only letters (a-z, A-Z), numbers (0-9),
|
|
549
|
+
# underscores (_), and hyphens (-). The maximum length is 40 characters.
|
|
467
550
|
class DeleteClusterRequest; end
|
|
468
551
|
|
|
469
552
|
# Request to get the resource representation for a cluster in a project.
|
|
@@ -519,10 +602,10 @@ module Google
|
|
|
519
602
|
# The list of all clusters in a project.
|
|
520
603
|
# @!attribute [rw] clusters
|
|
521
604
|
# @return [Array<Google::Cloud::Dataproc::V1::Cluster>]
|
|
522
|
-
# Output
|
|
605
|
+
# Output only. The clusters in the project.
|
|
523
606
|
# @!attribute [rw] next_page_token
|
|
524
607
|
# @return [String]
|
|
525
|
-
# Output
|
|
608
|
+
# Output only. This token is included in the response if there are more
|
|
526
609
|
# results to fetch. To fetch additional results, provide this value as the
|
|
527
610
|
# `page_token` in a subsequent `ListClustersRequest`.
|
|
528
611
|
class ListClustersResponse; end
|
|
@@ -543,7 +626,7 @@ module Google
|
|
|
543
626
|
# The location of diagnostic output.
|
|
544
627
|
# @!attribute [rw] output_uri
|
|
545
628
|
# @return [String]
|
|
546
|
-
# Output
|
|
629
|
+
# Output only. The Cloud Storage URI of the diagnostic output.
|
|
547
630
|
# The output report is a plain text file with a summary of collected
|
|
548
631
|
# diagnostics.
|
|
549
632
|
class DiagnoseClusterResults; end
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2019 Google LLC
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -292,24 +292,24 @@ module Google
|
|
|
292
292
|
# Required. The name of the cluster where the job will be submitted.
|
|
293
293
|
# @!attribute [rw] cluster_uuid
|
|
294
294
|
# @return [String]
|
|
295
|
-
# Output
|
|
295
|
+
# Output only. A cluster UUID generated by the Cloud Dataproc service when
|
|
296
296
|
# the job is submitted.
|
|
297
297
|
class JobPlacement; end
|
|
298
298
|
|
|
299
299
|
# Cloud Dataproc job status.
|
|
300
300
|
# @!attribute [rw] state
|
|
301
301
|
# @return [Google::Cloud::Dataproc::V1::JobStatus::State]
|
|
302
|
-
# Output
|
|
302
|
+
# Output only. A state message specifying the overall job state.
|
|
303
303
|
# @!attribute [rw] details
|
|
304
304
|
# @return [String]
|
|
305
|
-
# Output
|
|
305
|
+
# Output only. Optional job state details, such as an error
|
|
306
306
|
# description if the state is <code>ERROR</code>.
|
|
307
307
|
# @!attribute [rw] state_start_time
|
|
308
308
|
# @return [Google::Protobuf::Timestamp]
|
|
309
|
-
# Output
|
|
309
|
+
# Output only. The time when this state was entered.
|
|
310
310
|
# @!attribute [rw] substate
|
|
311
311
|
# @return [Google::Cloud::Dataproc::V1::JobStatus::Substate]
|
|
312
|
-
# Output
|
|
312
|
+
# Output only. Additional state information, which includes
|
|
313
313
|
# status reported by the agent.
|
|
314
314
|
class JobStatus
|
|
315
315
|
# The job state.
|
|
@@ -350,7 +350,9 @@ module Google
|
|
|
350
350
|
ATTEMPT_FAILURE = 9
|
|
351
351
|
end
|
|
352
352
|
|
|
353
|
+
# The job substate.
|
|
353
354
|
module Substate
|
|
355
|
+
# The job substate is unknown.
|
|
354
356
|
UNSPECIFIED = 0
|
|
355
357
|
|
|
356
358
|
# The Job is submitted to the agent.
|
|
@@ -472,25 +474,25 @@ module Google
|
|
|
472
474
|
# Job is a SparkSql job.
|
|
473
475
|
# @!attribute [rw] status
|
|
474
476
|
# @return [Google::Cloud::Dataproc::V1::JobStatus]
|
|
475
|
-
# Output
|
|
477
|
+
# Output only. The job status. Additional application-specific
|
|
476
478
|
# status information may be contained in the <code>type_job</code>
|
|
477
479
|
# and <code>yarn_applications</code> fields.
|
|
478
480
|
# @!attribute [rw] status_history
|
|
479
481
|
# @return [Array<Google::Cloud::Dataproc::V1::JobStatus>]
|
|
480
|
-
# Output
|
|
482
|
+
# Output only. The previous job status.
|
|
481
483
|
# @!attribute [rw] yarn_applications
|
|
482
484
|
# @return [Array<Google::Cloud::Dataproc::V1::YarnApplication>]
|
|
483
|
-
# Output
|
|
485
|
+
# Output only. The collection of YARN applications spun up by this job.
|
|
484
486
|
#
|
|
485
487
|
# **Beta** Feature: This report is available for testing purposes only. It may
|
|
486
488
|
# be changed before final release.
|
|
487
489
|
# @!attribute [rw] driver_output_resource_uri
|
|
488
490
|
# @return [String]
|
|
489
|
-
# Output
|
|
491
|
+
# Output only. A URI pointing to the location of the stdout of the job's
|
|
490
492
|
# driver program.
|
|
491
493
|
# @!attribute [rw] driver_control_files_uri
|
|
492
494
|
# @return [String]
|
|
493
|
-
# Output
|
|
495
|
+
# Output only. If present, the location of miscellaneous control files
|
|
494
496
|
# which may be used as part of job setup and handling. If not present,
|
|
495
497
|
# control files may be placed in the same location as `driver_output_uri`.
|
|
496
498
|
# @!attribute [rw] labels
|
|
@@ -504,12 +506,14 @@ module Google
|
|
|
504
506
|
# @!attribute [rw] scheduling
|
|
505
507
|
# @return [Google::Cloud::Dataproc::V1::JobScheduling]
|
|
506
508
|
# Optional. Job scheduling configuration.
|
|
509
|
+
# @!attribute [rw] job_uuid
|
|
510
|
+
# @return [String]
|
|
511
|
+
# Output only. A UUID that uniquely identifies a job within the project
|
|
512
|
+
# over time. This is in contrast to a user-settable reference.job_id that
|
|
513
|
+
# may be reused over time.
|
|
507
514
|
class Job; end
|
|
508
515
|
|
|
509
516
|
# Job scheduling options.
|
|
510
|
-
#
|
|
511
|
-
# **Beta Feature**: These options are available for testing purposes only.
|
|
512
|
-
# They may be changed before final release.
|
|
513
517
|
# @!attribute [rw] max_failures_per_hour
|
|
514
518
|
# @return [Integer]
|
|
515
519
|
# Optional. Maximum number of times per hour a driver may be restarted as
|
|
@@ -533,6 +537,19 @@ module Google
|
|
|
533
537
|
# @!attribute [rw] job
|
|
534
538
|
# @return [Google::Cloud::Dataproc::V1::Job]
|
|
535
539
|
# Required. The job resource.
|
|
540
|
+
# @!attribute [rw] request_id
|
|
541
|
+
# @return [String]
|
|
542
|
+
# Optional. A unique id used to identify the request. If the server
|
|
543
|
+
# receives two {Google::Cloud::Dataproc::V1::SubmitJobRequest SubmitJobRequest} requests with the same
|
|
544
|
+
# id, then the second request will be ignored and the
|
|
545
|
+
# first {Google::Cloud::Dataproc::V1::Job Job} created and stored in the backend
|
|
546
|
+
# is returned.
|
|
547
|
+
#
|
|
548
|
+
# It is recommended to always set this value to a
|
|
549
|
+
# [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
|
|
550
|
+
#
|
|
551
|
+
# The id must contain only letters (a-z, A-Z), numbers (0-9),
|
|
552
|
+
# underscores (_), and hyphens (-). The maximum length is 40 characters.
|
|
536
553
|
class SubmitJobRequest; end
|
|
537
554
|
|
|
538
555
|
# A request to get the resource representation for a job in a project.
|
|
@@ -631,7 +648,7 @@ module Google
|
|
|
631
648
|
# A list of jobs in a project.
|
|
632
649
|
# @!attribute [rw] jobs
|
|
633
650
|
# @return [Array<Google::Cloud::Dataproc::V1::Job>]
|
|
634
|
-
# Output
|
|
651
|
+
# Output only. Jobs list.
|
|
635
652
|
# @!attribute [rw] next_page_token
|
|
636
653
|
# @return [String]
|
|
637
654
|
# Optional. This token is included in the response if there are more results
|