google-cloud-dataproc 0.10.0 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (72) hide show
  1. checksums.yaml +4 -4
  2. data/.yardopts +3 -2
  3. data/AUTHENTICATION.md +51 -59
  4. data/LICENSE.md +203 -0
  5. data/MIGRATING.md +310 -0
  6. data/README.md +33 -44
  7. data/lib/{google/cloud/dataproc/v1/doc/google/protobuf/empty.rb → google-cloud-dataproc.rb} +4 -14
  8. data/lib/google/cloud/dataproc.rb +167 -305
  9. data/lib/google/cloud/dataproc/version.rb +6 -2
  10. metadata +97 -100
  11. data/LICENSE +0 -201
  12. data/lib/google/cloud/dataproc/v1.rb +0 -371
  13. data/lib/google/cloud/dataproc/v1/autoscaling_policies_pb.rb +0 -80
  14. data/lib/google/cloud/dataproc/v1/autoscaling_policies_services_pb.rb +0 -59
  15. data/lib/google/cloud/dataproc/v1/autoscaling_policy_service_client.rb +0 -494
  16. data/lib/google/cloud/dataproc/v1/autoscaling_policy_service_client_config.json +0 -51
  17. data/lib/google/cloud/dataproc/v1/cluster_controller_client.rb +0 -824
  18. data/lib/google/cloud/dataproc/v1/cluster_controller_client_config.json +0 -59
  19. data/lib/google/cloud/dataproc/v1/clusters_pb.rb +0 -234
  20. data/lib/google/cloud/dataproc/v1/clusters_services_pb.rb +0 -69
  21. data/lib/google/cloud/dataproc/v1/credentials.rb +0 -41
  22. data/lib/google/cloud/dataproc/v1/doc/google/cloud/dataproc/v1/autoscaling_policies.rb +0 -238
  23. data/lib/google/cloud/dataproc/v1/doc/google/cloud/dataproc/v1/clusters.rb +0 -819
  24. data/lib/google/cloud/dataproc/v1/doc/google/cloud/dataproc/v1/jobs.rb +0 -759
  25. data/lib/google/cloud/dataproc/v1/doc/google/cloud/dataproc/v1/workflow_templates.rb +0 -566
  26. data/lib/google/cloud/dataproc/v1/doc/google/longrunning/operations.rb +0 -51
  27. data/lib/google/cloud/dataproc/v1/doc/google/protobuf/any.rb +0 -131
  28. data/lib/google/cloud/dataproc/v1/doc/google/protobuf/duration.rb +0 -91
  29. data/lib/google/cloud/dataproc/v1/doc/google/protobuf/field_mask.rb +0 -222
  30. data/lib/google/cloud/dataproc/v1/doc/google/protobuf/timestamp.rb +0 -113
  31. data/lib/google/cloud/dataproc/v1/doc/google/rpc/status.rb +0 -39
  32. data/lib/google/cloud/dataproc/v1/job_controller_client.rb +0 -592
  33. data/lib/google/cloud/dataproc/v1/job_controller_client_config.json +0 -59
  34. data/lib/google/cloud/dataproc/v1/jobs_pb.rb +0 -273
  35. data/lib/google/cloud/dataproc/v1/jobs_services_pb.rb +0 -61
  36. data/lib/google/cloud/dataproc/v1/operations_pb.rb +0 -45
  37. data/lib/google/cloud/dataproc/v1/shared_pb.rb +0 -26
  38. data/lib/google/cloud/dataproc/v1/workflow_template_service_client.rb +0 -770
  39. data/lib/google/cloud/dataproc/v1/workflow_template_service_client_config.json +0 -64
  40. data/lib/google/cloud/dataproc/v1/workflow_templates_pb.rb +0 -184
  41. data/lib/google/cloud/dataproc/v1/workflow_templates_services_pb.rb +0 -105
  42. data/lib/google/cloud/dataproc/v1beta2.rb +0 -371
  43. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policies_pb.rb +0 -80
  44. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policies_services_pb.rb +0 -59
  45. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service_client.rb +0 -494
  46. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service_client_config.json +0 -51
  47. data/lib/google/cloud/dataproc/v1beta2/cluster_controller_client.rb +0 -833
  48. data/lib/google/cloud/dataproc/v1beta2/cluster_controller_client_config.json +0 -59
  49. data/lib/google/cloud/dataproc/v1beta2/clusters_pb.rb +0 -241
  50. data/lib/google/cloud/dataproc/v1beta2/clusters_services_pb.rb +0 -69
  51. data/lib/google/cloud/dataproc/v1beta2/credentials.rb +0 -41
  52. data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/autoscaling_policies.rb +0 -238
  53. data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/clusters.rb +0 -841
  54. data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/jobs.rb +0 -728
  55. data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/workflow_templates.rb +0 -579
  56. data/lib/google/cloud/dataproc/v1beta2/doc/google/longrunning/operations.rb +0 -51
  57. data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/any.rb +0 -131
  58. data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/duration.rb +0 -91
  59. data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/empty.rb +0 -29
  60. data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/field_mask.rb +0 -222
  61. data/lib/google/cloud/dataproc/v1beta2/doc/google/protobuf/timestamp.rb +0 -113
  62. data/lib/google/cloud/dataproc/v1beta2/doc/google/rpc/status.rb +0 -39
  63. data/lib/google/cloud/dataproc/v1beta2/job_controller_client.rb +0 -592
  64. data/lib/google/cloud/dataproc/v1beta2/job_controller_client_config.json +0 -59
  65. data/lib/google/cloud/dataproc/v1beta2/jobs_pb.rb +0 -261
  66. data/lib/google/cloud/dataproc/v1beta2/jobs_services_pb.rb +0 -61
  67. data/lib/google/cloud/dataproc/v1beta2/operations_pb.rb +0 -44
  68. data/lib/google/cloud/dataproc/v1beta2/shared_pb.rb +0 -30
  69. data/lib/google/cloud/dataproc/v1beta2/workflow_template_service_client.rb +0 -778
  70. data/lib/google/cloud/dataproc/v1beta2/workflow_template_service_client_config.json +0 -64
  71. data/lib/google/cloud/dataproc/v1beta2/workflow_templates_pb.rb +0 -186
  72. data/lib/google/cloud/dataproc/v1beta2/workflow_templates_services_pb.rb +0 -105
@@ -1,841 +0,0 @@
1
- # Copyright 2020 Google LLC
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # https://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
-
15
-
16
- module Google
17
- module Cloud
18
- module Dataproc
19
- module V1beta2
20
- # Describes the identifying information, config, and status of
21
- # a cluster of Compute Engine instances.
22
- # @!attribute [rw] project_id
23
- # @return [String]
24
- # Required. The Google Cloud Platform project ID that the cluster belongs to.
25
- # @!attribute [rw] cluster_name
26
- # @return [String]
27
- # Required. The cluster name. Cluster names within a project must be
28
- # unique. Names of deleted clusters can be reused.
29
- # @!attribute [rw] config
30
- # @return [Google::Cloud::Dataproc::V1beta2::ClusterConfig]
31
- # Required. The cluster config. Note that Dataproc may set
32
- # default values, and values may change when clusters are updated.
33
- # @!attribute [rw] labels
34
- # @return [Hash{String => String}]
35
- # Optional. The labels to associate with this cluster.
36
- # Label **keys** must contain 1 to 63 characters, and must conform to
37
- # [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
38
- # Label **values** may be empty, but, if present, must contain 1 to 63
39
- # characters, and must conform to [RFC
40
- # 1035](https://www.ietf.org/rfc/rfc1035.txt). No more than 32 labels can be
41
- # associated with a cluster.
42
- # @!attribute [rw] status
43
- # @return [Google::Cloud::Dataproc::V1beta2::ClusterStatus]
44
- # Output only. Cluster status.
45
- # @!attribute [rw] status_history
46
- # @return [Array<Google::Cloud::Dataproc::V1beta2::ClusterStatus>]
47
- # Output only. The previous cluster status.
48
- # @!attribute [rw] cluster_uuid
49
- # @return [String]
50
- # Output only. A cluster UUID (Unique Universal Identifier). Dataproc
51
- # generates this value when it creates the cluster.
52
- # @!attribute [rw] metrics
53
- # @return [Google::Cloud::Dataproc::V1beta2::ClusterMetrics]
54
- # Output only. Contains cluster daemon metrics such as HDFS and YARN stats.
55
- #
56
- # **Beta Feature**: This report is available for testing purposes only. It
57
- # may be changed before final release.
58
- class Cluster; end
59
-
60
- # The cluster config.
61
- # @!attribute [rw] config_bucket
62
- # @return [String]
63
- # Optional. A Cloud Storage bucket used to stage job
64
- # dependencies, config files, and job driver console output.
65
- # If you do not specify a staging bucket, Cloud
66
- # Dataproc will determine a Cloud Storage location (US,
67
- # ASIA, or EU) for your cluster's staging bucket according to the
68
- # Compute Engine zone where your cluster is deployed, and then create
69
- # and manage this project-level, per-location bucket (see
70
- # [Dataproc staging
71
- # bucket](/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
72
- # @!attribute [rw] gce_cluster_config
73
- # @return [Google::Cloud::Dataproc::V1beta2::GceClusterConfig]
74
- # Optional. The shared Compute Engine config settings for
75
- # all instances in a cluster.
76
- # @!attribute [rw] master_config
77
- # @return [Google::Cloud::Dataproc::V1beta2::InstanceGroupConfig]
78
- # Optional. The Compute Engine config settings for
79
- # the master instance in a cluster.
80
- # @!attribute [rw] worker_config
81
- # @return [Google::Cloud::Dataproc::V1beta2::InstanceGroupConfig]
82
- # Optional. The Compute Engine config settings for
83
- # worker instances in a cluster.
84
- # @!attribute [rw] secondary_worker_config
85
- # @return [Google::Cloud::Dataproc::V1beta2::InstanceGroupConfig]
86
- # Optional. The Compute Engine config settings for
87
- # additional worker instances in a cluster.
88
- # @!attribute [rw] software_config
89
- # @return [Google::Cloud::Dataproc::V1beta2::SoftwareConfig]
90
- # Optional. The config settings for software inside the cluster.
91
- # @!attribute [rw] lifecycle_config
92
- # @return [Google::Cloud::Dataproc::V1beta2::LifecycleConfig]
93
- # Optional. The config setting for auto delete cluster schedule.
94
- # @!attribute [rw] initialization_actions
95
- # @return [Array<Google::Cloud::Dataproc::V1beta2::NodeInitializationAction>]
96
- # Optional. Commands to execute on each node after config is
97
- # completed. By default, executables are run on master and all worker nodes.
98
- # You can test a node's <code>role</code> metadata to run an executable on
99
- # a master or worker node, as shown below using `curl` (you can also use
100
- # `wget`):
101
- #
102
- # ROLE=$(curl -H Metadata-Flavor:Google
103
- # http://metadata/computeMetadata/v1beta2/instance/attributes/dataproc-role)
104
- # if [[ "${ROLE}" == 'Master' ]]; then
105
- # ... master specific actions ...
106
- # else
107
- # ... worker specific actions ...
108
- # fi
109
- # @!attribute [rw] encryption_config
110
- # @return [Google::Cloud::Dataproc::V1beta2::EncryptionConfig]
111
- # Optional. Encryption settings for the cluster.
112
- # @!attribute [rw] autoscaling_config
113
- # @return [Google::Cloud::Dataproc::V1beta2::AutoscalingConfig]
114
- # Optional. Autoscaling config for the policy associated with the cluster.
115
- # Cluster does not autoscale if this field is unset.
116
- # @!attribute [rw] endpoint_config
117
- # @return [Google::Cloud::Dataproc::V1beta2::EndpointConfig]
118
- # Optional. Port/endpoint configuration for this cluster
119
- # @!attribute [rw] security_config
120
- # @return [Google::Cloud::Dataproc::V1beta2::SecurityConfig]
121
- # Optional. Security related configuration.
122
- class ClusterConfig; end
123
-
124
- # Endpoint config for this cluster
125
- # @!attribute [rw] http_ports
126
- # @return [Hash{String => String}]
127
- # Output only. The map of port descriptions to URLs. Will only be populated
128
- # if enable_http_port_access is true.
129
- # @!attribute [rw] enable_http_port_access
130
- # @return [true, false]
131
- # Optional. If true, enable http access to specific ports on the cluster
132
- # from external sources. Defaults to false.
133
- class EndpointConfig; end
134
-
135
- # Autoscaling Policy config associated with the cluster.
136
- # @!attribute [rw] policy_uri
137
- # @return [String]
138
- # Optional. The autoscaling policy used by the cluster.
139
- #
140
- # Only resource names including projectid and location (region) are valid.
141
- # Examples:
142
- #
143
- # * `https://www.googleapis.com/compute/v1/projects/[project_id]/locations/[dataproc_region]/autoscalingPolicies/[policy_id]`
144
- # * `projects/[project_id]/locations/[dataproc_region]/autoscalingPolicies/[policy_id]`
145
- #
146
- # Note that the policy must be in the same project and Dataproc region.
147
- class AutoscalingConfig; end
148
-
149
- # Encryption settings for the cluster.
150
- # @!attribute [rw] gce_pd_kms_key_name
151
- # @return [String]
152
- # Optional. The Cloud KMS key name to use for PD disk encryption for all
153
- # instances in the cluster.
154
- class EncryptionConfig; end
155
-
156
- # Common config settings for resources of Compute Engine cluster
157
- # instances, applicable to all instances in the cluster.
158
- # @!attribute [rw] zone_uri
159
- # @return [String]
160
- # Optional. The zone where the Compute Engine cluster will be located.
161
- # On a create request, it is required in the "global" region. If omitted
162
- # in a non-global Dataproc region, the service will pick a zone in the
163
- # corresponding Compute Engine region. On a get request, zone will always be
164
- # present.
165
- #
166
- # A full URL, partial URI, or short name are valid. Examples:
167
- #
168
- # * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/[zone]`
169
- # * `projects/[project_id]/zones/[zone]`
170
- # * `us-central1-f`
171
- # @!attribute [rw] network_uri
172
- # @return [String]
173
- # Optional. The Compute Engine network to be used for machine
174
- # communications. Cannot be specified with subnetwork_uri. If neither
175
- # `network_uri` nor `subnetwork_uri` is specified, the "default" network of
176
- # the project is used, if it exists. Cannot be a "Custom Subnet Network" (see
177
- # [Using Subnetworks](https://cloud.google.com/compute/docs/subnetworks) for more information).
178
- #
179
- # A full URL, partial URI, or short name are valid. Examples:
180
- #
181
- # * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/global/default`
182
- # * `projects/[project_id]/regions/global/default`
183
- # * `default`
184
- # @!attribute [rw] subnetwork_uri
185
- # @return [String]
186
- # Optional. The Compute Engine subnetwork to be used for machine
187
- # communications. Cannot be specified with network_uri.
188
- #
189
- # A full URL, partial URI, or short name are valid. Examples:
190
- #
191
- # * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/us-east1/subnetworks/sub0`
192
- # * `projects/[project_id]/regions/us-east1/subnetworks/sub0`
193
- # * `sub0`
194
- # @!attribute [rw] internal_ip_only
195
- # @return [true, false]
196
- # Optional. If true, all instances in the cluster will only have internal IP
197
- # addresses. By default, clusters are not restricted to internal IP
198
- # addresses, and will have ephemeral external IP addresses assigned to each
199
- # instance. This `internal_ip_only` restriction can only be enabled for
200
- # subnetwork enabled networks, and all off-cluster dependencies must be
201
- # configured to be accessible without external IP addresses.
202
- # @!attribute [rw] service_account
203
- # @return [String]
204
- # Optional. The [Dataproc service
205
- # account](/dataproc/docs/concepts/configuring-clusters/service-accounts#service_accounts_in_cloud_dataproc)
206
- # (also see [VM Data Plane
207
- # identity](/dataproc/docs/concepts/iam/dataproc-principals#vm_service_account_data_plane_identity))
208
- # used by Dataproc cluster VM instances to access Google Cloud Platform
209
- # services.
210
- #
211
- # If not specified, the
212
- # [Compute Engine default service
213
- # account](/compute/docs/access/service-accounts#default_service_account)
214
- # is used.
215
- # @!attribute [rw] service_account_scopes
216
- # @return [Array<String>]
217
- # Optional. The URIs of service account scopes to be included in
218
- # Compute Engine instances. The following base set of scopes is always
219
- # included:
220
- #
221
- # * https://www.googleapis.com/auth/cloud.useraccounts.readonly
222
- # * https://www.googleapis.com/auth/devstorage.read_write
223
- # * https://www.googleapis.com/auth/logging.write
224
- #
225
- # If no scopes are specified, the following defaults are also provided:
226
- #
227
- # * https://www.googleapis.com/auth/bigquery
228
- # * https://www.googleapis.com/auth/bigtable.admin.table
229
- # * https://www.googleapis.com/auth/bigtable.data
230
- # * https://www.googleapis.com/auth/devstorage.full_control
231
- # @!attribute [rw] tags
232
- # @return [Array<String>]
233
- # The Compute Engine tags to add to all instances (see
234
- # [Tagging instances](https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).
235
- # @!attribute [rw] metadata
236
- # @return [Hash{String => String}]
237
- # The Compute Engine metadata entries to add to all instances (see
238
- # [Project and instance
239
- # metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).
240
- # @!attribute [rw] reservation_affinity
241
- # @return [Google::Cloud::Dataproc::V1beta2::ReservationAffinity]
242
- # Optional. Reservation Affinity for consuming Zonal reservation.
243
- class GceClusterConfig; end
244
-
245
- # The config settings for Compute Engine resources in
246
- # an instance group, such as a master or worker group.
247
- # @!attribute [rw] num_instances
248
- # @return [Integer]
249
- # Optional. The number of VM instances in the instance group.
250
- # For master instance groups, must be set to 1.
251
- # @!attribute [rw] instance_names
252
- # @return [Array<String>]
253
- # Output only. The list of instance names. Dataproc derives the names
254
- # from `cluster_name`, `num_instances`, and the instance group.
255
- # @!attribute [rw] image_uri
256
- # @return [String]
257
- # Optional. The Compute Engine image resource used for cluster
258
- # instances. It can be specified or may be inferred from
259
- # `SoftwareConfig.image_version`.
260
- # @!attribute [rw] machine_type_uri
261
- # @return [String]
262
- # Optional. The Compute Engine machine type used for cluster instances.
263
- #
264
- # A full URL, partial URI, or short name are valid. Examples:
265
- #
266
- # * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2`
267
- # * `projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2`
268
- # * `n1-standard-2`
269
- #
270
- # **Auto Zone Exception**: If you are using the Dataproc
271
- # [Auto Zone
272
- # Placement](/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
273
- # feature, you must use the short name of the machine type
274
- # resource, for example, `n1-standard-2`.
275
- # @!attribute [rw] disk_config
276
- # @return [Google::Cloud::Dataproc::V1beta2::DiskConfig]
277
- # Optional. Disk option config settings.
278
- # @!attribute [rw] is_preemptible
279
- # @return [true, false]
280
- # Optional. Specifies that this instance group contains preemptible
281
- # instances.
282
- # @!attribute [rw] managed_group_config
283
- # @return [Google::Cloud::Dataproc::V1beta2::ManagedGroupConfig]
284
- # Output only. The config for Compute Engine Instance Group
285
- # Manager that manages this group.
286
- # This is only used for preemptible instance groups.
287
- # @!attribute [rw] accelerators
288
- # @return [Array<Google::Cloud::Dataproc::V1beta2::AcceleratorConfig>]
289
- # Optional. The Compute Engine accelerator configuration for these
290
- # instances.
291
- # @!attribute [rw] min_cpu_platform
292
- # @return [String]
293
- # Specifies the minimum cpu platform for the Instance Group.
294
- # See [Dataproc&rarr;Minimum CPU Platform]
295
- # (/dataproc/docs/concepts/compute/dataproc-min-cpu).
296
- class InstanceGroupConfig; end
297
-
298
- # Specifies the resources used to actively manage an instance group.
299
- # @!attribute [rw] instance_template_name
300
- # @return [String]
301
- # Output only. The name of the Instance Template used for the Managed
302
- # Instance Group.
303
- # @!attribute [rw] instance_group_manager_name
304
- # @return [String]
305
- # Output only. The name of the Instance Group Manager for this group.
306
- class ManagedGroupConfig; end
307
-
308
- # Specifies the type and number of accelerator cards attached to the instances
309
- # of an instance group (see [GPUs on Compute Engine](https://cloud.google.com/compute/docs/gpus/)).
310
- # @!attribute [rw] accelerator_type_uri
311
- # @return [String]
312
- # Full URL, partial URI, or short name of the accelerator type resource to
313
- # expose to this instance. See
314
- # [Compute Engine
315
- # AcceleratorTypes](/compute/docs/reference/beta/acceleratorTypes)
316
- #
317
- # Examples
318
- # * `https://www.googleapis.com/compute/beta/projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80`
319
- # * `projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80`
320
- # * `nvidia-tesla-k80`
321
- #
322
- # **Auto Zone Exception**: If you are using the Dataproc
323
- # [Auto Zone
324
- # Placement](/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
325
- # feature, you must use the short name of the accelerator type
326
- # resource, for example, `nvidia-tesla-k80`.
327
- # @!attribute [rw] accelerator_count
328
- # @return [Integer]
329
- # The number of the accelerator cards of this type exposed to this instance.
330
- class AcceleratorConfig; end
331
-
332
- # Specifies the config of disk options for a group of VM instances.
333
- # @!attribute [rw] boot_disk_type
334
- # @return [String]
335
- # Optional. Type of the boot disk (default is "pd-standard").
336
- # Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or
337
- # "pd-standard" (Persistent Disk Hard Disk Drive).
338
- # @!attribute [rw] boot_disk_size_gb
339
- # @return [Integer]
340
- # Optional. Size in GB of the boot disk (default is 500GB).
341
- # @!attribute [rw] num_local_ssds
342
- # @return [Integer]
343
- # Number of attached SSDs, from 0 to 4 (default is 0).
344
- # If SSDs are not attached, the boot disk is used to store runtime logs and
345
- # [HDFS](https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data.
346
- # If one or more SSDs are attached, this runtime bulk
347
- # data is spread across them, and the boot disk contains only basic
348
- # config and installed binaries.
349
- class DiskConfig; end
350
-
351
- # Specifies the cluster auto-delete schedule configuration.
352
- # @!attribute [rw] idle_delete_ttl
353
- # @return [Google::Protobuf::Duration]
354
- # Optional. The duration to keep the cluster alive while idling (when no jobs
355
- # are running). Passing this threshold will cause the cluster to be
356
- # deleted. Minimum value is 10 minutes; maximum value is 14 days (see JSON
357
- # representation of
358
- # [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json).
359
- # @!attribute [rw] auto_delete_time
360
- # @return [Google::Protobuf::Timestamp]
361
- # Optional. The time when cluster will be auto-deleted. (see JSON representation of
362
- # [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
363
- # @!attribute [rw] auto_delete_ttl
364
- # @return [Google::Protobuf::Duration]
365
- # Optional. The lifetime duration of cluster. The cluster will be
366
- # auto-deleted at the end of this period. Minimum value is 10 minutes;
367
- # maximum value is 14 days (see JSON representation of
368
- # [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
369
- # @!attribute [rw] idle_start_time
370
- # @return [Google::Protobuf::Timestamp]
371
- # Output only. The time when cluster became idle (most recent job finished)
372
- # and became eligible for deletion due to idleness (see JSON representation
373
- # of
374
- # [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
375
- class LifecycleConfig; end
376
-
377
- # Security related configuration, including encryption, Kerberos, etc.
378
- # @!attribute [rw] kerberos_config
379
- # @return [Google::Cloud::Dataproc::V1beta2::KerberosConfig]
380
- # Kerberos related configuration.
381
- class SecurityConfig; end
382
-
383
- # Specifies Kerberos related configuration.
384
- # @!attribute [rw] enable_kerberos
385
- # @return [true, false]
386
- # Optional. Flag to indicate whether to Kerberize the cluster.
387
- # @!attribute [rw] root_principal_password_uri
388
- # @return [String]
389
- # Required. The Cloud Storage URI of a KMS encrypted file containing the root
390
- # principal password.
391
- # @!attribute [rw] kms_key_uri
392
- # @return [String]
393
- # Required. The uri of the KMS key used to encrypt various sensitive
394
- # files.
395
- # @!attribute [rw] keystore_uri
396
- # @return [String]
397
- # Optional. The Cloud Storage URI of the keystore file used for SSL
398
- # encryption. If not provided, Dataproc will provide a self-signed
399
- # certificate.
400
- # @!attribute [rw] truststore_uri
401
- # @return [String]
402
- # Optional. The Cloud Storage URI of the truststore file used for SSL
403
- # encryption. If not provided, Dataproc will provide a self-signed
404
- # certificate.
405
- # @!attribute [rw] keystore_password_uri
406
- # @return [String]
407
- # Optional. The Cloud Storage URI of a KMS encrypted file containing the
408
- # password to the user provided keystore. For the self-signed certificate,
409
- # this password is generated by Dataproc.
410
- # @!attribute [rw] key_password_uri
411
- # @return [String]
412
- # Optional. The Cloud Storage URI of a KMS encrypted file containing the
413
- # password to the user provided key. For the self-signed certificate, this
414
- # password is generated by Dataproc.
415
- # @!attribute [rw] truststore_password_uri
416
- # @return [String]
417
- # Optional. The Cloud Storage URI of a KMS encrypted file containing the
418
- # password to the user provided truststore. For the self-signed certificate,
419
- # this password is generated by Dataproc.
420
- # @!attribute [rw] cross_realm_trust_realm
421
- # @return [String]
422
- # Optional. The remote realm the Dataproc on-cluster KDC will trust, should
423
- # the user enable cross realm trust.
424
- # @!attribute [rw] cross_realm_trust_kdc
425
- # @return [String]
426
- # Optional. The KDC (IP or hostname) for the remote trusted realm in a cross
427
- # realm trust relationship.
428
- # @!attribute [rw] cross_realm_trust_admin_server
429
- # @return [String]
430
- # Optional. The admin server (IP or hostname) for the remote trusted realm in
431
- # a cross realm trust relationship.
432
- # @!attribute [rw] cross_realm_trust_shared_password_uri
433
- # @return [String]
434
- # Optional. The Cloud Storage URI of a KMS encrypted file containing the
435
- # shared password between the on-cluster Kerberos realm and the remote
436
- # trusted realm, in a cross realm trust relationship.
437
- # @!attribute [rw] kdc_db_key_uri
438
- # @return [String]
439
- # Optional. The Cloud Storage URI of a KMS encrypted file containing the
440
- # master key of the KDC database.
441
- # @!attribute [rw] tgt_lifetime_hours
442
- # @return [Integer]
443
- # Optional. The lifetime of the ticket granting ticket, in hours.
444
- # If not specified, or user specifies 0, then default value 10
445
- # will be used.
446
- # @!attribute [rw] realm
447
- # @return [String]
448
- # Optional. The name of the on-cluster Kerberos realm.
449
- # If not specified, the uppercased domain of hostnames will be the realm.
450
- class KerberosConfig; end
451
-
452
- # Specifies an executable to run on a fully configured node and a
453
- # timeout period for executable completion.
454
- # @!attribute [rw] executable_file
455
- # @return [String]
456
- # Required. Cloud Storage URI of executable file.
457
- # @!attribute [rw] execution_timeout
458
- # @return [Google::Protobuf::Duration]
459
- # Optional. Amount of time executable has to complete. Default is
460
- # 10 minutes (see JSON representation of
461
- # [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
462
- #
463
- # Cluster creation fails with an explanatory error message (the
464
- # name of the executable that caused the error and the exceeded timeout
465
- # period) if the executable is not completed at end of the timeout period.
466
- class NodeInitializationAction; end
467
-
468
- # The status of a cluster and its instances.
469
- # @!attribute [rw] state
470
- # @return [Google::Cloud::Dataproc::V1beta2::ClusterStatus::State]
471
- # Output only. The cluster's state.
472
- # @!attribute [rw] detail
473
- # @return [String]
474
- # Output only. Optional details of cluster's state.
475
- # @!attribute [rw] state_start_time
476
- # @return [Google::Protobuf::Timestamp]
477
- # Output only. Time when this state was entered (see JSON representation of
478
- # [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
479
- # @!attribute [rw] substate
480
- # @return [Google::Cloud::Dataproc::V1beta2::ClusterStatus::Substate]
481
- # Output only. Additional state information that includes
482
- # status reported by the agent.
483
- class ClusterStatus
484
- # The cluster state.
485
- module State
486
- # The cluster state is unknown.
487
- UNKNOWN = 0
488
-
489
- # The cluster is being created and set up. It is not ready for use.
490
- CREATING = 1
491
-
492
- # The cluster is currently running and healthy. It is ready for use.
493
- RUNNING = 2
494
-
495
- # The cluster encountered an error. It is not ready for use.
496
- ERROR = 3
497
-
498
- # The cluster is being deleted. It cannot be used.
499
- DELETING = 4
500
-
501
- # The cluster is being updated. It continues to accept and process jobs.
502
- UPDATING = 5
503
- end
504
-
505
- # The cluster substate.
506
- module Substate
507
- # The cluster substate is unknown.
508
- UNSPECIFIED = 0
509
-
510
- # The cluster is known to be in an unhealthy state
511
- # (for example, critical daemons are not running or HDFS capacity is
512
- # exhausted).
513
- #
514
- # Applies to RUNNING state.
515
- UNHEALTHY = 1
516
-
517
- # The agent-reported status is out of date (may occur if
518
- # Dataproc loses communication with Agent).
519
- #
520
- # Applies to RUNNING state.
521
- STALE_STATUS = 2
522
- end
523
- end
524
-
525
- # Specifies the selection and config of software inside the cluster.
526
- # @!attribute [rw] image_version
527
- # @return [String]
528
- # Optional. The version of software inside the cluster. It must be one of the
529
- # supported [Dataproc
530
- # Versions](/dataproc/docs/concepts/versioning/dataproc-versions#supported_cloud_dataproc_versions),
531
- # such as "1.2" (including a subminor version, such as "1.2.29"), or the
532
- # ["preview"
533
- # version](/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
534
- # If unspecified, it defaults to the latest Debian version.
535
- # @!attribute [rw] properties
536
- # @return [Hash{String => String}]
537
- # Optional. The properties to set on daemon config files.
538
- #
539
- # Property keys are specified in `prefix:property` format, for example
540
- # `core:hadoop.tmp.dir`. The following are supported prefixes
541
- # and their mappings:
542
- #
543
- # * capacity-scheduler: `capacity-scheduler.xml`
544
- # * core: `core-site.xml`
545
- # * distcp: `distcp-default.xml`
546
- # * hdfs: `hdfs-site.xml`
547
- # * hive: `hive-site.xml`
548
- # * mapred: `mapred-site.xml`
549
- # * pig: `pig.properties`
550
- # * spark: `spark-defaults.conf`
551
- # * yarn: `yarn-site.xml`
552
- #
553
- # For more information, see
554
- # [Cluster properties](https://cloud.google.com/dataproc/docs/concepts/cluster-properties).
555
- # @!attribute [rw] optional_components
556
- # @return [Array<Google::Cloud::Dataproc::V1beta2::Component>]
557
- # The set of optional components to activate on the cluster.
558
- class SoftwareConfig; end
559
-
560
- # Contains cluster daemon metrics, such as HDFS and YARN stats.
561
- #
562
- # **Beta Feature**: This report is available for testing purposes only. It may
563
- # be changed before final release.
564
- # @!attribute [rw] hdfs_metrics
565
- # @return [Hash{String => Integer}]
566
- # The HDFS metrics.
567
- # @!attribute [rw] yarn_metrics
568
- # @return [Hash{String => Integer}]
569
- # The YARN metrics.
570
- class ClusterMetrics; end
571
-
572
- # A request to create a cluster.
573
- # @!attribute [rw] project_id
574
- # @return [String]
575
- # Required. The ID of the Google Cloud Platform project that the cluster
576
- # belongs to.
577
- # @!attribute [rw] region
578
- # @return [String]
579
- # Required. The Dataproc region in which to handle the request.
580
- # @!attribute [rw] cluster
581
- # @return [Google::Cloud::Dataproc::V1beta2::Cluster]
582
- # Required. The cluster to create.
583
- # @!attribute [rw] request_id
584
- # @return [String]
585
- # Optional. A unique id used to identify the request. If the server
586
- # receives two {Google::Cloud::Dataproc::V1beta2::CreateClusterRequest CreateClusterRequest} requests with the same
587
- # id, then the second request will be ignored and the
588
- # first {Google::Longrunning::Operation} created and stored in the backend
589
- # is returned.
590
- #
591
- # It is recommended to always set this value to a
592
- # [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
593
- #
594
- # The id must contain only letters (a-z, A-Z), numbers (0-9),
595
- # underscores (_), and hyphens (-). The maximum length is 40 characters.
596
- class CreateClusterRequest; end
597
-
598
- # A request to update a cluster.
599
- # @!attribute [rw] project_id
600
- # @return [String]
601
- # Required. The ID of the Google Cloud Platform project the
602
- # cluster belongs to.
603
- # @!attribute [rw] region
604
- # @return [String]
605
- # Required. The Dataproc region in which to handle the request.
606
- # @!attribute [rw] cluster_name
607
- # @return [String]
608
- # Required. The cluster name.
609
- # @!attribute [rw] cluster
610
- # @return [Google::Cloud::Dataproc::V1beta2::Cluster]
611
- # Required. The changes to the cluster.
612
- # @!attribute [rw] graceful_decommission_timeout
613
- # @return [Google::Protobuf::Duration]
614
- # Optional. Timeout for graceful YARN decomissioning. Graceful
615
- # decommissioning allows removing nodes from the cluster without
616
- # interrupting jobs in progress. Timeout specifies how long to wait for jobs
617
- # in progress to finish before forcefully removing nodes (and potentially
618
- # interrupting jobs). Default timeout is 0 (for forceful decommission), and
619
- # the maximum allowed timeout is 1 day (see JSON representation of
620
- # [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
621
- #
622
- # Only supported on Dataproc image versions 1.2 and higher.
623
- # @!attribute [rw] update_mask
624
- # @return [Google::Protobuf::FieldMask]
625
- # Required. Specifies the path, relative to `Cluster`, of
626
- # the field to update. For example, to change the number of workers
627
- # in a cluster to 5, the `update_mask` parameter would be
628
- # specified as `config.worker_config.num_instances`,
629
- # and the `PATCH` request body would specify the new value, as follows:
630
- #
631
- # {
632
- # "config":{
633
- # "workerConfig":{
634
- # "numInstances":"5"
635
- # }
636
- # }
637
- # }
638
- #
639
- # Similarly, to change the number of preemptible workers in a cluster to 5,
640
- # the `update_mask` parameter would be
641
- # `config.secondary_worker_config.num_instances`, and the `PATCH` request
642
- # body would be set as follows:
643
- #
644
- # {
645
- # "config":{
646
- # "secondaryWorkerConfig":{
647
- # "numInstances":"5"
648
- # }
649
- # }
650
- # }
651
- # <strong>Note:</strong> currently only the following fields can be updated:
652
- #
653
- # <table>
654
- # <tr>
655
- # <td><strong>Mask</strong></td><td><strong>Purpose</strong></td>
656
- # </tr>
657
- # <tr>
658
- # <td>labels</td><td>Updates labels</td>
659
- # </tr>
660
- # <tr>
661
- # <td>config.worker_config.num_instances</td><td>Resize primary worker
662
- # group</td>
663
- # </tr>
664
- # <tr>
665
- # <td>config.secondary_worker_config.num_instances</td><td>Resize secondary
666
- # worker group</td>
667
- # </tr>
668
- # <tr>
669
- # <td>config.lifecycle_config.auto_delete_ttl</td><td>Reset MAX TTL
670
- # duration</td>
671
- # </tr>
672
- # <tr>
673
- # <td>config.lifecycle_config.auto_delete_time</td><td>Update MAX TTL
674
- # deletion timestamp</td>
675
- # </tr>
676
- # <tr>
677
- # <td>config.lifecycle_config.idle_delete_ttl</td><td>Update Idle TTL
678
- # duration</td>
679
- # </tr>
680
- # <tr>
681
- # <td>config.autoscaling_config.policy_uri</td><td>Use, stop using, or change
682
- # autoscaling policies</td>
683
- # </tr>
684
- # </table>
685
- # @!attribute [rw] request_id
686
- # @return [String]
687
- # Optional. A unique id used to identify the request. If the server
688
- # receives two {Google::Cloud::Dataproc::V1beta2::UpdateClusterRequest UpdateClusterRequest} requests with the same
689
- # id, then the second request will be ignored and the
690
- # first {Google::Longrunning::Operation} created and stored in the
691
- # backend is returned.
692
- #
693
- # It is recommended to always set this value to a
694
- # [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
695
- #
696
- # The id must contain only letters (a-z, A-Z), numbers (0-9),
697
- # underscores (_), and hyphens (-). The maximum length is 40 characters.
698
- class UpdateClusterRequest; end
699
-
700
- # A request to delete a cluster.
701
- # @!attribute [rw] project_id
702
- # @return [String]
703
- # Required. The ID of the Google Cloud Platform project that the cluster
704
- # belongs to.
705
- # @!attribute [rw] region
706
- # @return [String]
707
- # Required. The Dataproc region in which to handle the request.
708
- # @!attribute [rw] cluster_name
709
- # @return [String]
710
- # Required. The cluster name.
711
- # @!attribute [rw] cluster_uuid
712
- # @return [String]
713
- # Optional. Specifying the `cluster_uuid` means the RPC should fail
714
- # (with error NOT_FOUND) if cluster with specified UUID does not exist.
715
- # @!attribute [rw] request_id
716
- # @return [String]
717
- # Optional. A unique id used to identify the request. If the server
718
- # receives two {Google::Cloud::Dataproc::V1beta2::DeleteClusterRequest DeleteClusterRequest} requests with the same
719
- # id, then the second request will be ignored and the
720
- # first {Google::Longrunning::Operation} created and stored in the
721
- # backend is returned.
722
- #
723
- # It is recommended to always set this value to a
724
- # [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
725
- #
726
- # The id must contain only letters (a-z, A-Z), numbers (0-9),
727
- # underscores (_), and hyphens (-). The maximum length is 40 characters.
728
- class DeleteClusterRequest; end
729
-
730
- # Request to get the resource representation for a cluster in a project.
731
- # @!attribute [rw] project_id
732
- # @return [String]
733
- # Required. The ID of the Google Cloud Platform project that the cluster
734
- # belongs to.
735
- # @!attribute [rw] region
736
- # @return [String]
737
- # Required. The Dataproc region in which to handle the request.
738
- # @!attribute [rw] cluster_name
739
- # @return [String]
740
- # Required. The cluster name.
741
- class GetClusterRequest; end
742
-
743
- # A request to list the clusters in a project.
744
- # @!attribute [rw] project_id
745
- # @return [String]
746
- # Required. The ID of the Google Cloud Platform project that the cluster
747
- # belongs to.
748
- # @!attribute [rw] region
749
- # @return [String]
750
- # Required. The Dataproc region in which to handle the request.
751
- # @!attribute [rw] filter
752
- # @return [String]
753
- # Optional. A filter constraining the clusters to list. Filters are
754
- # case-sensitive and have the following syntax:
755
- #
756
- # field = value [AND [field = value]] ...
757
- #
758
- # where **field** is one of `status.state`, `clusterName`, or `labels.[KEY]`,
759
- # and `[KEY]` is a label key. **value** can be `*` to match all values.
760
- # `status.state` can be one of the following: `ACTIVE`, `INACTIVE`,
761
- # `CREATING`, `RUNNING`, `ERROR`, `DELETING`, or `UPDATING`. `ACTIVE`
762
- # contains the `CREATING`, `UPDATING`, and `RUNNING` states. `INACTIVE`
763
- # contains the `DELETING` and `ERROR` states.
764
- # `clusterName` is the name of the cluster provided at creation time.
765
- # Only the logical `AND` operator is supported; space-separated items are
766
- # treated as having an implicit `AND` operator.
767
- #
768
- # Example filter:
769
- #
770
- # status.state = ACTIVE AND clusterName = mycluster
771
- # AND labels.env = staging AND labels.starred = *
772
- # @!attribute [rw] page_size
773
- # @return [Integer]
774
- # Optional. The standard List page size.
775
- # @!attribute [rw] page_token
776
- # @return [String]
777
- # Optional. The standard List page token.
778
- class ListClustersRequest; end
779
-
780
- # The list of all clusters in a project.
781
- # @!attribute [rw] clusters
782
- # @return [Array<Google::Cloud::Dataproc::V1beta2::Cluster>]
783
- # Output only. The clusters in the project.
784
- # @!attribute [rw] next_page_token
785
- # @return [String]
786
- # Output only. This token is included in the response if there are more
787
- # results to fetch. To fetch additional results, provide this value as the
788
- # `page_token` in a subsequent <code>ListClustersRequest</code>.
789
- class ListClustersResponse; end
790
-
791
- # A request to collect cluster diagnostic information.
792
- # @!attribute [rw] project_id
793
- # @return [String]
794
- # Required. The ID of the Google Cloud Platform project that the cluster
795
- # belongs to.
796
- # @!attribute [rw] region
797
- # @return [String]
798
- # Required. The Dataproc region in which to handle the request.
799
- # @!attribute [rw] cluster_name
800
- # @return [String]
801
- # Required. The cluster name.
802
- class DiagnoseClusterRequest; end
803
-
804
- # The location of diagnostic output.
805
- # @!attribute [rw] output_uri
806
- # @return [String]
807
- # Output only. The Cloud Storage URI of the diagnostic output.
808
- # The output report is a plain text file with a summary of collected
809
- # diagnostics.
810
- class DiagnoseClusterResults; end
811
-
812
- # Reservation Affinity for consuming Zonal reservation.
813
- # @!attribute [rw] consume_reservation_type
814
- # @return [Google::Cloud::Dataproc::V1beta2::ReservationAffinity::Type]
815
- # Optional. Type of reservation to consume
816
- # @!attribute [rw] key
817
- # @return [String]
818
- # Optional. Corresponds to the label key of reservation resource.
819
- # @!attribute [rw] values
820
- # @return [Array<String>]
821
- # Optional. Corresponds to the label values of reservation resource.
822
- class ReservationAffinity
823
- # Indicates whether to consume capacity from an reservation or not.
824
- module Type
825
- TYPE_UNSPECIFIED = 0
826
-
827
- # Do not consume from any allocated capacity.
828
- NO_RESERVATION = 1
829
-
830
- # Consume any reservation available.
831
- ANY_RESERVATION = 2
832
-
833
- # Must consume from a specific reservation. Must specify key value fields
834
- # for specifying the reservations.
835
- SPECIFIC_RESERVATION = 3
836
- end
837
- end
838
- end
839
- end
840
- end
841
- end