google-cloud-dataproc-v1beta2 0.2.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE.md +188 -190
- data/README.md +66 -2
- data/lib/google/cloud/dataproc/v1beta2.rb +3 -0
- data/lib/google/cloud/dataproc/v1beta2/autoscaling_policies_services_pb.rb +5 -5
- data/lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service/client.rb +25 -19
- data/lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service/paths.rb +1 -1
- data/lib/google/cloud/dataproc/v1beta2/cluster_controller.rb +1 -0
- data/lib/google/cloud/dataproc/v1beta2/cluster_controller/client.rb +62 -51
- data/lib/google/cloud/dataproc/v1beta2/cluster_controller/operations.rb +95 -10
- data/lib/google/cloud/dataproc/v1beta2/cluster_controller/paths.rb +52 -0
- data/lib/google/cloud/dataproc/v1beta2/clusters_pb.rb +8 -0
- data/lib/google/cloud/dataproc/v1beta2/clusters_services_pb.rb +6 -6
- data/lib/google/cloud/dataproc/v1beta2/job_controller/client.rb +50 -44
- data/lib/google/cloud/dataproc/v1beta2/job_controller/operations.rb +95 -10
- data/lib/google/cloud/dataproc/v1beta2/jobs_services_pb.rb +7 -7
- data/lib/google/cloud/dataproc/v1beta2/shared_pb.rb +1 -2
- data/lib/google/cloud/dataproc/v1beta2/version.rb +1 -1
- data/lib/google/cloud/dataproc/v1beta2/workflow_template_service/client.rb +41 -34
- data/lib/google/cloud/dataproc/v1beta2/workflow_template_service/operations.rb +95 -10
- data/lib/google/cloud/dataproc/v1beta2/workflow_template_service/paths.rb +20 -1
- data/lib/google/cloud/dataproc/v1beta2/workflow_templates_pb.rb +5 -0
- data/lib/google/cloud/dataproc/v1beta2/workflow_templates_services_pb.rb +9 -8
- data/proto_docs/google/api/field_behavior.rb +6 -0
- data/proto_docs/google/api/resource.rb +50 -14
- data/proto_docs/google/cloud/dataproc/v1beta2/autoscaling_policies.rb +13 -7
- data/proto_docs/google/cloud/dataproc/v1beta2/clusters.rb +64 -21
- data/proto_docs/google/cloud/dataproc/v1beta2/jobs.rb +14 -13
- data/proto_docs/google/cloud/dataproc/v1beta2/shared.rb +3 -6
- data/proto_docs/google/cloud/dataproc/v1beta2/workflow_templates.rb +42 -8
- data/proto_docs/google/longrunning/operations.rb +17 -3
- data/proto_docs/google/protobuf/any.rb +5 -2
- data/proto_docs/google/protobuf/timestamp.rb +10 -1
- metadata +11 -7
@@ -54,6 +54,12 @@ module Google
|
|
54
54
|
# This indicates that the field may be set once in a request to create a
|
55
55
|
# resource, but may not be changed thereafter.
|
56
56
|
IMMUTABLE = 5
|
57
|
+
|
58
|
+
# Denotes that a (repeated) field is an unordered list.
|
59
|
+
# This indicates that the service may provide the elements of the list
|
60
|
+
# in any arbitrary order, rather than the order the user originally
|
61
|
+
# provided. Additionally, the list's order may or may not be stable.
|
62
|
+
UNORDERED_LIST = 6
|
57
63
|
end
|
58
64
|
end
|
59
65
|
end
|
@@ -43,12 +43,12 @@ module Google
|
|
43
43
|
#
|
44
44
|
# The ResourceDescriptor Yaml config will look like:
|
45
45
|
#
|
46
|
-
#
|
47
|
-
#
|
48
|
-
#
|
49
|
-
#
|
50
|
-
#
|
51
|
-
#
|
46
|
+
# resources:
|
47
|
+
# - type: "pubsub.googleapis.com/Topic"
|
48
|
+
# name_descriptor:
|
49
|
+
# - pattern: "projects/{project}/topics/{topic}"
|
50
|
+
# parent_type: "cloudresourcemanager.googleapis.com/Project"
|
51
|
+
# parent_name_extractor: "projects/{project}"
|
52
52
|
#
|
53
53
|
# Sometimes, resources have multiple patterns, typically because they can
|
54
54
|
# live under multiple parents.
|
@@ -183,15 +183,24 @@ module Google
|
|
183
183
|
# }
|
184
184
|
# @!attribute [rw] plural
|
185
185
|
# @return [::String]
|
186
|
-
# The plural name used in the resource name, such as
|
187
|
-
# the name of 'projects/\\{project}'
|
188
|
-
#
|
186
|
+
# The plural name used in the resource name and permission names, such as
|
187
|
+
# 'projects' for the resource name of 'projects/\\{project}' and the permission
|
188
|
+
# name of 'cloudresourcemanager.googleapis.com/projects.get'. It is the same
|
189
|
+
# concept of the `plural` field in k8s CRD spec
|
189
190
|
# https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
|
191
|
+
#
|
192
|
+
# Note: The plural form is required even for singleton resources. See
|
193
|
+
# https://aip.dev/156
|
190
194
|
# @!attribute [rw] singular
|
191
195
|
# @return [::String]
|
192
196
|
# The same concept of the `singular` field in k8s CRD spec
|
193
197
|
# https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
|
194
198
|
# Such as "project" for the `resourcemanager.googleapis.com/Project` type.
|
199
|
+
# @!attribute [rw] style
|
200
|
+
# @return [::Array<::Google::Api::ResourceDescriptor::Style>]
|
201
|
+
# Style flag(s) for this resource.
|
202
|
+
# These indicate that a resource is expected to conform to a given
|
203
|
+
# style. See the specific style flags for additional information.
|
195
204
|
class ResourceDescriptor
|
196
205
|
include ::Google::Protobuf::MessageExts
|
197
206
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -211,6 +220,22 @@ module Google
|
|
211
220
|
# that from being necessary once there are multiple patterns.)
|
212
221
|
FUTURE_MULTI_PATTERN = 2
|
213
222
|
end
|
223
|
+
|
224
|
+
# A flag representing a specific style that a resource claims to conform to.
|
225
|
+
module Style
|
226
|
+
# The unspecified value. Do not use.
|
227
|
+
STYLE_UNSPECIFIED = 0
|
228
|
+
|
229
|
+
# This resource is intended to be "declarative-friendly".
|
230
|
+
#
|
231
|
+
# Declarative-friendly resources must be more strictly consistent, and
|
232
|
+
# setting this to true communicates to tools that this resource should
|
233
|
+
# adhere to declarative-friendly expectations.
|
234
|
+
#
|
235
|
+
# Note: This is used by the API linter (linter.aip.dev) to enable
|
236
|
+
# additional checks.
|
237
|
+
DECLARATIVE_FRIENDLY = 1
|
238
|
+
end
|
214
239
|
end
|
215
240
|
|
216
241
|
# Defines a proto annotation that describes a string field that refers to
|
@@ -226,6 +251,17 @@ module Google
|
|
226
251
|
# type: "pubsub.googleapis.com/Topic"
|
227
252
|
# }];
|
228
253
|
# }
|
254
|
+
#
|
255
|
+
# Occasionally, a field may reference an arbitrary resource. In this case,
|
256
|
+
# APIs use the special value * in their resource reference.
|
257
|
+
#
|
258
|
+
# Example:
|
259
|
+
#
|
260
|
+
# message GetIamPolicyRequest {
|
261
|
+
# string resource = 2 [(google.api.resource_reference) = {
|
262
|
+
# type: "*"
|
263
|
+
# }];
|
264
|
+
# }
|
229
265
|
# @!attribute [rw] child_type
|
230
266
|
# @return [::String]
|
231
267
|
# The resource type of a child collection that the annotated field
|
@@ -234,11 +270,11 @@ module Google
|
|
234
270
|
#
|
235
271
|
# Example:
|
236
272
|
#
|
237
|
-
#
|
238
|
-
#
|
239
|
-
#
|
240
|
-
#
|
241
|
-
#
|
273
|
+
# message ListLogEntriesRequest {
|
274
|
+
# string parent = 1 [(google.api.resource_reference) = {
|
275
|
+
# child_type: "logging.googleapis.com/LogEntry"
|
276
|
+
# };
|
277
|
+
# }
|
242
278
|
class ResourceReference
|
243
279
|
include ::Google::Protobuf::MessageExts
|
244
280
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -80,20 +80,26 @@ module Google
|
|
80
80
|
# Bounds: [0s, 1d].
|
81
81
|
# @!attribute [rw] scale_up_factor
|
82
82
|
# @return [::Float]
|
83
|
-
# Required. Fraction of average pending memory in the last cooldown
|
84
|
-
# for which to add workers. A scale-up factor of 1.0 will result in
|
85
|
-
# up so that there is no pending memory remaining after the update
|
86
|
-
# aggressive scaling). A scale-up factor closer to 0 will result in a
|
87
|
-
# magnitude of scaling up (less aggressive scaling).
|
83
|
+
# Required. Fraction of average YARN pending memory in the last cooldown
|
84
|
+
# period for which to add workers. A scale-up factor of 1.0 will result in
|
85
|
+
# scaling up so that there is no pending memory remaining after the update
|
86
|
+
# (more aggressive scaling). A scale-up factor closer to 0 will result in a
|
87
|
+
# smaller magnitude of scaling up (less aggressive scaling). See [How
|
88
|
+
# autoscaling
|
89
|
+
# works](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/autoscaling#how_autoscaling_works)
|
90
|
+
# for more information.
|
88
91
|
#
|
89
92
|
# Bounds: [0.0, 1.0].
|
90
93
|
# @!attribute [rw] scale_down_factor
|
91
94
|
# @return [::Float]
|
92
|
-
# Required. Fraction of average pending memory in the last cooldown
|
93
|
-
# for which to remove workers. A scale-down factor of 1 will result in
|
95
|
+
# Required. Fraction of average YARN pending memory in the last cooldown
|
96
|
+
# period for which to remove workers. A scale-down factor of 1 will result in
|
94
97
|
# scaling down so that there is no available memory remaining after the
|
95
98
|
# update (more aggressive scaling). A scale-down factor of 0 disables
|
96
99
|
# removing workers, which can be beneficial for autoscaling a single job.
|
100
|
+
# See [How autoscaling
|
101
|
+
# works](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/autoscaling#how_autoscaling_works)
|
102
|
+
# for more information.
|
97
103
|
#
|
98
104
|
# Bounds: [0.0, 1.0].
|
99
105
|
# @!attribute [rw] scale_up_min_worker_fraction
|
@@ -85,6 +85,15 @@ module Google
|
|
85
85
|
# and manage this project-level, per-location bucket (see
|
86
86
|
# [Dataproc staging
|
87
87
|
# bucket](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
|
88
|
+
# @!attribute [rw] temp_bucket
|
89
|
+
# @return [::String]
|
90
|
+
# Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs
|
91
|
+
# data, such as Spark and MapReduce history files. If you do not specify a
|
92
|
+
# temp bucket, Dataproc will determine a Cloud Storage location (US, ASIA, or
|
93
|
+
# EU) for your cluster's temp bucket according to the Compute Engine zone
|
94
|
+
# where your cluster is deployed, and then create and manage this
|
95
|
+
# project-level, per-location bucket. The default bucket has a TTL of 90
|
96
|
+
# days, but you can use any TTL (or none) if you specify a bucket.
|
88
97
|
# @!attribute [rw] gce_cluster_config
|
89
98
|
# @return [::Google::Cloud::Dataproc::V1beta2::GceClusterConfig]
|
90
99
|
# Optional. The shared Compute Engine config settings for
|
@@ -137,10 +146,10 @@ module Google
|
|
137
146
|
# Optional. Security related configuration.
|
138
147
|
# @!attribute [rw] gke_cluster_config
|
139
148
|
# @return [::Google::Cloud::Dataproc::V1beta2::GkeClusterConfig]
|
140
|
-
# Optional. The Kubernetes Engine config for Dataproc clusters deployed to
|
141
|
-
# Setting this is considered mutually exclusive with Compute
|
142
|
-
# options such as `gce_cluster_config`, `master_config`,
|
143
|
-
# `secondary_worker_config`, and `autoscaling_config`.
|
149
|
+
# Optional. The Kubernetes Engine config for Dataproc clusters deployed to
|
150
|
+
# Kubernetes. Setting this is considered mutually exclusive with Compute
|
151
|
+
# Engine-based options such as `gce_cluster_config`, `master_config`,
|
152
|
+
# `worker_config`, `secondary_worker_config`, and `autoscaling_config`.
|
144
153
|
class ClusterConfig
|
145
154
|
include ::Google::Protobuf::MessageExts
|
146
155
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -268,7 +277,7 @@ module Google
|
|
268
277
|
# @!attribute [rw] service_account
|
269
278
|
# @return [::String]
|
270
279
|
# Optional. The [Dataproc service
|
271
|
-
# account](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#
|
280
|
+
# account](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#service_accounts_in_dataproc)
|
272
281
|
# (also see [VM Data Plane
|
273
282
|
# identity](https://cloud.google.com/dataproc/docs/concepts/iam/dataproc-principals#vm_service_account_data_plane_identity))
|
274
283
|
# used by Dataproc cluster VM instances to access Google Cloud Platform
|
@@ -372,6 +381,15 @@ module Google
|
|
372
381
|
# @return [::Boolean]
|
373
382
|
# Output only. Specifies that this instance group contains preemptible
|
374
383
|
# instances.
|
384
|
+
# @!attribute [rw] preemptibility
|
385
|
+
# @return [::Google::Cloud::Dataproc::V1beta2::InstanceGroupConfig::Preemptibility]
|
386
|
+
# Optional. Specifies the preemptibility of the instance group.
|
387
|
+
#
|
388
|
+
# The default value for master and worker groups is
|
389
|
+
# `NON_PREEMPTIBLE`. This default cannot be changed.
|
390
|
+
#
|
391
|
+
# The default value for secondary instances is
|
392
|
+
# `PREEMPTIBLE`.
|
375
393
|
# @!attribute [r] managed_group_config
|
376
394
|
# @return [::Google::Cloud::Dataproc::V1beta2::ManagedGroupConfig]
|
377
395
|
# Output only. The config for Compute Engine Instance Group
|
@@ -389,6 +407,27 @@ module Google
|
|
389
407
|
class InstanceGroupConfig
|
390
408
|
include ::Google::Protobuf::MessageExts
|
391
409
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
410
|
+
|
411
|
+
# Controls the use of
|
412
|
+
# [preemptible instances]
|
413
|
+
# (https://cloud.google.com/compute/docs/instances/preemptible)
|
414
|
+
# within the group.
|
415
|
+
module Preemptibility
|
416
|
+
# Preemptibility is unspecified, the system will choose the
|
417
|
+
# appropriate setting for each instance group.
|
418
|
+
PREEMPTIBILITY_UNSPECIFIED = 0
|
419
|
+
|
420
|
+
# Instances are non-preemptible.
|
421
|
+
#
|
422
|
+
# This option is allowed for all instance groups and is the only valid
|
423
|
+
# value for Master and Worker instance groups.
|
424
|
+
NON_PREEMPTIBLE = 1
|
425
|
+
|
426
|
+
# Instances are preemptible.
|
427
|
+
#
|
428
|
+
# This option is allowed only for secondary worker groups.
|
429
|
+
PREEMPTIBLE = 2
|
430
|
+
end
|
392
431
|
end
|
393
432
|
|
394
433
|
# Specifies the resources used to actively manage an instance group.
|
@@ -464,7 +503,8 @@ module Google
|
|
464
503
|
# [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json).
|
465
504
|
# @!attribute [rw] auto_delete_time
|
466
505
|
# @return [::Google::Protobuf::Timestamp]
|
467
|
-
# Optional. The time when cluster will be auto-deleted. (see JSON
|
506
|
+
# Optional. The time when cluster will be auto-deleted. (see JSON
|
507
|
+
# representation of
|
468
508
|
# [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
|
469
509
|
# @!attribute [rw] auto_delete_ttl
|
470
510
|
# @return [::Google::Protobuf::Duration]
|
@@ -495,8 +535,8 @@ module Google
|
|
495
535
|
# Specifies Kerberos related configuration.
|
496
536
|
# @!attribute [rw] enable_kerberos
|
497
537
|
# @return [::Boolean]
|
498
|
-
# Optional. Flag to indicate whether to Kerberize the cluster (default:
|
499
|
-
# this field to true to enable Kerberos on a cluster.
|
538
|
+
# Optional. Flag to indicate whether to Kerberize the cluster (default:
|
539
|
+
# false). Set this field to true to enable Kerberos on a cluster.
|
500
540
|
# @!attribute [rw] root_principal_password_uri
|
501
541
|
# @return [::String]
|
502
542
|
# Required. The Cloud Storage URI of a KMS encrypted file containing the root
|
@@ -658,7 +698,7 @@ module Google
|
|
658
698
|
# @return [::String]
|
659
699
|
# Optional. The version of software inside the cluster. It must be one of the
|
660
700
|
# supported [Dataproc
|
661
|
-
# Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#
|
701
|
+
# Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported_dataproc_versions),
|
662
702
|
# such as "1.2" (including a subminor version, such as "1.2.29"), or the
|
663
703
|
# ["preview"
|
664
704
|
# version](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
|
@@ -747,10 +787,11 @@ module Google
|
|
747
787
|
# @!attribute [rw] request_id
|
748
788
|
# @return [::String]
|
749
789
|
# Optional. A unique id used to identify the request. If the server
|
750
|
-
# receives two
|
751
|
-
#
|
752
|
-
#
|
753
|
-
#
|
790
|
+
# receives two
|
791
|
+
# {::Google::Cloud::Dataproc::V1beta2::CreateClusterRequest CreateClusterRequest}
|
792
|
+
# requests with the same id, then the second request will be ignored and the
|
793
|
+
# first {::Google::Longrunning::Operation google.longrunning.Operation} created
|
794
|
+
# and stored in the backend is returned.
|
754
795
|
#
|
755
796
|
# It is recommended to always set this value to a
|
756
797
|
# [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
|
@@ -852,10 +893,11 @@ module Google
|
|
852
893
|
# @!attribute [rw] request_id
|
853
894
|
# @return [::String]
|
854
895
|
# Optional. A unique id used to identify the request. If the server
|
855
|
-
# receives two
|
856
|
-
#
|
857
|
-
#
|
858
|
-
#
|
896
|
+
# receives two
|
897
|
+
# {::Google::Cloud::Dataproc::V1beta2::UpdateClusterRequest UpdateClusterRequest}
|
898
|
+
# requests with the same id, then the second request will be ignored and the
|
899
|
+
# first {::Google::Longrunning::Operation google.longrunning.Operation} created
|
900
|
+
# and stored in the backend is returned.
|
859
901
|
#
|
860
902
|
# It is recommended to always set this value to a
|
861
903
|
# [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
|
@@ -885,10 +927,11 @@ module Google
|
|
885
927
|
# @!attribute [rw] request_id
|
886
928
|
# @return [::String]
|
887
929
|
# Optional. A unique id used to identify the request. If the server
|
888
|
-
# receives two
|
889
|
-
#
|
890
|
-
#
|
891
|
-
#
|
930
|
+
# receives two
|
931
|
+
# {::Google::Cloud::Dataproc::V1beta2::DeleteClusterRequest DeleteClusterRequest}
|
932
|
+
# requests with the same id, then the second request will be ignored and the
|
933
|
+
# first {::Google::Longrunning::Operation google.longrunning.Operation} created
|
934
|
+
# and stored in the backend is returned.
|
892
935
|
#
|
893
936
|
# It is recommended to always set this value to a
|
894
937
|
# [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
|
@@ -158,12 +158,12 @@ module Google
|
|
158
158
|
# Spark driver and tasks.
|
159
159
|
# @!attribute [rw] file_uris
|
160
160
|
# @return [::Array<::String>]
|
161
|
-
# Optional. HCFS URIs of files to be
|
162
|
-
#
|
161
|
+
# Optional. HCFS URIs of files to be placed in the working directory of
|
162
|
+
# each executor. Useful for naively parallel tasks.
|
163
163
|
# @!attribute [rw] archive_uris
|
164
164
|
# @return [::Array<::String>]
|
165
|
-
# Optional. HCFS URIs of archives to be extracted
|
166
|
-
# of
|
165
|
+
# Optional. HCFS URIs of archives to be extracted into the working directory
|
166
|
+
# of each executor. Supported file types:
|
167
167
|
# .jar, .tar, .tar.gz, .tgz, and .zip.
|
168
168
|
# @!attribute [rw] properties
|
169
169
|
# @return [::Google::Protobuf::Map{::String => ::String}]
|
@@ -211,11 +211,12 @@ module Google
|
|
211
211
|
# Python driver and tasks.
|
212
212
|
# @!attribute [rw] file_uris
|
213
213
|
# @return [::Array<::String>]
|
214
|
-
# Optional. HCFS URIs of files to be
|
215
|
-
#
|
214
|
+
# Optional. HCFS URIs of files to be placed in the working directory of
|
215
|
+
# each executor. Useful for naively parallel tasks.
|
216
216
|
# @!attribute [rw] archive_uris
|
217
217
|
# @return [::Array<::String>]
|
218
|
-
# Optional. HCFS URIs of archives to be extracted
|
218
|
+
# Optional. HCFS URIs of archives to be extracted into the working directory
|
219
|
+
# of each executor. Supported file types:
|
219
220
|
# .jar, .tar, .tar.gz, .tgz, and .zip.
|
220
221
|
# @!attribute [rw] properties
|
221
222
|
# @return [::Google::Protobuf::Map{::String => ::String}]
|
@@ -426,12 +427,12 @@ module Google
|
|
426
427
|
# occur that causes an incorrect job submission.
|
427
428
|
# @!attribute [rw] file_uris
|
428
429
|
# @return [::Array<::String>]
|
429
|
-
# Optional. HCFS URIs of files to be
|
430
|
-
#
|
430
|
+
# Optional. HCFS URIs of files to be placed in the working directory of
|
431
|
+
# each executor. Useful for naively parallel tasks.
|
431
432
|
# @!attribute [rw] archive_uris
|
432
433
|
# @return [::Array<::String>]
|
433
|
-
# Optional. HCFS URIs of archives to be extracted
|
434
|
-
#
|
434
|
+
# Optional. HCFS URIs of archives to be extracted into the working directory
|
435
|
+
# of each executor. Supported file types:
|
435
436
|
# .jar, .tar, .tar.gz, .tgz, and .zip.
|
436
437
|
# @!attribute [rw] properties
|
437
438
|
# @return [::Google::Protobuf::Map{::String => ::String}]
|
@@ -600,8 +601,8 @@ module Google
|
|
600
601
|
# Encapsulates the full scoping used to reference a job.
|
601
602
|
# @!attribute [rw] project_id
|
602
603
|
# @return [::String]
|
603
|
-
#
|
604
|
-
#
|
604
|
+
# Optional. The ID of the Google Cloud Platform project that the job belongs to. If
|
605
|
+
# specified, must match the request project ID.
|
605
606
|
# @!attribute [rw] job_id
|
606
607
|
# @return [::String]
|
607
608
|
# Optional. The job ID, which must be unique within the project.
|
@@ -23,20 +23,17 @@ module Google
|
|
23
23
|
module V1beta2
|
24
24
|
# Cluster components that can be activated.
|
25
25
|
module Component
|
26
|
-
# Unspecified component.
|
26
|
+
# Unspecified component. Specifying this will cause Cluster creation to fail.
|
27
27
|
COMPONENT_UNSPECIFIED = 0
|
28
28
|
|
29
29
|
# The Anaconda python distribution.
|
30
30
|
ANACONDA = 5
|
31
31
|
|
32
|
-
# Docker
|
33
|
-
DOCKER = 13
|
34
|
-
|
35
32
|
# The Druid query engine.
|
36
33
|
DRUID = 9
|
37
34
|
|
38
|
-
#
|
39
|
-
|
35
|
+
# HBase.
|
36
|
+
HBASE = 11
|
40
37
|
|
41
38
|
# The Hive Web HCatalog (the REST service for accessing HCatalog).
|
42
39
|
HIVE_WEBHCAT = 3
|
@@ -85,6 +85,17 @@ module Google
|
|
85
85
|
# Optional. Template parameters whose values are substituted into the
|
86
86
|
# template. Values for parameters must be provided when the template is
|
87
87
|
# instantiated.
|
88
|
+
# @!attribute [rw] dag_timeout
|
89
|
+
# @return [::Google::Protobuf::Duration]
|
90
|
+
# Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h",
|
91
|
+
# and "d" suffixes for second, minute, hour, and day duration values,
|
92
|
+
# respectively. The timeout duration must be from 10 minutes ("10m") to 24
|
93
|
+
# hours ("24h" or "1d"). The timer begins when the first job is submitted. If
|
94
|
+
# the workflow is running at the end of the timeout period, any remaining
|
95
|
+
# jobs are cancelled, the workflow is terminated, and if the workflow was
|
96
|
+
# running on a [managed
|
97
|
+
# cluster](https://cloud.google.com/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster),
|
98
|
+
# the cluster is deleted.
|
88
99
|
class WorkflowTemplate
|
89
100
|
include ::Google::Protobuf::MessageExts
|
90
101
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -188,30 +199,36 @@ module Google
|
|
188
199
|
#
|
189
200
|
# The step id is used as prefix for job id, as job
|
190
201
|
# `goog-dataproc-workflow-step-id` label, and in
|
191
|
-
# {::Google::Cloud::Dataproc::V1beta2::OrderedJob#prerequisite_step_ids prerequisiteStepIds}
|
192
|
-
# steps.
|
202
|
+
# {::Google::Cloud::Dataproc::V1beta2::OrderedJob#prerequisite_step_ids prerequisiteStepIds}
|
203
|
+
# field from other steps.
|
193
204
|
#
|
194
205
|
# The id must contain only letters (a-z, A-Z), numbers (0-9),
|
195
206
|
# underscores (_), and hyphens (-). Cannot begin or end with underscore
|
196
207
|
# or hyphen. Must consist of between 3 and 50 characters.
|
197
208
|
# @!attribute [rw] hadoop_job
|
198
209
|
# @return [::Google::Cloud::Dataproc::V1beta2::HadoopJob]
|
210
|
+
# Optional. Job is a Hadoop job.
|
199
211
|
# @!attribute [rw] spark_job
|
200
212
|
# @return [::Google::Cloud::Dataproc::V1beta2::SparkJob]
|
213
|
+
# Optional. Job is a Spark job.
|
201
214
|
# @!attribute [rw] pyspark_job
|
202
215
|
# @return [::Google::Cloud::Dataproc::V1beta2::PySparkJob]
|
216
|
+
# Optional. Job is a PySpark job.
|
203
217
|
# @!attribute [rw] hive_job
|
204
218
|
# @return [::Google::Cloud::Dataproc::V1beta2::HiveJob]
|
219
|
+
# Optional. Job is a Hive job.
|
205
220
|
# @!attribute [rw] pig_job
|
206
221
|
# @return [::Google::Cloud::Dataproc::V1beta2::PigJob]
|
222
|
+
# Optional. Job is a Pig job.
|
207
223
|
# @!attribute [rw] spark_r_job
|
208
224
|
# @return [::Google::Cloud::Dataproc::V1beta2::SparkRJob]
|
209
|
-
#
|
225
|
+
# Optional. Job is a SparkR job.
|
210
226
|
# @!attribute [rw] spark_sql_job
|
211
227
|
# @return [::Google::Cloud::Dataproc::V1beta2::SparkSqlJob]
|
228
|
+
# Optional. Job is a SparkSql job.
|
212
229
|
# @!attribute [rw] presto_job
|
213
230
|
# @return [::Google::Cloud::Dataproc::V1beta2::PrestoJob]
|
214
|
-
# Presto job
|
231
|
+
# Optional. Job is a Presto job.
|
215
232
|
# @!attribute [rw] labels
|
216
233
|
# @return [::Google::Protobuf::Map{::String => ::String}]
|
217
234
|
# Optional. The labels to associate with this job.
|
@@ -269,10 +286,10 @@ module Google
|
|
269
286
|
# A field is allowed to appear in at most one parameter's list of field
|
270
287
|
# paths.
|
271
288
|
#
|
272
|
-
# A field path is similar in syntax to a
|
273
|
-
# For example, a
|
274
|
-
#
|
275
|
-
# `placement.clusterSelector.zone`.
|
289
|
+
# A field path is similar in syntax to a
|
290
|
+
# {::Google::Protobuf::FieldMask google.protobuf.FieldMask}. For example, a
|
291
|
+
# field path that references the zone field of a workflow template's cluster
|
292
|
+
# selector would be specified as `placement.clusterSelector.zone`.
|
276
293
|
#
|
277
294
|
# Also, field paths can reference fields using the following syntax:
|
278
295
|
#
|
@@ -397,6 +414,23 @@ module Google
|
|
397
414
|
# @!attribute [r] cluster_uuid
|
398
415
|
# @return [::String]
|
399
416
|
# Output only. The UUID of target cluster.
|
417
|
+
# @!attribute [r] dag_timeout
|
418
|
+
# @return [::Google::Protobuf::Duration]
|
419
|
+
# Output only. The timeout duration for the DAG of jobs.
|
420
|
+
# Minimum timeout duration is 10 minutes and maximum is 24 hours, expressed
|
421
|
+
# as a
|
422
|
+
# [google.protobuf.Duration][https://developers.google.com/protocol-buffers/docs/proto3#json_mapping].
|
423
|
+
# For example, "1800" = 1800 seconds/30 minutes duration.
|
424
|
+
# @!attribute [r] dag_start_time
|
425
|
+
# @return [::Google::Protobuf::Timestamp]
|
426
|
+
# Output only. DAG start time, which is only set for workflows with
|
427
|
+
# {::Google::Cloud::Dataproc::V1beta2::WorkflowMetadata#dag_timeout dag_timeout}
|
428
|
+
# when the DAG begins.
|
429
|
+
# @!attribute [r] dag_end_time
|
430
|
+
# @return [::Google::Protobuf::Timestamp]
|
431
|
+
# Output only. DAG end time, which is only set for workflows with
|
432
|
+
# {::Google::Cloud::Dataproc::V1beta2::WorkflowMetadata#dag_timeout dag_timeout}
|
433
|
+
# when the DAG ends.
|
400
434
|
class WorkflowMetadata
|
401
435
|
include ::Google::Protobuf::MessageExts
|
402
436
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|