google-cloud-dataproc-v1beta2 0.2.5 → 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (34) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.md +188 -190
  3. data/README.md +66 -2
  4. data/lib/google/cloud/dataproc/v1beta2.rb +3 -0
  5. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policies_services_pb.rb +5 -5
  6. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service/client.rb +25 -19
  7. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service/paths.rb +1 -1
  8. data/lib/google/cloud/dataproc/v1beta2/cluster_controller.rb +1 -0
  9. data/lib/google/cloud/dataproc/v1beta2/cluster_controller/client.rb +62 -51
  10. data/lib/google/cloud/dataproc/v1beta2/cluster_controller/operations.rb +95 -10
  11. data/lib/google/cloud/dataproc/v1beta2/cluster_controller/paths.rb +52 -0
  12. data/lib/google/cloud/dataproc/v1beta2/clusters_pb.rb +8 -0
  13. data/lib/google/cloud/dataproc/v1beta2/clusters_services_pb.rb +6 -6
  14. data/lib/google/cloud/dataproc/v1beta2/job_controller/client.rb +50 -44
  15. data/lib/google/cloud/dataproc/v1beta2/job_controller/operations.rb +95 -10
  16. data/lib/google/cloud/dataproc/v1beta2/jobs_services_pb.rb +7 -7
  17. data/lib/google/cloud/dataproc/v1beta2/shared_pb.rb +1 -2
  18. data/lib/google/cloud/dataproc/v1beta2/version.rb +1 -1
  19. data/lib/google/cloud/dataproc/v1beta2/workflow_template_service/client.rb +41 -34
  20. data/lib/google/cloud/dataproc/v1beta2/workflow_template_service/operations.rb +95 -10
  21. data/lib/google/cloud/dataproc/v1beta2/workflow_template_service/paths.rb +20 -1
  22. data/lib/google/cloud/dataproc/v1beta2/workflow_templates_pb.rb +5 -0
  23. data/lib/google/cloud/dataproc/v1beta2/workflow_templates_services_pb.rb +9 -8
  24. data/proto_docs/google/api/field_behavior.rb +6 -0
  25. data/proto_docs/google/api/resource.rb +50 -14
  26. data/proto_docs/google/cloud/dataproc/v1beta2/autoscaling_policies.rb +13 -7
  27. data/proto_docs/google/cloud/dataproc/v1beta2/clusters.rb +64 -21
  28. data/proto_docs/google/cloud/dataproc/v1beta2/jobs.rb +14 -13
  29. data/proto_docs/google/cloud/dataproc/v1beta2/shared.rb +3 -6
  30. data/proto_docs/google/cloud/dataproc/v1beta2/workflow_templates.rb +42 -8
  31. data/proto_docs/google/longrunning/operations.rb +17 -3
  32. data/proto_docs/google/protobuf/any.rb +5 -2
  33. data/proto_docs/google/protobuf/timestamp.rb +10 -1
  34. metadata +11 -7
@@ -54,6 +54,12 @@ module Google
54
54
  # This indicates that the field may be set once in a request to create a
55
55
  # resource, but may not be changed thereafter.
56
56
  IMMUTABLE = 5
57
+
58
+ # Denotes that a (repeated) field is an unordered list.
59
+ # This indicates that the service may provide the elements of the list
60
+ # in any arbitrary order, rather than the order the user originally
61
+ # provided. Additionally, the list's order may or may not be stable.
62
+ UNORDERED_LIST = 6
57
63
  end
58
64
  end
59
65
  end
@@ -43,12 +43,12 @@ module Google
43
43
  #
44
44
  # The ResourceDescriptor Yaml config will look like:
45
45
  #
46
- # resources:
47
- # - type: "pubsub.googleapis.com/Topic"
48
- # name_descriptor:
49
- # - pattern: "projects/\\{project}/topics/\\{topic}"
50
- # parent_type: "cloudresourcemanager.googleapis.com/Project"
51
- # parent_name_extractor: "projects/\\{project}"
46
+ # resources:
47
+ # - type: "pubsub.googleapis.com/Topic"
48
+ # name_descriptor:
49
+ # - pattern: "projects/{project}/topics/{topic}"
50
+ # parent_type: "cloudresourcemanager.googleapis.com/Project"
51
+ # parent_name_extractor: "projects/{project}"
52
52
  #
53
53
  # Sometimes, resources have multiple patterns, typically because they can
54
54
  # live under multiple parents.
@@ -183,15 +183,24 @@ module Google
183
183
  # }
184
184
  # @!attribute [rw] plural
185
185
  # @return [::String]
186
- # The plural name used in the resource name, such as 'projects' for
187
- # the name of 'projects/\\{project}'. It is the same concept of the `plural`
188
- # field in k8s CRD spec
186
+ # The plural name used in the resource name and permission names, such as
187
+ # 'projects' for the resource name of 'projects/\\{project}' and the permission
188
+ # name of 'cloudresourcemanager.googleapis.com/projects.get'. It is the same
189
+ # concept of the `plural` field in k8s CRD spec
189
190
  # https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
191
+ #
192
+ # Note: The plural form is required even for singleton resources. See
193
+ # https://aip.dev/156
190
194
  # @!attribute [rw] singular
191
195
  # @return [::String]
192
196
  # The same concept of the `singular` field in k8s CRD spec
193
197
  # https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
194
198
  # Such as "project" for the `resourcemanager.googleapis.com/Project` type.
199
+ # @!attribute [rw] style
200
+ # @return [::Array<::Google::Api::ResourceDescriptor::Style>]
201
+ # Style flag(s) for this resource.
202
+ # These indicate that a resource is expected to conform to a given
203
+ # style. See the specific style flags for additional information.
195
204
  class ResourceDescriptor
196
205
  include ::Google::Protobuf::MessageExts
197
206
  extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -211,6 +220,22 @@ module Google
211
220
  # that from being necessary once there are multiple patterns.)
212
221
  FUTURE_MULTI_PATTERN = 2
213
222
  end
223
+
224
+ # A flag representing a specific style that a resource claims to conform to.
225
+ module Style
226
+ # The unspecified value. Do not use.
227
+ STYLE_UNSPECIFIED = 0
228
+
229
+ # This resource is intended to be "declarative-friendly".
230
+ #
231
+ # Declarative-friendly resources must be more strictly consistent, and
232
+ # setting this to true communicates to tools that this resource should
233
+ # adhere to declarative-friendly expectations.
234
+ #
235
+ # Note: This is used by the API linter (linter.aip.dev) to enable
236
+ # additional checks.
237
+ DECLARATIVE_FRIENDLY = 1
238
+ end
214
239
  end
215
240
 
216
241
  # Defines a proto annotation that describes a string field that refers to
@@ -226,6 +251,17 @@ module Google
226
251
  # type: "pubsub.googleapis.com/Topic"
227
252
  # }];
228
253
  # }
254
+ #
255
+ # Occasionally, a field may reference an arbitrary resource. In this case,
256
+ # APIs use the special value * in their resource reference.
257
+ #
258
+ # Example:
259
+ #
260
+ # message GetIamPolicyRequest {
261
+ # string resource = 2 [(google.api.resource_reference) = {
262
+ # type: "*"
263
+ # }];
264
+ # }
229
265
  # @!attribute [rw] child_type
230
266
  # @return [::String]
231
267
  # The resource type of a child collection that the annotated field
@@ -234,11 +270,11 @@ module Google
234
270
  #
235
271
  # Example:
236
272
  #
237
- # message ListLogEntriesRequest {
238
- # string parent = 1 [(google.api.resource_reference) = {
239
- # child_type: "logging.googleapis.com/LogEntry"
240
- # };
241
- # }
273
+ # message ListLogEntriesRequest {
274
+ # string parent = 1 [(google.api.resource_reference) = {
275
+ # child_type: "logging.googleapis.com/LogEntry"
276
+ # };
277
+ # }
242
278
  class ResourceReference
243
279
  include ::Google::Protobuf::MessageExts
244
280
  extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -80,20 +80,26 @@ module Google
80
80
  # Bounds: [0s, 1d].
81
81
  # @!attribute [rw] scale_up_factor
82
82
  # @return [::Float]
83
- # Required. Fraction of average pending memory in the last cooldown period
84
- # for which to add workers. A scale-up factor of 1.0 will result in scaling
85
- # up so that there is no pending memory remaining after the update (more
86
- # aggressive scaling). A scale-up factor closer to 0 will result in a smaller
87
- # magnitude of scaling up (less aggressive scaling).
83
+ # Required. Fraction of average YARN pending memory in the last cooldown
84
+ # period for which to add workers. A scale-up factor of 1.0 will result in
85
+ # scaling up so that there is no pending memory remaining after the update
86
+ # (more aggressive scaling). A scale-up factor closer to 0 will result in a
87
+ # smaller magnitude of scaling up (less aggressive scaling). See [How
88
+ # autoscaling
89
+ # works](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/autoscaling#how_autoscaling_works)
90
+ # for more information.
88
91
  #
89
92
  # Bounds: [0.0, 1.0].
90
93
  # @!attribute [rw] scale_down_factor
91
94
  # @return [::Float]
92
- # Required. Fraction of average pending memory in the last cooldown period
93
- # for which to remove workers. A scale-down factor of 1 will result in
95
+ # Required. Fraction of average YARN pending memory in the last cooldown
96
+ # period for which to remove workers. A scale-down factor of 1 will result in
94
97
  # scaling down so that there is no available memory remaining after the
95
98
  # update (more aggressive scaling). A scale-down factor of 0 disables
96
99
  # removing workers, which can be beneficial for autoscaling a single job.
100
+ # See [How autoscaling
101
+ # works](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/autoscaling#how_autoscaling_works)
102
+ # for more information.
97
103
  #
98
104
  # Bounds: [0.0, 1.0].
99
105
  # @!attribute [rw] scale_up_min_worker_fraction
@@ -85,6 +85,15 @@ module Google
85
85
  # and manage this project-level, per-location bucket (see
86
86
  # [Dataproc staging
87
87
  # bucket](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
88
+ # @!attribute [rw] temp_bucket
89
+ # @return [::String]
90
+ # Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs
91
+ # data, such as Spark and MapReduce history files. If you do not specify a
92
+ # temp bucket, Dataproc will determine a Cloud Storage location (US, ASIA, or
93
+ # EU) for your cluster's temp bucket according to the Compute Engine zone
94
+ # where your cluster is deployed, and then create and manage this
95
+ # project-level, per-location bucket. The default bucket has a TTL of 90
96
+ # days, but you can use any TTL (or none) if you specify a bucket.
88
97
  # @!attribute [rw] gce_cluster_config
89
98
  # @return [::Google::Cloud::Dataproc::V1beta2::GceClusterConfig]
90
99
  # Optional. The shared Compute Engine config settings for
@@ -137,10 +146,10 @@ module Google
137
146
  # Optional. Security related configuration.
138
147
  # @!attribute [rw] gke_cluster_config
139
148
  # @return [::Google::Cloud::Dataproc::V1beta2::GkeClusterConfig]
140
- # Optional. The Kubernetes Engine config for Dataproc clusters deployed to Kubernetes.
141
- # Setting this is considered mutually exclusive with Compute Engine-based
142
- # options such as `gce_cluster_config`, `master_config`, `worker_config`,
143
- # `secondary_worker_config`, and `autoscaling_config`.
149
+ # Optional. The Kubernetes Engine config for Dataproc clusters deployed to
150
+ # Kubernetes. Setting this is considered mutually exclusive with Compute
151
+ # Engine-based options such as `gce_cluster_config`, `master_config`,
152
+ # `worker_config`, `secondary_worker_config`, and `autoscaling_config`.
144
153
  class ClusterConfig
145
154
  include ::Google::Protobuf::MessageExts
146
155
  extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -268,7 +277,7 @@ module Google
268
277
  # @!attribute [rw] service_account
269
278
  # @return [::String]
270
279
  # Optional. The [Dataproc service
271
- # account](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#service_accounts_in_cloud_dataproc)
280
+ # account](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#service_accounts_in_dataproc)
272
281
  # (also see [VM Data Plane
273
282
  # identity](https://cloud.google.com/dataproc/docs/concepts/iam/dataproc-principals#vm_service_account_data_plane_identity))
274
283
  # used by Dataproc cluster VM instances to access Google Cloud Platform
@@ -372,6 +381,15 @@ module Google
372
381
  # @return [::Boolean]
373
382
  # Output only. Specifies that this instance group contains preemptible
374
383
  # instances.
384
+ # @!attribute [rw] preemptibility
385
+ # @return [::Google::Cloud::Dataproc::V1beta2::InstanceGroupConfig::Preemptibility]
386
+ # Optional. Specifies the preemptibility of the instance group.
387
+ #
388
+ # The default value for master and worker groups is
389
+ # `NON_PREEMPTIBLE`. This default cannot be changed.
390
+ #
391
+ # The default value for secondary instances is
392
+ # `PREEMPTIBLE`.
375
393
  # @!attribute [r] managed_group_config
376
394
  # @return [::Google::Cloud::Dataproc::V1beta2::ManagedGroupConfig]
377
395
  # Output only. The config for Compute Engine Instance Group
@@ -389,6 +407,27 @@ module Google
389
407
  class InstanceGroupConfig
390
408
  include ::Google::Protobuf::MessageExts
391
409
  extend ::Google::Protobuf::MessageExts::ClassMethods
410
+
411
+ # Controls the use of
412
+ # [preemptible instances]
413
+ # (https://cloud.google.com/compute/docs/instances/preemptible)
414
+ # within the group.
415
+ module Preemptibility
416
+ # Preemptibility is unspecified, the system will choose the
417
+ # appropriate setting for each instance group.
418
+ PREEMPTIBILITY_UNSPECIFIED = 0
419
+
420
+ # Instances are non-preemptible.
421
+ #
422
+ # This option is allowed for all instance groups and is the only valid
423
+ # value for Master and Worker instance groups.
424
+ NON_PREEMPTIBLE = 1
425
+
426
+ # Instances are preemptible.
427
+ #
428
+ # This option is allowed only for secondary worker groups.
429
+ PREEMPTIBLE = 2
430
+ end
392
431
  end
393
432
 
394
433
  # Specifies the resources used to actively manage an instance group.
@@ -464,7 +503,8 @@ module Google
464
503
  # [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json).
465
504
  # @!attribute [rw] auto_delete_time
466
505
  # @return [::Google::Protobuf::Timestamp]
467
- # Optional. The time when cluster will be auto-deleted. (see JSON representation of
506
+ # Optional. The time when cluster will be auto-deleted. (see JSON
507
+ # representation of
468
508
  # [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
469
509
  # @!attribute [rw] auto_delete_ttl
470
510
  # @return [::Google::Protobuf::Duration]
@@ -495,8 +535,8 @@ module Google
495
535
  # Specifies Kerberos related configuration.
496
536
  # @!attribute [rw] enable_kerberos
497
537
  # @return [::Boolean]
498
- # Optional. Flag to indicate whether to Kerberize the cluster (default: false). Set
499
- # this field to true to enable Kerberos on a cluster.
538
+ # Optional. Flag to indicate whether to Kerberize the cluster (default:
539
+ # false). Set this field to true to enable Kerberos on a cluster.
500
540
  # @!attribute [rw] root_principal_password_uri
501
541
  # @return [::String]
502
542
  # Required. The Cloud Storage URI of a KMS encrypted file containing the root
@@ -658,7 +698,7 @@ module Google
658
698
  # @return [::String]
659
699
  # Optional. The version of software inside the cluster. It must be one of the
660
700
  # supported [Dataproc
661
- # Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported_cloud_dataproc_versions),
701
+ # Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported_dataproc_versions),
662
702
  # such as "1.2" (including a subminor version, such as "1.2.29"), or the
663
703
  # ["preview"
664
704
  # version](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
@@ -747,10 +787,11 @@ module Google
747
787
  # @!attribute [rw] request_id
748
788
  # @return [::String]
749
789
  # Optional. A unique id used to identify the request. If the server
750
- # receives two {::Google::Cloud::Dataproc::V1beta2::CreateClusterRequest CreateClusterRequest} requests with the same
751
- # id, then the second request will be ignored and the
752
- # first {::Google::Longrunning::Operation google.longrunning.Operation} created and stored in the backend
753
- # is returned.
790
+ # receives two
791
+ # {::Google::Cloud::Dataproc::V1beta2::CreateClusterRequest CreateClusterRequest}
792
+ # requests with the same id, then the second request will be ignored and the
793
+ # first {::Google::Longrunning::Operation google.longrunning.Operation} created
794
+ # and stored in the backend is returned.
754
795
  #
755
796
  # It is recommended to always set this value to a
756
797
  # [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
@@ -852,10 +893,11 @@ module Google
852
893
  # @!attribute [rw] request_id
853
894
  # @return [::String]
854
895
  # Optional. A unique id used to identify the request. If the server
855
- # receives two {::Google::Cloud::Dataproc::V1beta2::UpdateClusterRequest UpdateClusterRequest} requests with the same
856
- # id, then the second request will be ignored and the
857
- # first {::Google::Longrunning::Operation google.longrunning.Operation} created and stored in the
858
- # backend is returned.
896
+ # receives two
897
+ # {::Google::Cloud::Dataproc::V1beta2::UpdateClusterRequest UpdateClusterRequest}
898
+ # requests with the same id, then the second request will be ignored and the
899
+ # first {::Google::Longrunning::Operation google.longrunning.Operation} created
900
+ # and stored in the backend is returned.
859
901
  #
860
902
  # It is recommended to always set this value to a
861
903
  # [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
@@ -885,10 +927,11 @@ module Google
885
927
  # @!attribute [rw] request_id
886
928
  # @return [::String]
887
929
  # Optional. A unique id used to identify the request. If the server
888
- # receives two {::Google::Cloud::Dataproc::V1beta2::DeleteClusterRequest DeleteClusterRequest} requests with the same
889
- # id, then the second request will be ignored and the
890
- # first {::Google::Longrunning::Operation google.longrunning.Operation} created and stored in the
891
- # backend is returned.
930
+ # receives two
931
+ # {::Google::Cloud::Dataproc::V1beta2::DeleteClusterRequest DeleteClusterRequest}
932
+ # requests with the same id, then the second request will be ignored and the
933
+ # first {::Google::Longrunning::Operation google.longrunning.Operation} created
934
+ # and stored in the backend is returned.
892
935
  #
893
936
  # It is recommended to always set this value to a
894
937
  # [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
@@ -158,12 +158,12 @@ module Google
158
158
  # Spark driver and tasks.
159
159
  # @!attribute [rw] file_uris
160
160
  # @return [::Array<::String>]
161
- # Optional. HCFS URIs of files to be copied to the working directory of
162
- # Spark drivers and distributed tasks. Useful for naively parallel tasks.
161
+ # Optional. HCFS URIs of files to be placed in the working directory of
162
+ # each executor. Useful for naively parallel tasks.
163
163
  # @!attribute [rw] archive_uris
164
164
  # @return [::Array<::String>]
165
- # Optional. HCFS URIs of archives to be extracted in the working directory
166
- # of Spark drivers and tasks. Supported file types:
165
+ # Optional. HCFS URIs of archives to be extracted into the working directory
166
+ # of each executor. Supported file types:
167
167
  # .jar, .tar, .tar.gz, .tgz, and .zip.
168
168
  # @!attribute [rw] properties
169
169
  # @return [::Google::Protobuf::Map{::String => ::String}]
@@ -211,11 +211,12 @@ module Google
211
211
  # Python driver and tasks.
212
212
  # @!attribute [rw] file_uris
213
213
  # @return [::Array<::String>]
214
- # Optional. HCFS URIs of files to be copied to the working directory of
215
- # Python drivers and distributed tasks. Useful for naively parallel tasks.
214
+ # Optional. HCFS URIs of files to be placed in the working directory of
215
+ # each executor. Useful for naively parallel tasks.
216
216
  # @!attribute [rw] archive_uris
217
217
  # @return [::Array<::String>]
218
- # Optional. HCFS URIs of archives to be extracted in the working directory of
218
+ # Optional. HCFS URIs of archives to be extracted into the working directory
219
+ # of each executor. Supported file types:
219
220
  # .jar, .tar, .tar.gz, .tgz, and .zip.
220
221
  # @!attribute [rw] properties
221
222
  # @return [::Google::Protobuf::Map{::String => ::String}]
@@ -426,12 +427,12 @@ module Google
426
427
  # occur that causes an incorrect job submission.
427
428
  # @!attribute [rw] file_uris
428
429
  # @return [::Array<::String>]
429
- # Optional. HCFS URIs of files to be copied to the working directory of
430
- # R drivers and distributed tasks. Useful for naively parallel tasks.
430
+ # Optional. HCFS URIs of files to be placed in the working directory of
431
+ # each executor. Useful for naively parallel tasks.
431
432
  # @!attribute [rw] archive_uris
432
433
  # @return [::Array<::String>]
433
- # Optional. HCFS URIs of archives to be extracted in the working directory of
434
- # Spark drivers and tasks. Supported file types:
434
+ # Optional. HCFS URIs of archives to be extracted into the working directory
435
+ # of each executor. Supported file types:
435
436
  # .jar, .tar, .tar.gz, .tgz, and .zip.
436
437
  # @!attribute [rw] properties
437
438
  # @return [::Google::Protobuf::Map{::String => ::String}]
@@ -600,8 +601,8 @@ module Google
600
601
  # Encapsulates the full scoping used to reference a job.
601
602
  # @!attribute [rw] project_id
602
603
  # @return [::String]
603
- # Required. The ID of the Google Cloud Platform project that the job
604
- # belongs to.
604
+ # Optional. The ID of the Google Cloud Platform project that the job belongs to. If
605
+ # specified, must match the request project ID.
605
606
  # @!attribute [rw] job_id
606
607
  # @return [::String]
607
608
  # Optional. The job ID, which must be unique within the project.
@@ -23,20 +23,17 @@ module Google
23
23
  module V1beta2
24
24
  # Cluster components that can be activated.
25
25
  module Component
26
- # Unspecified component.
26
+ # Unspecified component. Specifying this will cause Cluster creation to fail.
27
27
  COMPONENT_UNSPECIFIED = 0
28
28
 
29
29
  # The Anaconda python distribution.
30
30
  ANACONDA = 5
31
31
 
32
- # Docker
33
- DOCKER = 13
34
-
35
32
  # The Druid query engine.
36
33
  DRUID = 9
37
34
 
38
- # Flink
39
- FLINK = 14
35
+ # HBase.
36
+ HBASE = 11
40
37
 
41
38
  # The Hive Web HCatalog (the REST service for accessing HCatalog).
42
39
  HIVE_WEBHCAT = 3
@@ -85,6 +85,17 @@ module Google
85
85
  # Optional. Template parameters whose values are substituted into the
86
86
  # template. Values for parameters must be provided when the template is
87
87
  # instantiated.
88
+ # @!attribute [rw] dag_timeout
89
+ # @return [::Google::Protobuf::Duration]
90
+ # Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h",
91
+ # and "d" suffixes for second, minute, hour, and day duration values,
92
+ # respectively. The timeout duration must be from 10 minutes ("10m") to 24
93
+ # hours ("24h" or "1d"). The timer begins when the first job is submitted. If
94
+ # the workflow is running at the end of the timeout period, any remaining
95
+ # jobs are cancelled, the workflow is terminated, and if the workflow was
96
+ # running on a [managed
97
+ # cluster](https://cloud.google.com/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster),
98
+ # the cluster is deleted.
88
99
  class WorkflowTemplate
89
100
  include ::Google::Protobuf::MessageExts
90
101
  extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -188,30 +199,36 @@ module Google
188
199
  #
189
200
  # The step id is used as prefix for job id, as job
190
201
  # `goog-dataproc-workflow-step-id` label, and in
191
- # {::Google::Cloud::Dataproc::V1beta2::OrderedJob#prerequisite_step_ids prerequisiteStepIds} field from other
192
- # steps.
202
+ # {::Google::Cloud::Dataproc::V1beta2::OrderedJob#prerequisite_step_ids prerequisiteStepIds}
203
+ # field from other steps.
193
204
  #
194
205
  # The id must contain only letters (a-z, A-Z), numbers (0-9),
195
206
  # underscores (_), and hyphens (-). Cannot begin or end with underscore
196
207
  # or hyphen. Must consist of between 3 and 50 characters.
197
208
  # @!attribute [rw] hadoop_job
198
209
  # @return [::Google::Cloud::Dataproc::V1beta2::HadoopJob]
210
+ # Optional. Job is a Hadoop job.
199
211
  # @!attribute [rw] spark_job
200
212
  # @return [::Google::Cloud::Dataproc::V1beta2::SparkJob]
213
+ # Optional. Job is a Spark job.
201
214
  # @!attribute [rw] pyspark_job
202
215
  # @return [::Google::Cloud::Dataproc::V1beta2::PySparkJob]
216
+ # Optional. Job is a PySpark job.
203
217
  # @!attribute [rw] hive_job
204
218
  # @return [::Google::Cloud::Dataproc::V1beta2::HiveJob]
219
+ # Optional. Job is a Hive job.
205
220
  # @!attribute [rw] pig_job
206
221
  # @return [::Google::Cloud::Dataproc::V1beta2::PigJob]
222
+ # Optional. Job is a Pig job.
207
223
  # @!attribute [rw] spark_r_job
208
224
  # @return [::Google::Cloud::Dataproc::V1beta2::SparkRJob]
209
- # Spark R job
225
+ # Optional. Job is a SparkR job.
210
226
  # @!attribute [rw] spark_sql_job
211
227
  # @return [::Google::Cloud::Dataproc::V1beta2::SparkSqlJob]
228
+ # Optional. Job is a SparkSql job.
212
229
  # @!attribute [rw] presto_job
213
230
  # @return [::Google::Cloud::Dataproc::V1beta2::PrestoJob]
214
- # Presto job
231
+ # Optional. Job is a Presto job.
215
232
  # @!attribute [rw] labels
216
233
  # @return [::Google::Protobuf::Map{::String => ::String}]
217
234
  # Optional. The labels to associate with this job.
@@ -269,10 +286,10 @@ module Google
269
286
  # A field is allowed to appear in at most one parameter's list of field
270
287
  # paths.
271
288
  #
272
- # A field path is similar in syntax to a {::Google::Protobuf::FieldMask google.protobuf.FieldMask}.
273
- # For example, a field path that references the zone field of a workflow
274
- # template's cluster selector would be specified as
275
- # `placement.clusterSelector.zone`.
289
+ # A field path is similar in syntax to a
290
+ # {::Google::Protobuf::FieldMask google.protobuf.FieldMask}. For example, a
291
+ # field path that references the zone field of a workflow template's cluster
292
+ # selector would be specified as `placement.clusterSelector.zone`.
276
293
  #
277
294
  # Also, field paths can reference fields using the following syntax:
278
295
  #
@@ -397,6 +414,23 @@ module Google
397
414
  # @!attribute [r] cluster_uuid
398
415
  # @return [::String]
399
416
  # Output only. The UUID of target cluster.
417
+ # @!attribute [r] dag_timeout
418
+ # @return [::Google::Protobuf::Duration]
419
+ # Output only. The timeout duration for the DAG of jobs.
420
+ # Minimum timeout duration is 10 minutes and maximum is 24 hours, expressed
421
+ # as a
422
+ # [google.protobuf.Duration][https://developers.google.com/protocol-buffers/docs/proto3#json_mapping].
423
+ # For example, "1800" = 1800 seconds/30 minutes duration.
424
+ # @!attribute [r] dag_start_time
425
+ # @return [::Google::Protobuf::Timestamp]
426
+ # Output only. DAG start time, which is only set for workflows with
427
+ # {::Google::Cloud::Dataproc::V1beta2::WorkflowMetadata#dag_timeout dag_timeout}
428
+ # when the DAG begins.
429
+ # @!attribute [r] dag_end_time
430
+ # @return [::Google::Protobuf::Timestamp]
431
+ # Output only. DAG end time, which is only set for workflows with
432
+ # {::Google::Cloud::Dataproc::V1beta2::WorkflowMetadata#dag_timeout dag_timeout}
433
+ # when the DAG ends.
400
434
  class WorkflowMetadata
401
435
  include ::Google::Protobuf::MessageExts
402
436
  extend ::Google::Protobuf::MessageExts::ClassMethods