google-cloud-dataproc-v1beta2 0.2.5 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.md +188 -190
  3. data/README.md +66 -2
  4. data/lib/google/cloud/dataproc/v1beta2.rb +3 -0
  5. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policies_services_pb.rb +5 -5
  6. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service/client.rb +25 -19
  7. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service/paths.rb +1 -1
  8. data/lib/google/cloud/dataproc/v1beta2/cluster_controller.rb +1 -0
  9. data/lib/google/cloud/dataproc/v1beta2/cluster_controller/client.rb +62 -51
  10. data/lib/google/cloud/dataproc/v1beta2/cluster_controller/operations.rb +95 -10
  11. data/lib/google/cloud/dataproc/v1beta2/cluster_controller/paths.rb +52 -0
  12. data/lib/google/cloud/dataproc/v1beta2/clusters_pb.rb +8 -0
  13. data/lib/google/cloud/dataproc/v1beta2/clusters_services_pb.rb +6 -6
  14. data/lib/google/cloud/dataproc/v1beta2/job_controller/client.rb +50 -44
  15. data/lib/google/cloud/dataproc/v1beta2/job_controller/operations.rb +95 -10
  16. data/lib/google/cloud/dataproc/v1beta2/jobs_services_pb.rb +7 -7
  17. data/lib/google/cloud/dataproc/v1beta2/shared_pb.rb +1 -2
  18. data/lib/google/cloud/dataproc/v1beta2/version.rb +1 -1
  19. data/lib/google/cloud/dataproc/v1beta2/workflow_template_service/client.rb +41 -34
  20. data/lib/google/cloud/dataproc/v1beta2/workflow_template_service/operations.rb +95 -10
  21. data/lib/google/cloud/dataproc/v1beta2/workflow_template_service/paths.rb +20 -1
  22. data/lib/google/cloud/dataproc/v1beta2/workflow_templates_pb.rb +5 -0
  23. data/lib/google/cloud/dataproc/v1beta2/workflow_templates_services_pb.rb +9 -8
  24. data/proto_docs/google/api/field_behavior.rb +6 -0
  25. data/proto_docs/google/api/resource.rb +50 -14
  26. data/proto_docs/google/cloud/dataproc/v1beta2/autoscaling_policies.rb +13 -7
  27. data/proto_docs/google/cloud/dataproc/v1beta2/clusters.rb +64 -21
  28. data/proto_docs/google/cloud/dataproc/v1beta2/jobs.rb +14 -13
  29. data/proto_docs/google/cloud/dataproc/v1beta2/shared.rb +3 -6
  30. data/proto_docs/google/cloud/dataproc/v1beta2/workflow_templates.rb +42 -8
  31. data/proto_docs/google/longrunning/operations.rb +17 -3
  32. data/proto_docs/google/protobuf/any.rb +5 -2
  33. data/proto_docs/google/protobuf/timestamp.rb +10 -1
  34. metadata +11 -7
@@ -54,6 +54,12 @@ module Google
54
54
  # This indicates that the field may be set once in a request to create a
55
55
  # resource, but may not be changed thereafter.
56
56
  IMMUTABLE = 5
57
+
58
+ # Denotes that a (repeated) field is an unordered list.
59
+ # This indicates that the service may provide the elements of the list
60
+ # in any arbitrary order, rather than the order the user originally
61
+ # provided. Additionally, the list's order may or may not be stable.
62
+ UNORDERED_LIST = 6
57
63
  end
58
64
  end
59
65
  end
@@ -43,12 +43,12 @@ module Google
43
43
  #
44
44
  # The ResourceDescriptor Yaml config will look like:
45
45
  #
46
- # resources:
47
- # - type: "pubsub.googleapis.com/Topic"
48
- # name_descriptor:
49
- # - pattern: "projects/\\{project}/topics/\\{topic}"
50
- # parent_type: "cloudresourcemanager.googleapis.com/Project"
51
- # parent_name_extractor: "projects/\\{project}"
46
+ # resources:
47
+ # - type: "pubsub.googleapis.com/Topic"
48
+ # name_descriptor:
49
+ # - pattern: "projects/{project}/topics/{topic}"
50
+ # parent_type: "cloudresourcemanager.googleapis.com/Project"
51
+ # parent_name_extractor: "projects/{project}"
52
52
  #
53
53
  # Sometimes, resources have multiple patterns, typically because they can
54
54
  # live under multiple parents.
@@ -183,15 +183,24 @@ module Google
183
183
  # }
184
184
  # @!attribute [rw] plural
185
185
  # @return [::String]
186
- # The plural name used in the resource name, such as 'projects' for
187
- # the name of 'projects/\\{project}'. It is the same concept of the `plural`
188
- # field in k8s CRD spec
186
+ # The plural name used in the resource name and permission names, such as
187
+ # 'projects' for the resource name of 'projects/\\{project}' and the permission
188
+ # name of 'cloudresourcemanager.googleapis.com/projects.get'. It is the same
189
+ # concept of the `plural` field in k8s CRD spec
189
190
  # https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
191
+ #
192
+ # Note: The plural form is required even for singleton resources. See
193
+ # https://aip.dev/156
190
194
  # @!attribute [rw] singular
191
195
  # @return [::String]
192
196
  # The same concept of the `singular` field in k8s CRD spec
193
197
  # https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
194
198
  # Such as "project" for the `resourcemanager.googleapis.com/Project` type.
199
+ # @!attribute [rw] style
200
+ # @return [::Array<::Google::Api::ResourceDescriptor::Style>]
201
+ # Style flag(s) for this resource.
202
+ # These indicate that a resource is expected to conform to a given
203
+ # style. See the specific style flags for additional information.
195
204
  class ResourceDescriptor
196
205
  include ::Google::Protobuf::MessageExts
197
206
  extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -211,6 +220,22 @@ module Google
211
220
  # that from being necessary once there are multiple patterns.)
212
221
  FUTURE_MULTI_PATTERN = 2
213
222
  end
223
+
224
+ # A flag representing a specific style that a resource claims to conform to.
225
+ module Style
226
+ # The unspecified value. Do not use.
227
+ STYLE_UNSPECIFIED = 0
228
+
229
+ # This resource is intended to be "declarative-friendly".
230
+ #
231
+ # Declarative-friendly resources must be more strictly consistent, and
232
+ # setting this to true communicates to tools that this resource should
233
+ # adhere to declarative-friendly expectations.
234
+ #
235
+ # Note: This is used by the API linter (linter.aip.dev) to enable
236
+ # additional checks.
237
+ DECLARATIVE_FRIENDLY = 1
238
+ end
214
239
  end
215
240
 
216
241
  # Defines a proto annotation that describes a string field that refers to
@@ -226,6 +251,17 @@ module Google
226
251
  # type: "pubsub.googleapis.com/Topic"
227
252
  # }];
228
253
  # }
254
+ #
255
+ # Occasionally, a field may reference an arbitrary resource. In this case,
256
+ # APIs use the special value * in their resource reference.
257
+ #
258
+ # Example:
259
+ #
260
+ # message GetIamPolicyRequest {
261
+ # string resource = 2 [(google.api.resource_reference) = {
262
+ # type: "*"
263
+ # }];
264
+ # }
229
265
  # @!attribute [rw] child_type
230
266
  # @return [::String]
231
267
  # The resource type of a child collection that the annotated field
@@ -234,11 +270,11 @@ module Google
234
270
  #
235
271
  # Example:
236
272
  #
237
- # message ListLogEntriesRequest {
238
- # string parent = 1 [(google.api.resource_reference) = {
239
- # child_type: "logging.googleapis.com/LogEntry"
240
- # };
241
- # }
273
+ # message ListLogEntriesRequest {
274
+ # string parent = 1 [(google.api.resource_reference) = {
275
+ # child_type: "logging.googleapis.com/LogEntry"
276
+ # };
277
+ # }
242
278
  class ResourceReference
243
279
  include ::Google::Protobuf::MessageExts
244
280
  extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -80,20 +80,26 @@ module Google
80
80
  # Bounds: [0s, 1d].
81
81
  # @!attribute [rw] scale_up_factor
82
82
  # @return [::Float]
83
- # Required. Fraction of average pending memory in the last cooldown period
84
- # for which to add workers. A scale-up factor of 1.0 will result in scaling
85
- # up so that there is no pending memory remaining after the update (more
86
- # aggressive scaling). A scale-up factor closer to 0 will result in a smaller
87
- # magnitude of scaling up (less aggressive scaling).
83
+ # Required. Fraction of average YARN pending memory in the last cooldown
84
+ # period for which to add workers. A scale-up factor of 1.0 will result in
85
+ # scaling up so that there is no pending memory remaining after the update
86
+ # (more aggressive scaling). A scale-up factor closer to 0 will result in a
87
+ # smaller magnitude of scaling up (less aggressive scaling). See [How
88
+ # autoscaling
89
+ # works](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/autoscaling#how_autoscaling_works)
90
+ # for more information.
88
91
  #
89
92
  # Bounds: [0.0, 1.0].
90
93
  # @!attribute [rw] scale_down_factor
91
94
  # @return [::Float]
92
- # Required. Fraction of average pending memory in the last cooldown period
93
- # for which to remove workers. A scale-down factor of 1 will result in
95
+ # Required. Fraction of average YARN pending memory in the last cooldown
96
+ # period for which to remove workers. A scale-down factor of 1 will result in
94
97
  # scaling down so that there is no available memory remaining after the
95
98
  # update (more aggressive scaling). A scale-down factor of 0 disables
96
99
  # removing workers, which can be beneficial for autoscaling a single job.
100
+ # See [How autoscaling
101
+ # works](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/autoscaling#how_autoscaling_works)
102
+ # for more information.
97
103
  #
98
104
  # Bounds: [0.0, 1.0].
99
105
  # @!attribute [rw] scale_up_min_worker_fraction
@@ -85,6 +85,15 @@ module Google
85
85
  # and manage this project-level, per-location bucket (see
86
86
  # [Dataproc staging
87
87
  # bucket](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
88
+ # @!attribute [rw] temp_bucket
89
+ # @return [::String]
90
+ # Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs
91
+ # data, such as Spark and MapReduce history files. If you do not specify a
92
+ # temp bucket, Dataproc will determine a Cloud Storage location (US, ASIA, or
93
+ # EU) for your cluster's temp bucket according to the Compute Engine zone
94
+ # where your cluster is deployed, and then create and manage this
95
+ # project-level, per-location bucket. The default bucket has a TTL of 90
96
+ # days, but you can use any TTL (or none) if you specify a bucket.
88
97
  # @!attribute [rw] gce_cluster_config
89
98
  # @return [::Google::Cloud::Dataproc::V1beta2::GceClusterConfig]
90
99
  # Optional. The shared Compute Engine config settings for
@@ -137,10 +146,10 @@ module Google
137
146
  # Optional. Security related configuration.
138
147
  # @!attribute [rw] gke_cluster_config
139
148
  # @return [::Google::Cloud::Dataproc::V1beta2::GkeClusterConfig]
140
- # Optional. The Kubernetes Engine config for Dataproc clusters deployed to Kubernetes.
141
- # Setting this is considered mutually exclusive with Compute Engine-based
142
- # options such as `gce_cluster_config`, `master_config`, `worker_config`,
143
- # `secondary_worker_config`, and `autoscaling_config`.
149
+ # Optional. The Kubernetes Engine config for Dataproc clusters deployed to
150
+ # Kubernetes. Setting this is considered mutually exclusive with Compute
151
+ # Engine-based options such as `gce_cluster_config`, `master_config`,
152
+ # `worker_config`, `secondary_worker_config`, and `autoscaling_config`.
144
153
  class ClusterConfig
145
154
  include ::Google::Protobuf::MessageExts
146
155
  extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -268,7 +277,7 @@ module Google
268
277
  # @!attribute [rw] service_account
269
278
  # @return [::String]
270
279
  # Optional. The [Dataproc service
271
- # account](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#service_accounts_in_cloud_dataproc)
280
+ # account](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#service_accounts_in_dataproc)
272
281
  # (also see [VM Data Plane
273
282
  # identity](https://cloud.google.com/dataproc/docs/concepts/iam/dataproc-principals#vm_service_account_data_plane_identity))
274
283
  # used by Dataproc cluster VM instances to access Google Cloud Platform
@@ -372,6 +381,15 @@ module Google
372
381
  # @return [::Boolean]
373
382
  # Output only. Specifies that this instance group contains preemptible
374
383
  # instances.
384
+ # @!attribute [rw] preemptibility
385
+ # @return [::Google::Cloud::Dataproc::V1beta2::InstanceGroupConfig::Preemptibility]
386
+ # Optional. Specifies the preemptibility of the instance group.
387
+ #
388
+ # The default value for master and worker groups is
389
+ # `NON_PREEMPTIBLE`. This default cannot be changed.
390
+ #
391
+ # The default value for secondary instances is
392
+ # `PREEMPTIBLE`.
375
393
  # @!attribute [r] managed_group_config
376
394
  # @return [::Google::Cloud::Dataproc::V1beta2::ManagedGroupConfig]
377
395
  # Output only. The config for Compute Engine Instance Group
@@ -389,6 +407,27 @@ module Google
389
407
  class InstanceGroupConfig
390
408
  include ::Google::Protobuf::MessageExts
391
409
  extend ::Google::Protobuf::MessageExts::ClassMethods
410
+
411
+ # Controls the use of
412
+ # [preemptible instances]
413
+ # (https://cloud.google.com/compute/docs/instances/preemptible)
414
+ # within the group.
415
+ module Preemptibility
416
+ # Preemptibility is unspecified, the system will choose the
417
+ # appropriate setting for each instance group.
418
+ PREEMPTIBILITY_UNSPECIFIED = 0
419
+
420
+ # Instances are non-preemptible.
421
+ #
422
+ # This option is allowed for all instance groups and is the only valid
423
+ # value for Master and Worker instance groups.
424
+ NON_PREEMPTIBLE = 1
425
+
426
+ # Instances are preemptible.
427
+ #
428
+ # This option is allowed only for secondary worker groups.
429
+ PREEMPTIBLE = 2
430
+ end
392
431
  end
393
432
 
394
433
  # Specifies the resources used to actively manage an instance group.
@@ -464,7 +503,8 @@ module Google
464
503
  # [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json).
465
504
  # @!attribute [rw] auto_delete_time
466
505
  # @return [::Google::Protobuf::Timestamp]
467
- # Optional. The time when cluster will be auto-deleted. (see JSON representation of
506
+ # Optional. The time when cluster will be auto-deleted. (see JSON
507
+ # representation of
468
508
  # [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
469
509
  # @!attribute [rw] auto_delete_ttl
470
510
  # @return [::Google::Protobuf::Duration]
@@ -495,8 +535,8 @@ module Google
495
535
  # Specifies Kerberos related configuration.
496
536
  # @!attribute [rw] enable_kerberos
497
537
  # @return [::Boolean]
498
- # Optional. Flag to indicate whether to Kerberize the cluster (default: false). Set
499
- # this field to true to enable Kerberos on a cluster.
538
+ # Optional. Flag to indicate whether to Kerberize the cluster (default:
539
+ # false). Set this field to true to enable Kerberos on a cluster.
500
540
  # @!attribute [rw] root_principal_password_uri
501
541
  # @return [::String]
502
542
  # Required. The Cloud Storage URI of a KMS encrypted file containing the root
@@ -658,7 +698,7 @@ module Google
658
698
  # @return [::String]
659
699
  # Optional. The version of software inside the cluster. It must be one of the
660
700
  # supported [Dataproc
661
- # Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported_cloud_dataproc_versions),
701
+ # Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported_dataproc_versions),
662
702
  # such as "1.2" (including a subminor version, such as "1.2.29"), or the
663
703
  # ["preview"
664
704
  # version](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
@@ -747,10 +787,11 @@ module Google
747
787
  # @!attribute [rw] request_id
748
788
  # @return [::String]
749
789
  # Optional. A unique id used to identify the request. If the server
750
- # receives two {::Google::Cloud::Dataproc::V1beta2::CreateClusterRequest CreateClusterRequest} requests with the same
751
- # id, then the second request will be ignored and the
752
- # first {::Google::Longrunning::Operation google.longrunning.Operation} created and stored in the backend
753
- # is returned.
790
+ # receives two
791
+ # {::Google::Cloud::Dataproc::V1beta2::CreateClusterRequest CreateClusterRequest}
792
+ # requests with the same id, then the second request will be ignored and the
793
+ # first {::Google::Longrunning::Operation google.longrunning.Operation} created
794
+ # and stored in the backend is returned.
754
795
  #
755
796
  # It is recommended to always set this value to a
756
797
  # [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
@@ -852,10 +893,11 @@ module Google
852
893
  # @!attribute [rw] request_id
853
894
  # @return [::String]
854
895
  # Optional. A unique id used to identify the request. If the server
855
- # receives two {::Google::Cloud::Dataproc::V1beta2::UpdateClusterRequest UpdateClusterRequest} requests with the same
856
- # id, then the second request will be ignored and the
857
- # first {::Google::Longrunning::Operation google.longrunning.Operation} created and stored in the
858
- # backend is returned.
896
+ # receives two
897
+ # {::Google::Cloud::Dataproc::V1beta2::UpdateClusterRequest UpdateClusterRequest}
898
+ # requests with the same id, then the second request will be ignored and the
899
+ # first {::Google::Longrunning::Operation google.longrunning.Operation} created
900
+ # and stored in the backend is returned.
859
901
  #
860
902
  # It is recommended to always set this value to a
861
903
  # [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
@@ -885,10 +927,11 @@ module Google
885
927
  # @!attribute [rw] request_id
886
928
  # @return [::String]
887
929
  # Optional. A unique id used to identify the request. If the server
888
- # receives two {::Google::Cloud::Dataproc::V1beta2::DeleteClusterRequest DeleteClusterRequest} requests with the same
889
- # id, then the second request will be ignored and the
890
- # first {::Google::Longrunning::Operation google.longrunning.Operation} created and stored in the
891
- # backend is returned.
930
+ # receives two
931
+ # {::Google::Cloud::Dataproc::V1beta2::DeleteClusterRequest DeleteClusterRequest}
932
+ # requests with the same id, then the second request will be ignored and the
933
+ # first {::Google::Longrunning::Operation google.longrunning.Operation} created
934
+ # and stored in the backend is returned.
892
935
  #
893
936
  # It is recommended to always set this value to a
894
937
  # [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
@@ -158,12 +158,12 @@ module Google
158
158
  # Spark driver and tasks.
159
159
  # @!attribute [rw] file_uris
160
160
  # @return [::Array<::String>]
161
- # Optional. HCFS URIs of files to be copied to the working directory of
162
- # Spark drivers and distributed tasks. Useful for naively parallel tasks.
161
+ # Optional. HCFS URIs of files to be placed in the working directory of
162
+ # each executor. Useful for naively parallel tasks.
163
163
  # @!attribute [rw] archive_uris
164
164
  # @return [::Array<::String>]
165
- # Optional. HCFS URIs of archives to be extracted in the working directory
166
- # of Spark drivers and tasks. Supported file types:
165
+ # Optional. HCFS URIs of archives to be extracted into the working directory
166
+ # of each executor. Supported file types:
167
167
  # .jar, .tar, .tar.gz, .tgz, and .zip.
168
168
  # @!attribute [rw] properties
169
169
  # @return [::Google::Protobuf::Map{::String => ::String}]
@@ -211,11 +211,12 @@ module Google
211
211
  # Python driver and tasks.
212
212
  # @!attribute [rw] file_uris
213
213
  # @return [::Array<::String>]
214
- # Optional. HCFS URIs of files to be copied to the working directory of
215
- # Python drivers and distributed tasks. Useful for naively parallel tasks.
214
+ # Optional. HCFS URIs of files to be placed in the working directory of
215
+ # each executor. Useful for naively parallel tasks.
216
216
  # @!attribute [rw] archive_uris
217
217
  # @return [::Array<::String>]
218
- # Optional. HCFS URIs of archives to be extracted in the working directory of
218
+ # Optional. HCFS URIs of archives to be extracted into the working directory
219
+ # of each executor. Supported file types:
219
220
  # .jar, .tar, .tar.gz, .tgz, and .zip.
220
221
  # @!attribute [rw] properties
221
222
  # @return [::Google::Protobuf::Map{::String => ::String}]
@@ -426,12 +427,12 @@ module Google
426
427
  # occur that causes an incorrect job submission.
427
428
  # @!attribute [rw] file_uris
428
429
  # @return [::Array<::String>]
429
- # Optional. HCFS URIs of files to be copied to the working directory of
430
- # R drivers and distributed tasks. Useful for naively parallel tasks.
430
+ # Optional. HCFS URIs of files to be placed in the working directory of
431
+ # each executor. Useful for naively parallel tasks.
431
432
  # @!attribute [rw] archive_uris
432
433
  # @return [::Array<::String>]
433
- # Optional. HCFS URIs of archives to be extracted in the working directory of
434
- # Spark drivers and tasks. Supported file types:
434
+ # Optional. HCFS URIs of archives to be extracted into the working directory
435
+ # of each executor. Supported file types:
435
436
  # .jar, .tar, .tar.gz, .tgz, and .zip.
436
437
  # @!attribute [rw] properties
437
438
  # @return [::Google::Protobuf::Map{::String => ::String}]
@@ -600,8 +601,8 @@ module Google
600
601
  # Encapsulates the full scoping used to reference a job.
601
602
  # @!attribute [rw] project_id
602
603
  # @return [::String]
603
- # Required. The ID of the Google Cloud Platform project that the job
604
- # belongs to.
604
+ # Optional. The ID of the Google Cloud Platform project that the job belongs to. If
605
+ # specified, must match the request project ID.
605
606
  # @!attribute [rw] job_id
606
607
  # @return [::String]
607
608
  # Optional. The job ID, which must be unique within the project.
@@ -23,20 +23,17 @@ module Google
23
23
  module V1beta2
24
24
  # Cluster components that can be activated.
25
25
  module Component
26
- # Unspecified component.
26
+ # Unspecified component. Specifying this will cause Cluster creation to fail.
27
27
  COMPONENT_UNSPECIFIED = 0
28
28
 
29
29
  # The Anaconda python distribution.
30
30
  ANACONDA = 5
31
31
 
32
- # Docker
33
- DOCKER = 13
34
-
35
32
  # The Druid query engine.
36
33
  DRUID = 9
37
34
 
38
- # Flink
39
- FLINK = 14
35
+ # HBase.
36
+ HBASE = 11
40
37
 
41
38
  # The Hive Web HCatalog (the REST service for accessing HCatalog).
42
39
  HIVE_WEBHCAT = 3
@@ -85,6 +85,17 @@ module Google
85
85
  # Optional. Template parameters whose values are substituted into the
86
86
  # template. Values for parameters must be provided when the template is
87
87
  # instantiated.
88
+ # @!attribute [rw] dag_timeout
89
+ # @return [::Google::Protobuf::Duration]
90
+ # Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h",
91
+ # and "d" suffixes for second, minute, hour, and day duration values,
92
+ # respectively. The timeout duration must be from 10 minutes ("10m") to 24
93
+ # hours ("24h" or "1d"). The timer begins when the first job is submitted. If
94
+ # the workflow is running at the end of the timeout period, any remaining
95
+ # jobs are cancelled, the workflow is terminated, and if the workflow was
96
+ # running on a [managed
97
+ # cluster](https://cloud.google.com/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster),
98
+ # the cluster is deleted.
88
99
  class WorkflowTemplate
89
100
  include ::Google::Protobuf::MessageExts
90
101
  extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -188,30 +199,36 @@ module Google
188
199
  #
189
200
  # The step id is used as prefix for job id, as job
190
201
  # `goog-dataproc-workflow-step-id` label, and in
191
- # {::Google::Cloud::Dataproc::V1beta2::OrderedJob#prerequisite_step_ids prerequisiteStepIds} field from other
192
- # steps.
202
+ # {::Google::Cloud::Dataproc::V1beta2::OrderedJob#prerequisite_step_ids prerequisiteStepIds}
203
+ # field from other steps.
193
204
  #
194
205
  # The id must contain only letters (a-z, A-Z), numbers (0-9),
195
206
  # underscores (_), and hyphens (-). Cannot begin or end with underscore
196
207
  # or hyphen. Must consist of between 3 and 50 characters.
197
208
  # @!attribute [rw] hadoop_job
198
209
  # @return [::Google::Cloud::Dataproc::V1beta2::HadoopJob]
210
+ # Optional. Job is a Hadoop job.
199
211
  # @!attribute [rw] spark_job
200
212
  # @return [::Google::Cloud::Dataproc::V1beta2::SparkJob]
213
+ # Optional. Job is a Spark job.
201
214
  # @!attribute [rw] pyspark_job
202
215
  # @return [::Google::Cloud::Dataproc::V1beta2::PySparkJob]
216
+ # Optional. Job is a PySpark job.
203
217
  # @!attribute [rw] hive_job
204
218
  # @return [::Google::Cloud::Dataproc::V1beta2::HiveJob]
219
+ # Optional. Job is a Hive job.
205
220
  # @!attribute [rw] pig_job
206
221
  # @return [::Google::Cloud::Dataproc::V1beta2::PigJob]
222
+ # Optional. Job is a Pig job.
207
223
  # @!attribute [rw] spark_r_job
208
224
  # @return [::Google::Cloud::Dataproc::V1beta2::SparkRJob]
209
- # Spark R job
225
+ # Optional. Job is a SparkR job.
210
226
  # @!attribute [rw] spark_sql_job
211
227
  # @return [::Google::Cloud::Dataproc::V1beta2::SparkSqlJob]
228
+ # Optional. Job is a SparkSql job.
212
229
  # @!attribute [rw] presto_job
213
230
  # @return [::Google::Cloud::Dataproc::V1beta2::PrestoJob]
214
- # Presto job
231
+ # Optional. Job is a Presto job.
215
232
  # @!attribute [rw] labels
216
233
  # @return [::Google::Protobuf::Map{::String => ::String}]
217
234
  # Optional. The labels to associate with this job.
@@ -269,10 +286,10 @@ module Google
269
286
  # A field is allowed to appear in at most one parameter's list of field
270
287
  # paths.
271
288
  #
272
- # A field path is similar in syntax to a {::Google::Protobuf::FieldMask google.protobuf.FieldMask}.
273
- # For example, a field path that references the zone field of a workflow
274
- # template's cluster selector would be specified as
275
- # `placement.clusterSelector.zone`.
289
+ # A field path is similar in syntax to a
290
+ # {::Google::Protobuf::FieldMask google.protobuf.FieldMask}. For example, a
291
+ # field path that references the zone field of a workflow template's cluster
292
+ # selector would be specified as `placement.clusterSelector.zone`.
276
293
  #
277
294
  # Also, field paths can reference fields using the following syntax:
278
295
  #
@@ -397,6 +414,23 @@ module Google
397
414
  # @!attribute [r] cluster_uuid
398
415
  # @return [::String]
399
416
  # Output only. The UUID of target cluster.
417
+ # @!attribute [r] dag_timeout
418
+ # @return [::Google::Protobuf::Duration]
419
+ # Output only. The timeout duration for the DAG of jobs.
420
+ # Minimum timeout duration is 10 minutes and maximum is 24 hours, expressed
421
+ # as a
422
+ # [google.protobuf.Duration][https://developers.google.com/protocol-buffers/docs/proto3#json_mapping].
423
+ # For example, "1800" = 1800 seconds/30 minutes duration.
424
+ # @!attribute [r] dag_start_time
425
+ # @return [::Google::Protobuf::Timestamp]
426
+ # Output only. DAG start time, which is only set for workflows with
427
+ # {::Google::Cloud::Dataproc::V1beta2::WorkflowMetadata#dag_timeout dag_timeout}
428
+ # when the DAG begins.
429
+ # @!attribute [r] dag_end_time
430
+ # @return [::Google::Protobuf::Timestamp]
431
+ # Output only. DAG end time, which is only set for workflows with
432
+ # {::Google::Cloud::Dataproc::V1beta2::WorkflowMetadata#dag_timeout dag_timeout}
433
+ # when the DAG ends.
400
434
  class WorkflowMetadata
401
435
  include ::Google::Protobuf::MessageExts
402
436
  extend ::Google::Protobuf::MessageExts::ClassMethods