google-cloud-dataproc-v1 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/google/cloud/dataproc/v1/cluster_controller.rb +1 -0
- data/lib/google/cloud/dataproc/v1/cluster_controller/client.rb +205 -7
- data/lib/google/cloud/dataproc/v1/cluster_controller/paths.rb +71 -0
- data/lib/google/cloud/dataproc/v1/clusters_pb.rb +60 -0
- data/lib/google/cloud/dataproc/v1/clusters_services_pb.rb +4 -0
- data/lib/google/cloud/dataproc/v1/job_controller/client.rb +6 -4
- data/lib/google/cloud/dataproc/v1/jobs_pb.rb +2 -0
- data/lib/google/cloud/dataproc/v1/shared_pb.rb +7 -0
- data/lib/google/cloud/dataproc/v1/version.rb +1 -1
- data/lib/google/cloud/dataproc/v1/workflow_template_service/client.rb +3 -2
- data/lib/google/cloud/dataproc/v1/workflow_template_service/paths.rb +38 -0
- data/lib/google/cloud/dataproc/v1/workflow_templates_pb.rb +5 -0
- data/lib/google/cloud/dataproc/v1/workflow_templates_services_pb.rb +2 -1
- data/proto_docs/google/cloud/dataproc/v1/clusters.rb +240 -16
- data/proto_docs/google/cloud/dataproc/v1/jobs.rb +29 -10
- data/proto_docs/google/cloud/dataproc/v1/shared.rb +24 -1
- data/proto_docs/google/cloud/dataproc/v1/workflow_templates.rb +34 -7
- metadata +5 -4
@@ -239,9 +239,9 @@ module Google
|
|
239
239
|
# A list of queries to run on a cluster.
|
240
240
|
# @!attribute [rw] queries
|
241
241
|
# @return [::Array<::String>]
|
242
|
-
# Required. The queries to execute. You do not need to
|
243
|
-
# with a semicolon. Multiple queries can be specified in one
|
244
|
-
# by separating each with a semicolon. Here is an example of
|
242
|
+
# Required. The queries to execute. You do not need to end a query expression
|
243
|
+
# with a semicolon. Multiple queries can be specified in one
|
244
|
+
# string by separating each with a semicolon. Here is an example of a
|
245
245
|
# Dataproc API snippet that uses a QueryList to specify a HiveJob:
|
246
246
|
#
|
247
247
|
# "hiveJob": {
|
@@ -505,9 +505,22 @@ module Google
|
|
505
505
|
# @return [::String]
|
506
506
|
# Output only. A cluster UUID generated by the Dataproc service when
|
507
507
|
# the job is submitted.
|
508
|
+
# @!attribute [rw] cluster_labels
|
509
|
+
# @return [::Google::Protobuf::Map{::String => ::String}]
|
510
|
+
# Optional. Cluster labels to identify a cluster where the job will be
|
511
|
+
# submitted.
|
508
512
|
class JobPlacement
|
509
513
|
include ::Google::Protobuf::MessageExts
|
510
514
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
515
|
+
|
516
|
+
# @!attribute [rw] key
|
517
|
+
# @return [::String]
|
518
|
+
# @!attribute [rw] value
|
519
|
+
# @return [::String]
|
520
|
+
class ClusterLabelsEntry
|
521
|
+
include ::Google::Protobuf::MessageExts
|
522
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
523
|
+
end
|
511
524
|
end
|
512
525
|
|
513
526
|
# Dataproc job status.
|
@@ -596,8 +609,8 @@ module Google
|
|
596
609
|
# Encapsulates the full scoping used to reference a job.
|
597
610
|
# @!attribute [rw] project_id
|
598
611
|
# @return [::String]
|
599
|
-
# Optional. The ID of the Google Cloud Platform project that the job belongs
|
600
|
-
# specified, must match the request project ID.
|
612
|
+
# Optional. The ID of the Google Cloud Platform project that the job belongs
|
613
|
+
# to. If specified, must match the request project ID.
|
601
614
|
# @!attribute [rw] job_id
|
602
615
|
# @return [::String]
|
603
616
|
# Optional. The job ID, which must be unique within the project.
|
@@ -744,8 +757,8 @@ module Google
|
|
744
757
|
# may be reused over time.
|
745
758
|
# @!attribute [r] done
|
746
759
|
# @return [::Boolean]
|
747
|
-
# Output only. Indicates whether the job is completed. If the value is
|
748
|
-
# the job is still in progress. If `true`, the job is completed, and
|
760
|
+
# Output only. Indicates whether the job is completed. If the value is
|
761
|
+
# `false`, the job is still in progress. If `true`, the job is completed, and
|
749
762
|
# `status.state` field will indicate if it was successful, failed,
|
750
763
|
# or cancelled.
|
751
764
|
class Job
|
@@ -766,13 +779,18 @@ module Google
|
|
766
779
|
# @!attribute [rw] max_failures_per_hour
|
767
780
|
# @return [::Integer]
|
768
781
|
# Optional. Maximum number of times per hour a driver may be restarted as
|
769
|
-
# a result of driver
|
782
|
+
# a result of driver exiting with non-zero code before job is
|
770
783
|
# reported failed.
|
771
784
|
#
|
772
785
|
# A job may be reported as thrashing if driver exits with non-zero code
|
773
786
|
# 4 times within 10 minute window.
|
774
787
|
#
|
775
788
|
# Maximum value is 10.
|
789
|
+
# @!attribute [rw] max_failures_total
|
790
|
+
# @return [::Integer]
|
791
|
+
# Optional. Maximum number of times in total a driver may be restarted as a
|
792
|
+
# result of driver exiting with non-zero code before job is reported failed.
|
793
|
+
# Maximum value is 240.
|
776
794
|
class JobScheduling
|
777
795
|
include ::Google::Protobuf::MessageExts
|
778
796
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -792,8 +810,9 @@ module Google
|
|
792
810
|
# @!attribute [rw] request_id
|
793
811
|
# @return [::String]
|
794
812
|
# Optional. A unique id used to identify the request. If the server
|
795
|
-
# receives two
|
796
|
-
#
|
813
|
+
# receives two
|
814
|
+
# [SubmitJobRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.SubmitJobRequest)s
|
815
|
+
# with the same id, then the second request will be ignored and the
|
797
816
|
# first {::Google::Cloud::Dataproc::V1::Job Job} created and stored in the backend
|
798
817
|
# is returned.
|
799
818
|
#
|
@@ -22,13 +22,30 @@ module Google
|
|
22
22
|
module Dataproc
|
23
23
|
module V1
|
24
24
|
# Cluster components that can be activated.
|
25
|
+
# Next ID: 16.
|
25
26
|
module Component
|
26
27
|
# Unspecified component. Specifying this will cause Cluster creation to fail.
|
27
28
|
COMPONENT_UNSPECIFIED = 0
|
28
29
|
|
29
|
-
# The Anaconda python distribution.
|
30
|
+
# The Anaconda python distribution. The Anaconda component is not supported
|
31
|
+
# in the Dataproc
|
32
|
+
# <a
|
33
|
+
# href="/dataproc/docs/concepts/versioning/dataproc-release-2.0">2.0
|
34
|
+
# image</a>. The 2.0 image is pre-installed with Miniconda.
|
30
35
|
ANACONDA = 5
|
31
36
|
|
37
|
+
# Docker
|
38
|
+
DOCKER = 13
|
39
|
+
|
40
|
+
# The Druid query engine. (alpha)
|
41
|
+
DRUID = 9
|
42
|
+
|
43
|
+
# Flink
|
44
|
+
FLINK = 14
|
45
|
+
|
46
|
+
# HBase. (beta)
|
47
|
+
HBASE = 11
|
48
|
+
|
32
49
|
# The Hive Web HCatalog (the REST service for accessing HCatalog).
|
33
50
|
HIVE_WEBHCAT = 3
|
34
51
|
|
@@ -38,6 +55,12 @@ module Google
|
|
38
55
|
# The Presto query engine.
|
39
56
|
PRESTO = 6
|
40
57
|
|
58
|
+
# The Ranger service.
|
59
|
+
RANGER = 12
|
60
|
+
|
61
|
+
# The Solr service.
|
62
|
+
SOLR = 10
|
63
|
+
|
41
64
|
# The Zeppelin notebook.
|
42
65
|
ZEPPELIN = 4
|
43
66
|
|
@@ -78,6 +78,18 @@ module Google
|
|
78
78
|
# Optional. Template parameters whose values are substituted into the
|
79
79
|
# template. Values for parameters must be provided when the template is
|
80
80
|
# instantiated.
|
81
|
+
# @!attribute [rw] dag_timeout
|
82
|
+
# @return [::Google::Protobuf::Duration]
|
83
|
+
# Optional. Timeout duration for the DAG of jobs, expressed in seconds (see
|
84
|
+
# [JSON representation of
|
85
|
+
# duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
|
86
|
+
# The timeout duration must be from 10 minutes ("600s") to 24 hours
|
87
|
+
# ("86400s"). The timer begins when the first job is submitted. If the
|
88
|
+
# workflow is running at the end of the timeout period, any remaining jobs
|
89
|
+
# are cancelled, the workflow is ended, and if the workflow was running on a
|
90
|
+
# [managed
|
91
|
+
# cluster](/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster),
|
92
|
+
# the cluster is deleted.
|
81
93
|
class WorkflowTemplate
|
82
94
|
include ::Google::Protobuf::MessageExts
|
83
95
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -181,8 +193,8 @@ module Google
|
|
181
193
|
#
|
182
194
|
# The step id is used as prefix for job id, as job
|
183
195
|
# `goog-dataproc-workflow-step-id` label, and in
|
184
|
-
# {::Google::Cloud::Dataproc::V1::OrderedJob#prerequisite_step_ids prerequisiteStepIds}
|
185
|
-
# steps.
|
196
|
+
# {::Google::Cloud::Dataproc::V1::OrderedJob#prerequisite_step_ids prerequisiteStepIds}
|
197
|
+
# field from other steps.
|
186
198
|
#
|
187
199
|
# The id must contain only letters (a-z, A-Z), numbers (0-9),
|
188
200
|
# underscores (_), and hyphens (-). Cannot begin or end with underscore
|
@@ -268,10 +280,10 @@ module Google
|
|
268
280
|
# A field is allowed to appear in at most one parameter's list of field
|
269
281
|
# paths.
|
270
282
|
#
|
271
|
-
# A field path is similar in syntax to a
|
272
|
-
# For example, a
|
273
|
-
#
|
274
|
-
# `placement.clusterSelector.zone`.
|
283
|
+
# A field path is similar in syntax to a
|
284
|
+
# {::Google::Protobuf::FieldMask google.protobuf.FieldMask}. For example, a
|
285
|
+
# field path that references the zone field of a workflow template's cluster
|
286
|
+
# selector would be specified as `placement.clusterSelector.zone`.
|
275
287
|
#
|
276
288
|
# Also, field paths can reference fields using the following syntax:
|
277
289
|
#
|
@@ -396,6 +408,21 @@ module Google
|
|
396
408
|
# @!attribute [r] cluster_uuid
|
397
409
|
# @return [::String]
|
398
410
|
# Output only. The UUID of target cluster.
|
411
|
+
# @!attribute [r] dag_timeout
|
412
|
+
# @return [::Google::Protobuf::Duration]
|
413
|
+
# Output only. The timeout duration for the DAG of jobs, expressed in seconds
|
414
|
+
# (see [JSON representation of
|
415
|
+
# duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
|
416
|
+
# @!attribute [r] dag_start_time
|
417
|
+
# @return [::Google::Protobuf::Timestamp]
|
418
|
+
# Output only. DAG start time, only set for workflows with
|
419
|
+
# {::Google::Cloud::Dataproc::V1::WorkflowMetadata#dag_timeout dag_timeout} when
|
420
|
+
# DAG begins.
|
421
|
+
# @!attribute [r] dag_end_time
|
422
|
+
# @return [::Google::Protobuf::Timestamp]
|
423
|
+
# Output only. DAG end time, only set for workflows with
|
424
|
+
# {::Google::Cloud::Dataproc::V1::WorkflowMetadata#dag_timeout dag_timeout} when
|
425
|
+
# DAG ends.
|
399
426
|
class WorkflowMetadata
|
400
427
|
include ::Google::Protobuf::MessageExts
|
401
428
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -572,7 +599,7 @@ module Google
|
|
572
599
|
# @!attribute [rw] parameters
|
573
600
|
# @return [::Google::Protobuf::Map{::String => ::String}]
|
574
601
|
# Optional. Map from parameter names to values that should be used for those
|
575
|
-
# parameters. Values may not exceed
|
602
|
+
# parameters. Values may not exceed 1000 characters.
|
576
603
|
class InstantiateWorkflowTemplateRequest
|
577
604
|
include ::Google::Protobuf::MessageExts
|
578
605
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: google-cloud-dataproc-v1
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Google LLC
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-04-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gapic-common
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '0.
|
19
|
+
version: '0.4'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '0.
|
26
|
+
version: '0.4'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: google-cloud-errors
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -175,6 +175,7 @@ files:
|
|
175
175
|
- lib/google/cloud/dataproc/v1/cluster_controller/client.rb
|
176
176
|
- lib/google/cloud/dataproc/v1/cluster_controller/credentials.rb
|
177
177
|
- lib/google/cloud/dataproc/v1/cluster_controller/operations.rb
|
178
|
+
- lib/google/cloud/dataproc/v1/cluster_controller/paths.rb
|
178
179
|
- lib/google/cloud/dataproc/v1/clusters_pb.rb
|
179
180
|
- lib/google/cloud/dataproc/v1/clusters_services_pb.rb
|
180
181
|
- lib/google/cloud/dataproc/v1/job_controller.rb
|