google-cloud-dataproc-v1 0.3.0 → 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/LICENSE.md +188 -190
- data/README.md +66 -2
- data/lib/google/cloud/dataproc/v1.rb +3 -0
- data/lib/google/cloud/dataproc/v1/autoscaling_policies_services_pb.rb +5 -5
- data/lib/google/cloud/dataproc/v1/autoscaling_policy_service/client.rb +26 -20
- data/lib/google/cloud/dataproc/v1/autoscaling_policy_service/paths.rb +1 -1
- data/lib/google/cloud/dataproc/v1/cluster_controller.rb +1 -0
- data/lib/google/cloud/dataproc/v1/cluster_controller/client.rb +251 -47
- data/lib/google/cloud/dataproc/v1/cluster_controller/operations.rb +96 -11
- data/lib/google/cloud/dataproc/v1/cluster_controller/paths.rb +71 -0
- data/lib/google/cloud/dataproc/v1/clusters_pb.rb +60 -0
- data/lib/google/cloud/dataproc/v1/clusters_services_pb.rb +10 -6
- data/lib/google/cloud/dataproc/v1/job_controller/client.rb +57 -49
- data/lib/google/cloud/dataproc/v1/job_controller/operations.rb +96 -11
- data/lib/google/cloud/dataproc/v1/jobs_pb.rb +2 -0
- data/lib/google/cloud/dataproc/v1/jobs_services_pb.rb +7 -7
- data/lib/google/cloud/dataproc/v1/shared_pb.rb +7 -0
- data/lib/google/cloud/dataproc/v1/version.rb +1 -1
- data/lib/google/cloud/dataproc/v1/workflow_template_service/client.rb +43 -36
- data/lib/google/cloud/dataproc/v1/workflow_template_service/operations.rb +96 -11
- data/lib/google/cloud/dataproc/v1/workflow_template_service/paths.rb +39 -1
- data/lib/google/cloud/dataproc/v1/workflow_templates_pb.rb +5 -0
- data/lib/google/cloud/dataproc/v1/workflow_templates_services_pb.rb +9 -8
- data/proto_docs/google/api/field_behavior.rb +6 -0
- data/proto_docs/google/api/resource.rb +50 -14
- data/proto_docs/google/cloud/dataproc/v1/clusters.rb +240 -16
- data/proto_docs/google/cloud/dataproc/v1/jobs.rb +29 -10
- data/proto_docs/google/cloud/dataproc/v1/shared.rb +24 -1
- data/proto_docs/google/cloud/dataproc/v1/workflow_templates.rb +34 -7
- data/proto_docs/google/longrunning/operations.rb +17 -3
- data/proto_docs/google/protobuf/any.rb +5 -2
- data/proto_docs/google/protobuf/timestamp.rb +10 -1
- metadata +13 -9
@@ -239,9 +239,9 @@ module Google
|
|
239
239
|
# A list of queries to run on a cluster.
|
240
240
|
# @!attribute [rw] queries
|
241
241
|
# @return [::Array<::String>]
|
242
|
-
# Required. The queries to execute. You do not need to
|
243
|
-
# with a semicolon. Multiple queries can be specified in one
|
244
|
-
# by separating each with a semicolon. Here is an example of
|
242
|
+
# Required. The queries to execute. You do not need to end a query expression
|
243
|
+
# with a semicolon. Multiple queries can be specified in one
|
244
|
+
# string by separating each with a semicolon. Here is an example of a
|
245
245
|
# Dataproc API snippet that uses a QueryList to specify a HiveJob:
|
246
246
|
#
|
247
247
|
# "hiveJob": {
|
@@ -505,9 +505,22 @@ module Google
|
|
505
505
|
# @return [::String]
|
506
506
|
# Output only. A cluster UUID generated by the Dataproc service when
|
507
507
|
# the job is submitted.
|
508
|
+
# @!attribute [rw] cluster_labels
|
509
|
+
# @return [::Google::Protobuf::Map{::String => ::String}]
|
510
|
+
# Optional. Cluster labels to identify a cluster where the job will be
|
511
|
+
# submitted.
|
508
512
|
class JobPlacement
|
509
513
|
include ::Google::Protobuf::MessageExts
|
510
514
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
515
|
+
|
516
|
+
# @!attribute [rw] key
|
517
|
+
# @return [::String]
|
518
|
+
# @!attribute [rw] value
|
519
|
+
# @return [::String]
|
520
|
+
class ClusterLabelsEntry
|
521
|
+
include ::Google::Protobuf::MessageExts
|
522
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
523
|
+
end
|
511
524
|
end
|
512
525
|
|
513
526
|
# Dataproc job status.
|
@@ -596,8 +609,8 @@ module Google
|
|
596
609
|
# Encapsulates the full scoping used to reference a job.
|
597
610
|
# @!attribute [rw] project_id
|
598
611
|
# @return [::String]
|
599
|
-
# Optional. The ID of the Google Cloud Platform project that the job belongs
|
600
|
-
# specified, must match the request project ID.
|
612
|
+
# Optional. The ID of the Google Cloud Platform project that the job belongs
|
613
|
+
# to. If specified, must match the request project ID.
|
601
614
|
# @!attribute [rw] job_id
|
602
615
|
# @return [::String]
|
603
616
|
# Optional. The job ID, which must be unique within the project.
|
@@ -744,8 +757,8 @@ module Google
|
|
744
757
|
# may be reused over time.
|
745
758
|
# @!attribute [r] done
|
746
759
|
# @return [::Boolean]
|
747
|
-
# Output only. Indicates whether the job is completed. If the value is
|
748
|
-
# the job is still in progress. If `true`, the job is completed, and
|
760
|
+
# Output only. Indicates whether the job is completed. If the value is
|
761
|
+
# `false`, the job is still in progress. If `true`, the job is completed, and
|
749
762
|
# `status.state` field will indicate if it was successful, failed,
|
750
763
|
# or cancelled.
|
751
764
|
class Job
|
@@ -766,13 +779,18 @@ module Google
|
|
766
779
|
# @!attribute [rw] max_failures_per_hour
|
767
780
|
# @return [::Integer]
|
768
781
|
# Optional. Maximum number of times per hour a driver may be restarted as
|
769
|
-
# a result of driver
|
782
|
+
# a result of driver exiting with non-zero code before job is
|
770
783
|
# reported failed.
|
771
784
|
#
|
772
785
|
# A job may be reported as thrashing if driver exits with non-zero code
|
773
786
|
# 4 times within 10 minute window.
|
774
787
|
#
|
775
788
|
# Maximum value is 10.
|
789
|
+
# @!attribute [rw] max_failures_total
|
790
|
+
# @return [::Integer]
|
791
|
+
# Optional. Maximum number of times in total a driver may be restarted as a
|
792
|
+
# result of driver exiting with non-zero code before job is reported failed.
|
793
|
+
# Maximum value is 240.
|
776
794
|
class JobScheduling
|
777
795
|
include ::Google::Protobuf::MessageExts
|
778
796
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -792,8 +810,9 @@ module Google
|
|
792
810
|
# @!attribute [rw] request_id
|
793
811
|
# @return [::String]
|
794
812
|
# Optional. A unique id used to identify the request. If the server
|
795
|
-
# receives two
|
796
|
-
#
|
813
|
+
# receives two
|
814
|
+
# [SubmitJobRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.SubmitJobRequest)s
|
815
|
+
# with the same id, then the second request will be ignored and the
|
797
816
|
# first {::Google::Cloud::Dataproc::V1::Job Job} created and stored in the backend
|
798
817
|
# is returned.
|
799
818
|
#
|
@@ -22,13 +22,30 @@ module Google
|
|
22
22
|
module Dataproc
|
23
23
|
module V1
|
24
24
|
# Cluster components that can be activated.
|
25
|
+
# Next ID: 16.
|
25
26
|
module Component
|
26
27
|
# Unspecified component. Specifying this will cause Cluster creation to fail.
|
27
28
|
COMPONENT_UNSPECIFIED = 0
|
28
29
|
|
29
|
-
# The Anaconda python distribution.
|
30
|
+
# The Anaconda python distribution. The Anaconda component is not supported
|
31
|
+
# in the Dataproc
|
32
|
+
# <a
|
33
|
+
# href="/dataproc/docs/concepts/versioning/dataproc-release-2.0">2.0
|
34
|
+
# image</a>. The 2.0 image is pre-installed with Miniconda.
|
30
35
|
ANACONDA = 5
|
31
36
|
|
37
|
+
# Docker
|
38
|
+
DOCKER = 13
|
39
|
+
|
40
|
+
# The Druid query engine. (alpha)
|
41
|
+
DRUID = 9
|
42
|
+
|
43
|
+
# Flink
|
44
|
+
FLINK = 14
|
45
|
+
|
46
|
+
# HBase. (beta)
|
47
|
+
HBASE = 11
|
48
|
+
|
32
49
|
# The Hive Web HCatalog (the REST service for accessing HCatalog).
|
33
50
|
HIVE_WEBHCAT = 3
|
34
51
|
|
@@ -38,6 +55,12 @@ module Google
|
|
38
55
|
# The Presto query engine.
|
39
56
|
PRESTO = 6
|
40
57
|
|
58
|
+
# The Ranger service.
|
59
|
+
RANGER = 12
|
60
|
+
|
61
|
+
# The Solr service.
|
62
|
+
SOLR = 10
|
63
|
+
|
41
64
|
# The Zeppelin notebook.
|
42
65
|
ZEPPELIN = 4
|
43
66
|
|
@@ -78,6 +78,18 @@ module Google
|
|
78
78
|
# Optional. Template parameters whose values are substituted into the
|
79
79
|
# template. Values for parameters must be provided when the template is
|
80
80
|
# instantiated.
|
81
|
+
# @!attribute [rw] dag_timeout
|
82
|
+
# @return [::Google::Protobuf::Duration]
|
83
|
+
# Optional. Timeout duration for the DAG of jobs, expressed in seconds (see
|
84
|
+
# [JSON representation of
|
85
|
+
# duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
|
86
|
+
# The timeout duration must be from 10 minutes ("600s") to 24 hours
|
87
|
+
# ("86400s"). The timer begins when the first job is submitted. If the
|
88
|
+
# workflow is running at the end of the timeout period, any remaining jobs
|
89
|
+
# are cancelled, the workflow is ended, and if the workflow was running on a
|
90
|
+
# [managed
|
91
|
+
# cluster](/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster),
|
92
|
+
# the cluster is deleted.
|
81
93
|
class WorkflowTemplate
|
82
94
|
include ::Google::Protobuf::MessageExts
|
83
95
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -181,8 +193,8 @@ module Google
|
|
181
193
|
#
|
182
194
|
# The step id is used as prefix for job id, as job
|
183
195
|
# `goog-dataproc-workflow-step-id` label, and in
|
184
|
-
# {::Google::Cloud::Dataproc::V1::OrderedJob#prerequisite_step_ids prerequisiteStepIds}
|
185
|
-
# steps.
|
196
|
+
# {::Google::Cloud::Dataproc::V1::OrderedJob#prerequisite_step_ids prerequisiteStepIds}
|
197
|
+
# field from other steps.
|
186
198
|
#
|
187
199
|
# The id must contain only letters (a-z, A-Z), numbers (0-9),
|
188
200
|
# underscores (_), and hyphens (-). Cannot begin or end with underscore
|
@@ -268,10 +280,10 @@ module Google
|
|
268
280
|
# A field is allowed to appear in at most one parameter's list of field
|
269
281
|
# paths.
|
270
282
|
#
|
271
|
-
# A field path is similar in syntax to a
|
272
|
-
# For example, a
|
273
|
-
#
|
274
|
-
# `placement.clusterSelector.zone`.
|
283
|
+
# A field path is similar in syntax to a
|
284
|
+
# {::Google::Protobuf::FieldMask google.protobuf.FieldMask}. For example, a
|
285
|
+
# field path that references the zone field of a workflow template's cluster
|
286
|
+
# selector would be specified as `placement.clusterSelector.zone`.
|
275
287
|
#
|
276
288
|
# Also, field paths can reference fields using the following syntax:
|
277
289
|
#
|
@@ -396,6 +408,21 @@ module Google
|
|
396
408
|
# @!attribute [r] cluster_uuid
|
397
409
|
# @return [::String]
|
398
410
|
# Output only. The UUID of target cluster.
|
411
|
+
# @!attribute [r] dag_timeout
|
412
|
+
# @return [::Google::Protobuf::Duration]
|
413
|
+
# Output only. The timeout duration for the DAG of jobs, expressed in seconds
|
414
|
+
# (see [JSON representation of
|
415
|
+
# duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
|
416
|
+
# @!attribute [r] dag_start_time
|
417
|
+
# @return [::Google::Protobuf::Timestamp]
|
418
|
+
# Output only. DAG start time, only set for workflows with
|
419
|
+
# {::Google::Cloud::Dataproc::V1::WorkflowMetadata#dag_timeout dag_timeout} when
|
420
|
+
# DAG begins.
|
421
|
+
# @!attribute [r] dag_end_time
|
422
|
+
# @return [::Google::Protobuf::Timestamp]
|
423
|
+
# Output only. DAG end time, only set for workflows with
|
424
|
+
# {::Google::Cloud::Dataproc::V1::WorkflowMetadata#dag_timeout dag_timeout} when
|
425
|
+
# DAG ends.
|
399
426
|
class WorkflowMetadata
|
400
427
|
include ::Google::Protobuf::MessageExts
|
401
428
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -572,7 +599,7 @@ module Google
|
|
572
599
|
# @!attribute [rw] parameters
|
573
600
|
# @return [::Google::Protobuf::Map{::String => ::String}]
|
574
601
|
# Optional. Map from parameter names to values that should be used for those
|
575
|
-
# parameters. Values may not exceed
|
602
|
+
# parameters. Values may not exceed 1000 characters.
|
576
603
|
class InstantiateWorkflowTemplateRequest
|
577
604
|
include ::Google::Protobuf::MessageExts
|
578
605
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -25,7 +25,7 @@ module Google
|
|
25
25
|
# @return [::String]
|
26
26
|
# The server-assigned name, which is only unique within the same service that
|
27
27
|
# originally returns it. If you use the default HTTP mapping, the
|
28
|
-
# `name` should
|
28
|
+
# `name` should be a resource name ending with `operations/{unique_id}`.
|
29
29
|
# @!attribute [rw] metadata
|
30
30
|
# @return [::Google::Protobuf::Any]
|
31
31
|
# Service-specific metadata associated with the operation. It typically
|
@@ -35,7 +35,7 @@ module Google
|
|
35
35
|
# @!attribute [rw] done
|
36
36
|
# @return [::Boolean]
|
37
37
|
# If the value is `false`, it means the operation is still in progress.
|
38
|
-
# If true
|
38
|
+
# If `true`, the operation is completed, and either `error` or `response` is
|
39
39
|
# available.
|
40
40
|
# @!attribute [rw] error
|
41
41
|
# @return [::Google::Rpc::Status]
|
@@ -67,7 +67,7 @@ module Google
|
|
67
67
|
# The request message for Operations.ListOperations.
|
68
68
|
# @!attribute [rw] name
|
69
69
|
# @return [::String]
|
70
|
-
# The name of the operation
|
70
|
+
# The name of the operation's parent resource.
|
71
71
|
# @!attribute [rw] filter
|
72
72
|
# @return [::String]
|
73
73
|
# The standard list filter.
|
@@ -112,6 +112,20 @@ module Google
|
|
112
112
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
113
113
|
end
|
114
114
|
|
115
|
+
# The request message for Operations.WaitOperation.
|
116
|
+
# @!attribute [rw] name
|
117
|
+
# @return [::String]
|
118
|
+
# The name of the operation resource to wait on.
|
119
|
+
# @!attribute [rw] timeout
|
120
|
+
# @return [::Google::Protobuf::Duration]
|
121
|
+
# The maximum duration to wait before timing out. If left blank, the wait
|
122
|
+
# will be at most the time permitted by the underlying HTTP/RPC protocol.
|
123
|
+
# If RPC context deadline is also specified, the shorter one will be used.
|
124
|
+
class WaitOperationRequest
|
125
|
+
include ::Google::Protobuf::MessageExts
|
126
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
127
|
+
end
|
128
|
+
|
115
129
|
# A message representing the message types used by a long-running operation.
|
116
130
|
#
|
117
131
|
# Example:
|
@@ -57,10 +57,13 @@ module Google
|
|
57
57
|
# Example 4: Pack and unpack a message in Go
|
58
58
|
#
|
59
59
|
# foo := &pb.Foo{...}
|
60
|
-
# any, err :=
|
60
|
+
# any, err := anypb.New(foo)
|
61
|
+
# if err != nil {
|
62
|
+
# ...
|
63
|
+
# }
|
61
64
|
# ...
|
62
65
|
# foo := &pb.Foo{}
|
63
|
-
# if err :=
|
66
|
+
# if err := any.UnmarshalTo(foo); err != nil {
|
64
67
|
# ...
|
65
68
|
# }
|
66
69
|
#
|
@@ -70,7 +70,16 @@ module Google
|
|
70
70
|
# .setNanos((int) ((millis % 1000) * 1000000)).build();
|
71
71
|
#
|
72
72
|
#
|
73
|
-
# Example 5: Compute Timestamp from
|
73
|
+
# Example 5: Compute Timestamp from Java `Instant.now()`.
|
74
|
+
#
|
75
|
+
# Instant now = Instant.now();
|
76
|
+
#
|
77
|
+
# Timestamp timestamp =
|
78
|
+
# Timestamp.newBuilder().setSeconds(now.getEpochSecond())
|
79
|
+
# .setNanos(now.getNano()).build();
|
80
|
+
#
|
81
|
+
#
|
82
|
+
# Example 6: Compute Timestamp from current time in Python.
|
74
83
|
#
|
75
84
|
# timestamp = Timestamp()
|
76
85
|
# timestamp.GetCurrentTime()
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: google-cloud-dataproc-v1
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Google LLC
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-04-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gapic-common
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '0.
|
19
|
+
version: '0.4'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '0.
|
26
|
+
version: '0.4'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: google-cloud-errors
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 1.
|
47
|
+
version: 1.25.1
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 1.
|
54
|
+
version: 1.25.1
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: minitest
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -150,7 +150,10 @@ dependencies:
|
|
150
150
|
- - "~>"
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0.9'
|
153
|
-
description: Manages Hadoop-based clusters and jobs on Google Cloud Platform.
|
153
|
+
description: Manages Hadoop-based clusters and jobs on Google Cloud Platform. Note
|
154
|
+
that google-cloud-dataproc-v1 is a version-specific client library. For most uses,
|
155
|
+
we recommend installing the main client library google-cloud-dataproc instead. See
|
156
|
+
the readme for more details.
|
154
157
|
email: googleapis-packages@google.com
|
155
158
|
executables: []
|
156
159
|
extensions: []
|
@@ -172,6 +175,7 @@ files:
|
|
172
175
|
- lib/google/cloud/dataproc/v1/cluster_controller/client.rb
|
173
176
|
- lib/google/cloud/dataproc/v1/cluster_controller/credentials.rb
|
174
177
|
- lib/google/cloud/dataproc/v1/cluster_controller/operations.rb
|
178
|
+
- lib/google/cloud/dataproc/v1/cluster_controller/paths.rb
|
175
179
|
- lib/google/cloud/dataproc/v1/clusters_pb.rb
|
176
180
|
- lib/google/cloud/dataproc/v1/clusters_services_pb.rb
|
177
181
|
- lib/google/cloud/dataproc/v1/job_controller.rb
|
@@ -218,14 +222,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
218
222
|
requirements:
|
219
223
|
- - ">="
|
220
224
|
- !ruby/object:Gem::Version
|
221
|
-
version: '2.
|
225
|
+
version: '2.5'
|
222
226
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
223
227
|
requirements:
|
224
228
|
- - ">="
|
225
229
|
- !ruby/object:Gem::Version
|
226
230
|
version: '0'
|
227
231
|
requirements: []
|
228
|
-
rubygems_version: 3.
|
232
|
+
rubygems_version: 3.2.13
|
229
233
|
signing_key:
|
230
234
|
specification_version: 4
|
231
235
|
summary: API Client library for the Cloud Dataproc V1 API
|