google-cloud-dataproc-v1 0.2.3 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.md +188 -190
  3. data/README.md +66 -2
  4. data/lib/google/cloud/dataproc/v1.rb +3 -0
  5. data/lib/google/cloud/dataproc/v1/autoscaling_policies_services_pb.rb +5 -5
  6. data/lib/google/cloud/dataproc/v1/autoscaling_policy_service/client.rb +26 -20
  7. data/lib/google/cloud/dataproc/v1/autoscaling_policy_service/paths.rb +1 -1
  8. data/lib/google/cloud/dataproc/v1/cluster_controller/client.rb +46 -40
  9. data/lib/google/cloud/dataproc/v1/cluster_controller/operations.rb +96 -11
  10. data/lib/google/cloud/dataproc/v1/clusters_pb.rb +14 -0
  11. data/lib/google/cloud/dataproc/v1/clusters_services_pb.rb +6 -6
  12. data/lib/google/cloud/dataproc/v1/job_controller/client.rb +51 -45
  13. data/lib/google/cloud/dataproc/v1/job_controller/operations.rb +96 -11
  14. data/lib/google/cloud/dataproc/v1/jobs_services_pb.rb +7 -7
  15. data/lib/google/cloud/dataproc/v1/version.rb +1 -1
  16. data/lib/google/cloud/dataproc/v1/workflow_template_service/client.rb +40 -34
  17. data/lib/google/cloud/dataproc/v1/workflow_template_service/operations.rb +96 -11
  18. data/lib/google/cloud/dataproc/v1/workflow_template_service/paths.rb +1 -1
  19. data/lib/google/cloud/dataproc/v1/workflow_templates_services_pb.rb +7 -7
  20. data/proto_docs/google/api/field_behavior.rb +6 -0
  21. data/proto_docs/google/api/resource.rb +50 -14
  22. data/proto_docs/google/cloud/dataproc/v1/autoscaling_policies.rb +8 -2
  23. data/proto_docs/google/cloud/dataproc/v1/clusters.rb +69 -2
  24. data/proto_docs/google/cloud/dataproc/v1/jobs.rb +14 -13
  25. data/proto_docs/google/cloud/dataproc/v1/shared.rb +1 -1
  26. data/proto_docs/google/cloud/dataproc/v1/workflow_templates.rb +9 -3
  27. data/proto_docs/google/longrunning/operations.rb +17 -3
  28. data/proto_docs/google/protobuf/any.rb +5 -2
  29. data/proto_docs/google/protobuf/timestamp.rb +10 -1
  30. metadata +12 -9
@@ -80,20 +80,26 @@ module Google
80
80
  # Bounds: [0s, 1d].
81
81
  # @!attribute [rw] scale_up_factor
82
82
  # @return [::Float]
83
- # Required. Fraction of average pending memory in the last cooldown period
83
+ # Required. Fraction of average YARN pending memory in the last cooldown period
84
84
  # for which to add workers. A scale-up factor of 1.0 will result in scaling
85
85
  # up so that there is no pending memory remaining after the update (more
86
86
  # aggressive scaling). A scale-up factor closer to 0 will result in a smaller
87
87
  # magnitude of scaling up (less aggressive scaling).
88
+ # See [How autoscaling
89
+ # works](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/autoscaling#how_autoscaling_works)
90
+ # for more information.
88
91
  #
89
92
  # Bounds: [0.0, 1.0].
90
93
  # @!attribute [rw] scale_down_factor
91
94
  # @return [::Float]
92
- # Required. Fraction of average pending memory in the last cooldown period
95
+ # Required. Fraction of average YARN pending memory in the last cooldown period
93
96
  # for which to remove workers. A scale-down factor of 1 will result in
94
97
  # scaling down so that there is no available memory remaining after the
95
98
  # update (more aggressive scaling). A scale-down factor of 0 disables
96
99
  # removing workers, which can be beneficial for autoscaling a single job.
100
+ # See [How autoscaling
101
+ # works](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/autoscaling#how_autoscaling_works)
102
+ # for more information.
97
103
  #
98
104
  # Bounds: [0.0, 1.0].
99
105
  # @!attribute [rw] scale_up_min_worker_fraction
@@ -85,6 +85,17 @@ module Google
85
85
  # and manage this project-level, per-location bucket (see
86
86
  # [Dataproc staging
87
87
  # bucket](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
88
+ # @!attribute [rw] temp_bucket
89
+ # @return [::String]
90
+ # Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data,
91
+ # such as Spark and MapReduce history files.
92
+ # If you do not specify a temp bucket,
93
+ # Dataproc will determine a Cloud Storage location (US,
94
+ # ASIA, or EU) for your cluster's temp bucket according to the
95
+ # Compute Engine zone where your cluster is deployed, and then create
96
+ # and manage this project-level, per-location bucket. The default bucket has
97
+ # a TTL of 90 days, but you can use any TTL (or none) if you specify a
98
+ # bucket.
88
99
  # @!attribute [rw] gce_cluster_config
89
100
  # @return [::Google::Cloud::Dataproc::V1::GceClusterConfig]
90
101
  # Optional. The shared Compute Engine config settings for
@@ -132,11 +143,37 @@ module Google
132
143
  # @!attribute [rw] lifecycle_config
133
144
  # @return [::Google::Cloud::Dataproc::V1::LifecycleConfig]
134
145
  # Optional. Lifecycle setting for the cluster.
146
+ # @!attribute [rw] endpoint_config
147
+ # @return [::Google::Cloud::Dataproc::V1::EndpointConfig]
148
+ # Optional. Port/endpoint configuration for this cluster
135
149
  class ClusterConfig
136
150
  include ::Google::Protobuf::MessageExts
137
151
  extend ::Google::Protobuf::MessageExts::ClassMethods
138
152
  end
139
153
 
154
+ # Endpoint config for this cluster
155
+ # @!attribute [r] http_ports
156
+ # @return [::Google::Protobuf::Map{::String => ::String}]
157
+ # Output only. The map of port descriptions to URLs. Will only be populated
158
+ # if enable_http_port_access is true.
159
+ # @!attribute [rw] enable_http_port_access
160
+ # @return [::Boolean]
161
+ # Optional. If true, enable http access to specific ports on the cluster
162
+ # from external sources. Defaults to false.
163
+ class EndpointConfig
164
+ include ::Google::Protobuf::MessageExts
165
+ extend ::Google::Protobuf::MessageExts::ClassMethods
166
+
167
+ # @!attribute [rw] key
168
+ # @return [::String]
169
+ # @!attribute [rw] value
170
+ # @return [::String]
171
+ class HttpPortsEntry
172
+ include ::Google::Protobuf::MessageExts
173
+ extend ::Google::Protobuf::MessageExts::ClassMethods
174
+ end
175
+ end
176
+
140
177
  # Autoscaling Policy config associated with the cluster.
141
178
  # @!attribute [rw] policy_uri
142
179
  # @return [::String]
@@ -214,7 +251,7 @@ module Google
214
251
  # @!attribute [rw] service_account
215
252
  # @return [::String]
216
253
  # Optional. The [Dataproc service
217
- # account](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#service_accounts_in_cloud_dataproc)
254
+ # account](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#service_accounts_in_dataproc)
218
255
  # (also see [VM Data Plane
219
256
  # identity](https://cloud.google.com/dataproc/docs/concepts/iam/dataproc-principals#vm_service_account_data_plane_identity))
220
257
  # used by Dataproc cluster VM instances to access Google Cloud Platform
@@ -318,6 +355,15 @@ module Google
318
355
  # @return [::Boolean]
319
356
  # Output only. Specifies that this instance group contains preemptible
320
357
  # instances.
358
+ # @!attribute [rw] preemptibility
359
+ # @return [::Google::Cloud::Dataproc::V1::InstanceGroupConfig::Preemptibility]
360
+ # Optional. Specifies the preemptibility of the instance group.
361
+ #
362
+ # The default value for master and worker groups is
363
+ # `NON_PREEMPTIBLE`. This default cannot be changed.
364
+ #
365
+ # The default value for secondary instances is
366
+ # `PREEMPTIBLE`.
321
367
  # @!attribute [r] managed_group_config
322
368
  # @return [::Google::Cloud::Dataproc::V1::ManagedGroupConfig]
323
369
  # Output only. The config for Compute Engine Instance Group
@@ -335,6 +381,27 @@ module Google
335
381
  class InstanceGroupConfig
336
382
  include ::Google::Protobuf::MessageExts
337
383
  extend ::Google::Protobuf::MessageExts::ClassMethods
384
+
385
+ # Controls the use of
386
+ # [preemptible instances]
387
+ # (https://cloud.google.com/compute/docs/instances/preemptible)
388
+ # within the group.
389
+ module Preemptibility
390
+ # Preemptibility is unspecified, the system will choose the
391
+ # appropriate setting for each instance group.
392
+ PREEMPTIBILITY_UNSPECIFIED = 0
393
+
394
+ # Instances are non-preemptible.
395
+ #
396
+ # This option is allowed for all instance groups and is the only valid
397
+ # value for Master and Worker instance groups.
398
+ NON_PREEMPTIBLE = 1
399
+
400
+ # Instances are preemptible.
401
+ #
402
+ # This option is allowed only for secondary worker groups.
403
+ PREEMPTIBLE = 2
404
+ end
338
405
  end
339
406
 
340
407
  # Specifies the resources used to actively manage an instance group.
@@ -567,7 +634,7 @@ module Google
567
634
  # @return [::String]
568
635
  # Optional. The version of software inside the cluster. It must be one of the
569
636
  # supported [Dataproc
570
- # Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported_cloud_dataproc_versions),
637
+ # Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported_dataproc_versions),
571
638
  # such as "1.2" (including a subminor version, such as "1.2.29"), or the
572
639
  # ["preview"
573
640
  # version](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
@@ -153,12 +153,12 @@ module Google
153
153
  # Spark driver and tasks.
154
154
  # @!attribute [rw] file_uris
155
155
  # @return [::Array<::String>]
156
- # Optional. HCFS URIs of files to be copied to the working directory of
157
- # Spark drivers and distributed tasks. Useful for naively parallel tasks.
156
+ # Optional. HCFS URIs of files to be placed in the working directory of
157
+ # each executor. Useful for naively parallel tasks.
158
158
  # @!attribute [rw] archive_uris
159
159
  # @return [::Array<::String>]
160
- # Optional. HCFS URIs of archives to be extracted in the working directory
161
- # of Spark drivers and tasks. Supported file types:
160
+ # Optional. HCFS URIs of archives to be extracted into the working directory
161
+ # of each executor. Supported file types:
162
162
  # .jar, .tar, .tar.gz, .tgz, and .zip.
163
163
  # @!attribute [rw] properties
164
164
  # @return [::Google::Protobuf::Map{::String => ::String}]
@@ -206,11 +206,12 @@ module Google
206
206
  # Python driver and tasks.
207
207
  # @!attribute [rw] file_uris
208
208
  # @return [::Array<::String>]
209
- # Optional. HCFS URIs of files to be copied to the working directory of
210
- # Python drivers and distributed tasks. Useful for naively parallel tasks.
209
+ # Optional. HCFS URIs of files to be placed in the working directory of
210
+ # each executor. Useful for naively parallel tasks.
211
211
  # @!attribute [rw] archive_uris
212
212
  # @return [::Array<::String>]
213
- # Optional. HCFS URIs of archives to be extracted in the working directory of
213
+ # Optional. HCFS URIs of archives to be extracted into the working directory
214
+ # of each executor. Supported file types:
214
215
  # .jar, .tar, .tar.gz, .tgz, and .zip.
215
216
  # @!attribute [rw] properties
216
217
  # @return [::Google::Protobuf::Map{::String => ::String}]
@@ -421,12 +422,12 @@ module Google
421
422
  # occur that causes an incorrect job submission.
422
423
  # @!attribute [rw] file_uris
423
424
  # @return [::Array<::String>]
424
- # Optional. HCFS URIs of files to be copied to the working directory of
425
- # R drivers and distributed tasks. Useful for naively parallel tasks.
425
+ # Optional. HCFS URIs of files to be placed in the working directory of
426
+ # each executor. Useful for naively parallel tasks.
426
427
  # @!attribute [rw] archive_uris
427
428
  # @return [::Array<::String>]
428
- # Optional. HCFS URIs of archives to be extracted in the working directory of
429
- # Spark drivers and tasks. Supported file types:
429
+ # Optional. HCFS URIs of archives to be extracted into the working directory
430
+ # of each executor. Supported file types:
430
431
  # .jar, .tar, .tar.gz, .tgz, and .zip.
431
432
  # @!attribute [rw] properties
432
433
  # @return [::Google::Protobuf::Map{::String => ::String}]
@@ -595,8 +596,8 @@ module Google
595
596
  # Encapsulates the full scoping used to reference a job.
596
597
  # @!attribute [rw] project_id
597
598
  # @return [::String]
598
- # Required. The ID of the Google Cloud Platform project that the job
599
- # belongs to.
599
+ # Optional. The ID of the Google Cloud Platform project that the job belongs to. If
600
+ # specified, must match the request project ID.
600
601
  # @!attribute [rw] job_id
601
602
  # @return [::String]
602
603
  # Optional. The job ID, which must be unique within the project.
@@ -23,7 +23,7 @@ module Google
23
23
  module V1
24
24
  # Cluster components that can be activated.
25
25
  module Component
26
- # Unspecified component.
26
+ # Unspecified component. Specifying this will cause Cluster creation to fail.
27
27
  COMPONENT_UNSPECIFIED = 0
28
28
 
29
29
  # The Anaconda python distribution.
@@ -75,7 +75,7 @@ module Google
75
75
  # Required. The Directed Acyclic Graph of Jobs to submit.
76
76
  # @!attribute [rw] parameters
77
77
  # @return [::Array<::Google::Cloud::Dataproc::V1::TemplateParameter>]
78
- # Optional. emplate parameters whose values are substituted into the
78
+ # Optional. Template parameters whose values are substituted into the
79
79
  # template. Values for parameters must be provided when the template is
80
80
  # instantiated.
81
81
  class WorkflowTemplate
@@ -189,22 +189,28 @@ module Google
189
189
  # or hyphen. Must consist of between 3 and 50 characters.
190
190
  # @!attribute [rw] hadoop_job
191
191
  # @return [::Google::Cloud::Dataproc::V1::HadoopJob]
192
+ # Optional. Job is a Hadoop job.
192
193
  # @!attribute [rw] spark_job
193
194
  # @return [::Google::Cloud::Dataproc::V1::SparkJob]
195
+ # Optional. Job is a Spark job.
194
196
  # @!attribute [rw] pyspark_job
195
197
  # @return [::Google::Cloud::Dataproc::V1::PySparkJob]
198
+ # Optional. Job is a PySpark job.
196
199
  # @!attribute [rw] hive_job
197
200
  # @return [::Google::Cloud::Dataproc::V1::HiveJob]
201
+ # Optional. Job is a Hive job.
198
202
  # @!attribute [rw] pig_job
199
203
  # @return [::Google::Cloud::Dataproc::V1::PigJob]
204
+ # Optional. Job is a Pig job.
200
205
  # @!attribute [rw] spark_r_job
201
206
  # @return [::Google::Cloud::Dataproc::V1::SparkRJob]
202
- # Spark R job
207
+ # Optional. Job is a SparkR job.
203
208
  # @!attribute [rw] spark_sql_job
204
209
  # @return [::Google::Cloud::Dataproc::V1::SparkSqlJob]
210
+ # Optional. Job is a SparkSql job.
205
211
  # @!attribute [rw] presto_job
206
212
  # @return [::Google::Cloud::Dataproc::V1::PrestoJob]
207
- # Presto job
213
+ # Optional. Job is a Presto job.
208
214
  # @!attribute [rw] labels
209
215
  # @return [::Google::Protobuf::Map{::String => ::String}]
210
216
  # Optional. The labels to associate with this job.
@@ -25,7 +25,7 @@ module Google
25
25
  # @return [::String]
26
26
  # The server-assigned name, which is only unique within the same service that
27
27
  # originally returns it. If you use the default HTTP mapping, the
28
- # `name` should have the format of `operations/some/unique/name`.
28
+ # `name` should be a resource name ending with `operations/{unique_id}`.
29
29
  # @!attribute [rw] metadata
30
30
  # @return [::Google::Protobuf::Any]
31
31
  # Service-specific metadata associated with the operation. It typically
@@ -35,7 +35,7 @@ module Google
35
35
  # @!attribute [rw] done
36
36
  # @return [::Boolean]
37
37
  # If the value is `false`, it means the operation is still in progress.
38
- # If true, the operation is completed, and either `error` or `response` is
38
+ # If `true`, the operation is completed, and either `error` or `response` is
39
39
  # available.
40
40
  # @!attribute [rw] error
41
41
  # @return [::Google::Rpc::Status]
@@ -67,7 +67,7 @@ module Google
67
67
  # The request message for Operations.ListOperations.
68
68
  # @!attribute [rw] name
69
69
  # @return [::String]
70
- # The name of the operation collection.
70
+ # The name of the operation's parent resource.
71
71
  # @!attribute [rw] filter
72
72
  # @return [::String]
73
73
  # The standard list filter.
@@ -112,6 +112,20 @@ module Google
112
112
  extend ::Google::Protobuf::MessageExts::ClassMethods
113
113
  end
114
114
 
115
+ # The request message for Operations.WaitOperation.
116
+ # @!attribute [rw] name
117
+ # @return [::String]
118
+ # The name of the operation resource to wait on.
119
+ # @!attribute [rw] timeout
120
+ # @return [::Google::Protobuf::Duration]
121
+ # The maximum duration to wait before timing out. If left blank, the wait
122
+ # will be at most the time permitted by the underlying HTTP/RPC protocol.
123
+ # If RPC context deadline is also specified, the shorter one will be used.
124
+ class WaitOperationRequest
125
+ include ::Google::Protobuf::MessageExts
126
+ extend ::Google::Protobuf::MessageExts::ClassMethods
127
+ end
128
+
115
129
  # A message representing the message types used by a long-running operation.
116
130
  #
117
131
  # Example:
@@ -57,10 +57,13 @@ module Google
57
57
  # Example 4: Pack and unpack a message in Go
58
58
  #
59
59
  # foo := &pb.Foo{...}
60
- # any, err := ptypes.MarshalAny(foo)
60
+ # any, err := anypb.New(foo)
61
+ # if err != nil {
62
+ # ...
63
+ # }
61
64
  # ...
62
65
  # foo := &pb.Foo{}
63
- # if err := ptypes.UnmarshalAny(any, foo); err != nil {
66
+ # if err := any.UnmarshalTo(foo); err != nil {
64
67
  # ...
65
68
  # }
66
69
  #
@@ -70,7 +70,16 @@ module Google
70
70
  # .setNanos((int) ((millis % 1000) * 1000000)).build();
71
71
  #
72
72
  #
73
- # Example 5: Compute Timestamp from current time in Python.
73
+ # Example 5: Compute Timestamp from Java `Instant.now()`.
74
+ #
75
+ # Instant now = Instant.now();
76
+ #
77
+ # Timestamp timestamp =
78
+ # Timestamp.newBuilder().setSeconds(now.getEpochSecond())
79
+ # .setNanos(now.getNano()).build();
80
+ #
81
+ #
82
+ # Example 6: Compute Timestamp from current time in Python.
74
83
  #
75
84
  # timestamp = Timestamp()
76
85
  # timestamp.GetCurrentTime()
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: google-cloud-dataproc-v1
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.3
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Google LLC
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-06-18 00:00:00.000000000 Z
11
+ date: 2021-03-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: gapic-common
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '0.2'
19
+ version: '0.3'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '0.2'
26
+ version: '0.3'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: google-cloud-errors
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -44,14 +44,14 @@ dependencies:
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: 1.24.0
47
+ version: 1.25.1
48
48
  type: :development
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: 1.24.0
54
+ version: 1.25.1
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: minitest
57
57
  requirement: !ruby/object:Gem::Requirement
@@ -150,7 +150,10 @@ dependencies:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
152
  version: '0.9'
153
- description: Manages Hadoop-based clusters and jobs on Google Cloud Platform.
153
+ description: Manages Hadoop-based clusters and jobs on Google Cloud Platform. Note
154
+ that google-cloud-dataproc-v1 is a version-specific client library. For most uses,
155
+ we recommend installing the main client library google-cloud-dataproc instead. See
156
+ the readme for more details.
154
157
  email: googleapis-packages@google.com
155
158
  executables: []
156
159
  extensions: []
@@ -218,14 +221,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
218
221
  requirements:
219
222
  - - ">="
220
223
  - !ruby/object:Gem::Version
221
- version: '2.4'
224
+ version: '2.5'
222
225
  required_rubygems_version: !ruby/object:Gem::Requirement
223
226
  requirements:
224
227
  - - ">="
225
228
  - !ruby/object:Gem::Version
226
229
  version: '0'
227
230
  requirements: []
228
- rubygems_version: 3.1.3
231
+ rubygems_version: 3.2.13
229
232
  signing_key:
230
233
  specification_version: 4
231
234
  summary: API Client library for the Cloud Dataproc V1 API