google-cloud-dataproc 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.yardopts +8 -0
- data/LICENSE +201 -0
- data/README.md +54 -0
- data/lib/google/cloud/dataproc.rb +177 -0
- data/lib/google/cloud/dataproc/credentials.rb +30 -0
- data/lib/google/cloud/dataproc/v1.rb +171 -0
- data/lib/google/cloud/dataproc/v1/cluster_controller_client.rb +670 -0
- data/lib/google/cloud/dataproc/v1/cluster_controller_client_config.json +56 -0
- data/lib/google/cloud/dataproc/v1/clusters_pb.rb +166 -0
- data/lib/google/cloud/dataproc/v1/clusters_services_pb.rb +58 -0
- data/lib/google/cloud/dataproc/v1/doc/google/cloud/dataproc/v1/clusters.rb +565 -0
- data/lib/google/cloud/dataproc/v1/doc/google/cloud/dataproc/v1/jobs.rb +682 -0
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/any.rb +124 -0
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/duration.rb +90 -0
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/field_mask.rb +223 -0
- data/lib/google/cloud/dataproc/v1/doc/google/protobuf/timestamp.rb +106 -0
- data/lib/google/cloud/dataproc/v1/doc/google/rpc/status.rb +83 -0
- data/lib/google/cloud/dataproc/v1/doc/overview.rb +77 -0
- data/lib/google/cloud/dataproc/v1/job_controller_client.rb +487 -0
- data/lib/google/cloud/dataproc/v1/job_controller_client_config.json +56 -0
- data/lib/google/cloud/dataproc/v1/jobs_pb.rb +244 -0
- data/lib/google/cloud/dataproc/v1/jobs_services_pb.rb +59 -0
- data/lib/google/cloud/dataproc/v1/operations_pb.rb +45 -0
- metadata +123 -0
@@ -0,0 +1,30 @@
|
|
1
|
+
# Copyright 2017 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require "googleauth"
|
16
|
+
|
17
|
+
module Google
|
18
|
+
module Cloud
|
19
|
+
module Dataproc
|
20
|
+
class Credentials < Google::Auth::Credentials
|
21
|
+
SCOPE = [
|
22
|
+
"https://www.googleapis.com/auth/cloud-platform"
|
23
|
+
].freeze
|
24
|
+
PATH_ENV_VARS = %w(DATAPROC_KEYFILE GOOGLE_CLOUD_KEYFILE GCLOUD_KEYFILE)
|
25
|
+
JSON_ENV_VARS = %w(DATAPROC_KEYFILE_JSON GOOGLE_CLOUD_KEYFILE_JSON GCLOUD_KEYFILE_JSON)
|
26
|
+
DEFAULT_PATHS = ["~/.config/gcloud/application_default_credentials.json"]
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,171 @@
|
|
1
|
+
# Copyright 2017 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require "google/cloud/dataproc/v1/cluster_controller_client"
|
16
|
+
require "google/cloud/dataproc/v1/job_controller_client"
|
17
|
+
|
18
|
+
module Google
|
19
|
+
module Cloud
|
20
|
+
# rubocop:disable LineLength
|
21
|
+
|
22
|
+
##
|
23
|
+
# # Ruby Client for Google Cloud Dataproc API ([Alpha](https://github.com/GoogleCloudPlatform/google-cloud-ruby#versioning))
|
24
|
+
#
|
25
|
+
# [Google Cloud Dataproc API][Product Documentation]:
|
26
|
+
# Manages Hadoop-based clusters and jobs on Google Cloud Platform.
|
27
|
+
# - [Product Documentation][]
|
28
|
+
#
|
29
|
+
# ## Quick Start
|
30
|
+
# In order to use this library, you first need to go through the following
|
31
|
+
# steps:
|
32
|
+
#
|
33
|
+
# 1. [Select or create a Cloud Platform project.](https://console.cloud.google.com/project)
|
34
|
+
# 2. [Enable billing for your project.](https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project)
|
35
|
+
# 3. [Enable the Google Cloud Dataproc API.](https://console.cloud.google.com/apis/api/dataproc)
|
36
|
+
# 4. [Setup Authentication.](https://googlecloudplatform.github.io/google-cloud-ruby/#/docs/google-cloud/master/guides/authentication)
|
37
|
+
#
|
38
|
+
# ### Preview
|
39
|
+
# #### ClusterControllerClient
|
40
|
+
# ```rb
|
41
|
+
# require "google/cloud/dataproc/v1"
|
42
|
+
#
|
43
|
+
# cluster_controller_client = Google::Cloud::Dataproc::V1::ClusterController.new
|
44
|
+
# project_id_2 = project_id
|
45
|
+
# region = "global"
|
46
|
+
#
|
47
|
+
# # Iterate over all results.
|
48
|
+
# cluster_controller_client.list_clusters(project_id_2, region).each do |element|
|
49
|
+
# # Process element.
|
50
|
+
# end
|
51
|
+
#
|
52
|
+
# # Or iterate over results one page at a time.
|
53
|
+
# cluster_controller_client.list_clusters(project_id_2, region).each_page do |page|
|
54
|
+
# # Process each page at a time.
|
55
|
+
# page.each do |element|
|
56
|
+
# # Process element.
|
57
|
+
# end
|
58
|
+
# end
|
59
|
+
# ```
|
60
|
+
#
|
61
|
+
# ### Next Steps
|
62
|
+
# - Read the [Google Cloud Dataproc API Product documentation][Product Documentation]
|
63
|
+
# to learn more about the product and see How-to Guides.
|
64
|
+
# - View this [repository's main README](https://github.com/GoogleCloudPlatform/google-cloud-ruby/blob/master/README.md)
|
65
|
+
# to see the full list of Cloud APIs that we cover.
|
66
|
+
#
|
67
|
+
# [Product Documentation]: https://cloud.google.com/dataproc
|
68
|
+
#
|
69
|
+
#
|
70
|
+
module Dataproc
|
71
|
+
module V1
|
72
|
+
# rubocop:enable LineLength
|
73
|
+
|
74
|
+
module ClusterController
|
75
|
+
##
|
76
|
+
# The ClusterControllerService provides methods to manage clusters
|
77
|
+
# of Google Compute Engine instances.
|
78
|
+
#
|
79
|
+
# @param credentials [Google::Auth::Credentials, String, Hash, GRPC::Core::Channel, GRPC::Core::ChannelCredentials, Proc]
|
80
|
+
# Provides the means for authenticating requests made by the client. This parameter can
|
81
|
+
# be many types.
|
82
|
+
# A `Google::Auth::Credentials` uses a the properties of its represented keyfile for
|
83
|
+
# authenticating requests made by this client.
|
84
|
+
# A `String` will be treated as the path to the keyfile to be used for the construction of
|
85
|
+
# credentials for this client.
|
86
|
+
# A `Hash` will be treated as the contents of a keyfile to be used for the construction of
|
87
|
+
# credentials for this client.
|
88
|
+
# A `GRPC::Core::Channel` will be used to make calls through.
|
89
|
+
# A `GRPC::Core::ChannelCredentials` for the setting up the RPC client. The channel credentials
|
90
|
+
# should already be composed with a `GRPC::Core::CallCredentials` object.
|
91
|
+
# A `Proc` will be used as an updater_proc for the Grpc channel. The proc transforms the
|
92
|
+
# metadata for requests, generally, to give OAuth credentials.
|
93
|
+
# @param scopes [Array<String>]
|
94
|
+
# The OAuth scopes for this service. This parameter is ignored if
|
95
|
+
# an updater_proc is supplied.
|
96
|
+
# @param client_config [Hash]
|
97
|
+
# A Hash for call options for each method. See
|
98
|
+
# Google::Gax#construct_settings for the structure of
|
99
|
+
# this data. Falls back to the default config if not specified
|
100
|
+
# or the specified config is missing data points.
|
101
|
+
# @param timeout [Numeric]
|
102
|
+
# The default timeout, in seconds, for calls made through this client.
|
103
|
+
def self.new \
|
104
|
+
credentials: nil,
|
105
|
+
scopes: nil,
|
106
|
+
client_config: nil,
|
107
|
+
timeout: nil,
|
108
|
+
lib_name: nil,
|
109
|
+
lib_version: nil
|
110
|
+
kwargs = {
|
111
|
+
credentials: credentials,
|
112
|
+
scopes: scopes,
|
113
|
+
client_config: client_config,
|
114
|
+
timeout: timeout,
|
115
|
+
lib_name: lib_name,
|
116
|
+
lib_version: lib_version
|
117
|
+
}.select { |_, v| v != nil }
|
118
|
+
Google::Cloud::Dataproc::V1::ClusterControllerClient.new(**kwargs)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
module JobController
|
123
|
+
##
|
124
|
+
# The JobController provides methods to manage jobs.
|
125
|
+
#
|
126
|
+
# @param credentials [Google::Auth::Credentials, String, Hash, GRPC::Core::Channel, GRPC::Core::ChannelCredentials, Proc]
|
127
|
+
# Provides the means for authenticating requests made by the client. This parameter can
|
128
|
+
# be many types.
|
129
|
+
# A `Google::Auth::Credentials` uses a the properties of its represented keyfile for
|
130
|
+
# authenticating requests made by this client.
|
131
|
+
# A `String` will be treated as the path to the keyfile to be used for the construction of
|
132
|
+
# credentials for this client.
|
133
|
+
# A `Hash` will be treated as the contents of a keyfile to be used for the construction of
|
134
|
+
# credentials for this client.
|
135
|
+
# A `GRPC::Core::Channel` will be used to make calls through.
|
136
|
+
# A `GRPC::Core::ChannelCredentials` for the setting up the RPC client. The channel credentials
|
137
|
+
# should already be composed with a `GRPC::Core::CallCredentials` object.
|
138
|
+
# A `Proc` will be used as an updater_proc for the Grpc channel. The proc transforms the
|
139
|
+
# metadata for requests, generally, to give OAuth credentials.
|
140
|
+
# @param scopes [Array<String>]
|
141
|
+
# The OAuth scopes for this service. This parameter is ignored if
|
142
|
+
# an updater_proc is supplied.
|
143
|
+
# @param client_config [Hash]
|
144
|
+
# A Hash for call options for each method. See
|
145
|
+
# Google::Gax#construct_settings for the structure of
|
146
|
+
# this data. Falls back to the default config if not specified
|
147
|
+
# or the specified config is missing data points.
|
148
|
+
# @param timeout [Numeric]
|
149
|
+
# The default timeout, in seconds, for calls made through this client.
|
150
|
+
def self.new \
|
151
|
+
credentials: nil,
|
152
|
+
scopes: nil,
|
153
|
+
client_config: nil,
|
154
|
+
timeout: nil,
|
155
|
+
lib_name: nil,
|
156
|
+
lib_version: nil
|
157
|
+
kwargs = {
|
158
|
+
credentials: credentials,
|
159
|
+
scopes: scopes,
|
160
|
+
client_config: client_config,
|
161
|
+
timeout: timeout,
|
162
|
+
lib_name: lib_name,
|
163
|
+
lib_version: lib_version
|
164
|
+
}.select { |_, v| v != nil }
|
165
|
+
Google::Cloud::Dataproc::V1::JobControllerClient.new(**kwargs)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
@@ -0,0 +1,670 @@
|
|
1
|
+
# Copyright 2017 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
#
|
15
|
+
# EDITING INSTRUCTIONS
|
16
|
+
# This file was generated from the file
|
17
|
+
# https://github.com/googleapis/googleapis/blob/master/google/cloud/dataproc/v1/clusters.proto,
|
18
|
+
# and updates to that file get reflected here through a refresh process.
|
19
|
+
# For the short term, the refresh process will only be runnable by Google
|
20
|
+
# engineers.
|
21
|
+
#
|
22
|
+
# The only allowed edits are to method and file documentation. A 3-way
|
23
|
+
# merge preserves those additions if the generated source changes.
|
24
|
+
|
25
|
+
require "json"
|
26
|
+
require "pathname"
|
27
|
+
|
28
|
+
require "google/gax"
|
29
|
+
require "google/gax/operation"
|
30
|
+
require "google/longrunning/operations_client"
|
31
|
+
|
32
|
+
require "google/cloud/dataproc/v1/clusters_pb"
|
33
|
+
require "google/cloud/dataproc/credentials"
|
34
|
+
|
35
|
+
module Google
|
36
|
+
module Cloud
|
37
|
+
module Dataproc
|
38
|
+
module V1
|
39
|
+
# The ClusterControllerService provides methods to manage clusters
|
40
|
+
# of Google Compute Engine instances.
|
41
|
+
#
|
42
|
+
# @!attribute [r] cluster_controller_stub
|
43
|
+
# @return [Google::Cloud::Dataproc::V1::ClusterController::Stub]
|
44
|
+
class ClusterControllerClient
|
45
|
+
attr_reader :cluster_controller_stub
|
46
|
+
|
47
|
+
# The default address of the service.
|
48
|
+
SERVICE_ADDRESS = "dataproc.googleapis.com".freeze
|
49
|
+
|
50
|
+
# The default port of the service.
|
51
|
+
DEFAULT_SERVICE_PORT = 443
|
52
|
+
|
53
|
+
DEFAULT_TIMEOUT = 30
|
54
|
+
|
55
|
+
PAGE_DESCRIPTORS = {
|
56
|
+
"list_clusters" => Google::Gax::PageDescriptor.new(
|
57
|
+
"page_token",
|
58
|
+
"next_page_token",
|
59
|
+
"clusters")
|
60
|
+
}.freeze
|
61
|
+
|
62
|
+
private_constant :PAGE_DESCRIPTORS
|
63
|
+
|
64
|
+
# The scopes needed to make gRPC calls to all of the methods defined in
|
65
|
+
# this service.
|
66
|
+
ALL_SCOPES = [
|
67
|
+
"https://www.googleapis.com/auth/cloud-platform"
|
68
|
+
].freeze
|
69
|
+
|
70
|
+
class OperationsClient < Google::Longrunning::OperationsClient
|
71
|
+
SERVICE_ADDRESS = SERVICE_ADDRESS
|
72
|
+
end
|
73
|
+
|
74
|
+
# @param credentials [Google::Auth::Credentials, String, Hash, GRPC::Core::Channel, GRPC::Core::ChannelCredentials, Proc]
|
75
|
+
# Provides the means for authenticating requests made by the client. This parameter can
|
76
|
+
# be many types.
|
77
|
+
# A `Google::Auth::Credentials` uses a the properties of its represented keyfile for
|
78
|
+
# authenticating requests made by this client.
|
79
|
+
# A `String` will be treated as the path to the keyfile to be used for the construction of
|
80
|
+
# credentials for this client.
|
81
|
+
# A `Hash` will be treated as the contents of a keyfile to be used for the construction of
|
82
|
+
# credentials for this client.
|
83
|
+
# A `GRPC::Core::Channel` will be used to make calls through.
|
84
|
+
# A `GRPC::Core::ChannelCredentials` for the setting up the RPC client. The channel credentials
|
85
|
+
# should already be composed with a `GRPC::Core::CallCredentials` object.
|
86
|
+
# A `Proc` will be used as an updater_proc for the Grpc channel. The proc transforms the
|
87
|
+
# metadata for requests, generally, to give OAuth credentials.
|
88
|
+
# @param scopes [Array<String>]
|
89
|
+
# The OAuth scopes for this service. This parameter is ignored if
|
90
|
+
# an updater_proc is supplied.
|
91
|
+
# @param client_config [Hash]
|
92
|
+
# A Hash for call options for each method. See
|
93
|
+
# Google::Gax#construct_settings for the structure of
|
94
|
+
# this data. Falls back to the default config if not specified
|
95
|
+
# or the specified config is missing data points.
|
96
|
+
# @param timeout [Numeric]
|
97
|
+
# The default timeout, in seconds, for calls made through this client.
|
98
|
+
def initialize \
|
99
|
+
credentials: nil,
|
100
|
+
scopes: ALL_SCOPES,
|
101
|
+
client_config: {},
|
102
|
+
timeout: DEFAULT_TIMEOUT,
|
103
|
+
lib_name: nil,
|
104
|
+
lib_version: ""
|
105
|
+
# These require statements are intentionally placed here to initialize
|
106
|
+
# the gRPC module only when it's required.
|
107
|
+
# See https://github.com/googleapis/toolkit/issues/446
|
108
|
+
require "google/gax/grpc"
|
109
|
+
require "google/cloud/dataproc/v1/clusters_services_pb"
|
110
|
+
|
111
|
+
credentials ||= Google::Cloud::Dataproc::Credentials.default
|
112
|
+
|
113
|
+
@operations_client = OperationsClient.new(
|
114
|
+
credentials: credentials,
|
115
|
+
scopes: scopes,
|
116
|
+
client_config: client_config,
|
117
|
+
timeout: timeout,
|
118
|
+
lib_name: lib_name,
|
119
|
+
lib_version: lib_version,
|
120
|
+
)
|
121
|
+
|
122
|
+
if credentials.is_a?(String) || credentials.is_a?(Hash)
|
123
|
+
updater_proc = Google::Cloud::Dataproc::Credentials.new(credentials).updater_proc
|
124
|
+
end
|
125
|
+
if credentials.is_a?(GRPC::Core::Channel)
|
126
|
+
channel = credentials
|
127
|
+
end
|
128
|
+
if credentials.is_a?(GRPC::Core::ChannelCredentials)
|
129
|
+
chan_creds = credentials
|
130
|
+
end
|
131
|
+
if credentials.is_a?(Proc)
|
132
|
+
updater_proc = credentials
|
133
|
+
end
|
134
|
+
if credentials.is_a?(Google::Auth::Credentials)
|
135
|
+
updater_proc = credentials.updater_proc
|
136
|
+
end
|
137
|
+
|
138
|
+
package_version = Gem.loaded_specs['google-cloud-dataproc'].version.version
|
139
|
+
|
140
|
+
google_api_client = "gl-ruby/#{RUBY_VERSION}"
|
141
|
+
google_api_client << " #{lib_name}/#{lib_version}" if lib_name
|
142
|
+
google_api_client << " gapic/#{package_version} gax/#{Google::Gax::VERSION}"
|
143
|
+
google_api_client << " grpc/#{GRPC::VERSION}"
|
144
|
+
google_api_client.freeze
|
145
|
+
|
146
|
+
headers = { :"x-goog-api-client" => google_api_client }
|
147
|
+
client_config_file = Pathname.new(__dir__).join(
|
148
|
+
"cluster_controller_client_config.json"
|
149
|
+
)
|
150
|
+
defaults = client_config_file.open do |f|
|
151
|
+
Google::Gax.construct_settings(
|
152
|
+
"google.cloud.dataproc.v1.ClusterController",
|
153
|
+
JSON.parse(f.read),
|
154
|
+
client_config,
|
155
|
+
Google::Gax::Grpc::STATUS_CODE_NAMES,
|
156
|
+
timeout,
|
157
|
+
page_descriptors: PAGE_DESCRIPTORS,
|
158
|
+
errors: Google::Gax::Grpc::API_ERRORS,
|
159
|
+
kwargs: headers
|
160
|
+
)
|
161
|
+
end
|
162
|
+
|
163
|
+
# Allow overriding the service path/port in subclasses.
|
164
|
+
service_path = self.class::SERVICE_ADDRESS
|
165
|
+
port = self.class::DEFAULT_SERVICE_PORT
|
166
|
+
@cluster_controller_stub = Google::Gax::Grpc.create_stub(
|
167
|
+
service_path,
|
168
|
+
port,
|
169
|
+
chan_creds: chan_creds,
|
170
|
+
channel: channel,
|
171
|
+
updater_proc: updater_proc,
|
172
|
+
scopes: scopes,
|
173
|
+
&Google::Cloud::Dataproc::V1::ClusterController::Stub.method(:new)
|
174
|
+
)
|
175
|
+
|
176
|
+
@create_cluster = Google::Gax.create_api_call(
|
177
|
+
@cluster_controller_stub.method(:create_cluster),
|
178
|
+
defaults["create_cluster"]
|
179
|
+
)
|
180
|
+
@update_cluster = Google::Gax.create_api_call(
|
181
|
+
@cluster_controller_stub.method(:update_cluster),
|
182
|
+
defaults["update_cluster"]
|
183
|
+
)
|
184
|
+
@delete_cluster = Google::Gax.create_api_call(
|
185
|
+
@cluster_controller_stub.method(:delete_cluster),
|
186
|
+
defaults["delete_cluster"]
|
187
|
+
)
|
188
|
+
@get_cluster = Google::Gax.create_api_call(
|
189
|
+
@cluster_controller_stub.method(:get_cluster),
|
190
|
+
defaults["get_cluster"]
|
191
|
+
)
|
192
|
+
@list_clusters = Google::Gax.create_api_call(
|
193
|
+
@cluster_controller_stub.method(:list_clusters),
|
194
|
+
defaults["list_clusters"]
|
195
|
+
)
|
196
|
+
@diagnose_cluster = Google::Gax.create_api_call(
|
197
|
+
@cluster_controller_stub.method(:diagnose_cluster),
|
198
|
+
defaults["diagnose_cluster"]
|
199
|
+
)
|
200
|
+
end
|
201
|
+
|
202
|
+
# Service calls
|
203
|
+
|
204
|
+
# Creates a cluster in a project.
|
205
|
+
#
|
206
|
+
# @param project_id [String]
|
207
|
+
# Required. The ID of the Google Cloud Platform project that the cluster
|
208
|
+
# belongs to.
|
209
|
+
# @param region [String]
|
210
|
+
# Required. The Cloud Dataproc region in which to handle the request.
|
211
|
+
# @param cluster [Google::Cloud::Dataproc::V1::Cluster | Hash]
|
212
|
+
# Required. The cluster to create.
|
213
|
+
# A hash of the same form as `Google::Cloud::Dataproc::V1::Cluster`
|
214
|
+
# can also be provided.
|
215
|
+
# @param options [Google::Gax::CallOptions]
|
216
|
+
# Overrides the default settings for this call, e.g, timeout,
|
217
|
+
# retries, etc.
|
218
|
+
# @return [Google::Gax::Operation]
|
219
|
+
# @raise [Google::Gax::GaxError] if the RPC is aborted.
|
220
|
+
# @example
|
221
|
+
# require "google/cloud/dataproc/v1"
|
222
|
+
#
|
223
|
+
# cluster_controller_client = Google::Cloud::Dataproc::V1::ClusterController.new
|
224
|
+
# project_id = ''
|
225
|
+
# region = ''
|
226
|
+
# cluster = {}
|
227
|
+
#
|
228
|
+
# # Register a callback during the method call.
|
229
|
+
# operation = cluster_controller_client.create_cluster(project_id, region, cluster) do |op|
|
230
|
+
# raise op.results.message if op.error?
|
231
|
+
# op_results = op.results
|
232
|
+
# # Process the results.
|
233
|
+
#
|
234
|
+
# metadata = op.metadata
|
235
|
+
# # Process the metadata.
|
236
|
+
# end
|
237
|
+
#
|
238
|
+
# # Or use the return value to register a callback.
|
239
|
+
# operation.on_done do |op|
|
240
|
+
# raise op.results.message if op.error?
|
241
|
+
# op_results = op.results
|
242
|
+
# # Process the results.
|
243
|
+
#
|
244
|
+
# metadata = op.metadata
|
245
|
+
# # Process the metadata.
|
246
|
+
# end
|
247
|
+
#
|
248
|
+
# # Manually reload the operation.
|
249
|
+
# operation.reload!
|
250
|
+
#
|
251
|
+
# # Or block until the operation completes, triggering callbacks on
|
252
|
+
# # completion.
|
253
|
+
# operation.wait_until_done!
|
254
|
+
|
255
|
+
def create_cluster \
|
256
|
+
project_id,
|
257
|
+
region,
|
258
|
+
cluster,
|
259
|
+
options: nil
|
260
|
+
req = {
|
261
|
+
project_id: project_id,
|
262
|
+
region: region,
|
263
|
+
cluster: cluster
|
264
|
+
}.delete_if { |_, v| v.nil? }
|
265
|
+
req = Google::Gax::to_proto(req, Google::Cloud::Dataproc::V1::CreateClusterRequest)
|
266
|
+
operation = Google::Gax::Operation.new(
|
267
|
+
@create_cluster.call(req, options),
|
268
|
+
@operations_client,
|
269
|
+
Google::Cloud::Dataproc::V1::Cluster,
|
270
|
+
Google::Cloud::Dataproc::V1::ClusterOperationMetadata,
|
271
|
+
call_options: options
|
272
|
+
)
|
273
|
+
operation.on_done { |operation| yield(operation) } if block_given?
|
274
|
+
operation
|
275
|
+
end
|
276
|
+
|
277
|
+
# Updates a cluster in a project.
|
278
|
+
#
|
279
|
+
# @param project_id [String]
|
280
|
+
# Required. The ID of the Google Cloud Platform project the
|
281
|
+
# cluster belongs to.
|
282
|
+
# @param region [String]
|
283
|
+
# Required. The Cloud Dataproc region in which to handle the request.
|
284
|
+
# @param cluster_name [String]
|
285
|
+
# Required. The cluster name.
|
286
|
+
# @param cluster [Google::Cloud::Dataproc::V1::Cluster | Hash]
|
287
|
+
# Required. The changes to the cluster.
|
288
|
+
# A hash of the same form as `Google::Cloud::Dataproc::V1::Cluster`
|
289
|
+
# can also be provided.
|
290
|
+
# @param update_mask [Google::Protobuf::FieldMask | Hash]
|
291
|
+
# Required. Specifies the path, relative to +Cluster+, of
|
292
|
+
# the field to update. For example, to change the number of workers
|
293
|
+
# in a cluster to 5, the +update_mask+ parameter would be
|
294
|
+
# specified as +config.worker_config.num_instances+,
|
295
|
+
# and the +PATCH+ request body would specify the new value, as follows:
|
296
|
+
#
|
297
|
+
# {
|
298
|
+
# "config":{
|
299
|
+
# "workerConfig":{
|
300
|
+
# "numInstances":"5"
|
301
|
+
# }
|
302
|
+
# }
|
303
|
+
# }
|
304
|
+
# Similarly, to change the number of preemptible workers in a cluster to 5,
|
305
|
+
# the +update_mask+ parameter would be
|
306
|
+
# +config.secondary_worker_config.num_instances+, and the +PATCH+ request
|
307
|
+
# body would be set as follows:
|
308
|
+
#
|
309
|
+
# {
|
310
|
+
# "config":{
|
311
|
+
# "secondaryWorkerConfig":{
|
312
|
+
# "numInstances":"5"
|
313
|
+
# }
|
314
|
+
# }
|
315
|
+
# }
|
316
|
+
# <strong>Note:</strong> Currently, only the following fields can be updated:
|
317
|
+
#
|
318
|
+
# <table>
|
319
|
+
# <tbody>
|
320
|
+
# <tr>
|
321
|
+
# <td><strong>Mask</strong></td>
|
322
|
+
# <td><strong>Purpose</strong></td>
|
323
|
+
# </tr>
|
324
|
+
# <tr>
|
325
|
+
# <td><strong><em>labels</em></strong></td>
|
326
|
+
# <td>Update labels</td>
|
327
|
+
# </tr>
|
328
|
+
# <tr>
|
329
|
+
# <td><strong><em>config.worker_config.num_instances</em></strong></td>
|
330
|
+
# <td>Resize primary worker group</td>
|
331
|
+
# </tr>
|
332
|
+
# <tr>
|
333
|
+
# <td><strong><em>config.secondary_worker_config.num_instances</em></strong></td>
|
334
|
+
# <td>Resize secondary worker group</td>
|
335
|
+
# </tr>
|
336
|
+
# </tbody>
|
337
|
+
# </table>
|
338
|
+
# A hash of the same form as `Google::Protobuf::FieldMask`
|
339
|
+
# can also be provided.
|
340
|
+
# @param options [Google::Gax::CallOptions]
|
341
|
+
# Overrides the default settings for this call, e.g, timeout,
|
342
|
+
# retries, etc.
|
343
|
+
# @return [Google::Gax::Operation]
|
344
|
+
# @raise [Google::Gax::GaxError] if the RPC is aborted.
|
345
|
+
# @example
|
346
|
+
# require "google/cloud/dataproc/v1"
|
347
|
+
#
|
348
|
+
# cluster_controller_client = Google::Cloud::Dataproc::V1::ClusterController.new
|
349
|
+
# project_id = ''
|
350
|
+
# region = ''
|
351
|
+
# cluster_name = ''
|
352
|
+
# cluster = {}
|
353
|
+
# update_mask = {}
|
354
|
+
#
|
355
|
+
# # Register a callback during the method call.
|
356
|
+
# operation = cluster_controller_client.update_cluster(project_id, region, cluster_name, cluster, update_mask) do |op|
|
357
|
+
# raise op.results.message if op.error?
|
358
|
+
# op_results = op.results
|
359
|
+
# # Process the results.
|
360
|
+
#
|
361
|
+
# metadata = op.metadata
|
362
|
+
# # Process the metadata.
|
363
|
+
# end
|
364
|
+
#
|
365
|
+
# # Or use the return value to register a callback.
|
366
|
+
# operation.on_done do |op|
|
367
|
+
# raise op.results.message if op.error?
|
368
|
+
# op_results = op.results
|
369
|
+
# # Process the results.
|
370
|
+
#
|
371
|
+
# metadata = op.metadata
|
372
|
+
# # Process the metadata.
|
373
|
+
# end
|
374
|
+
#
|
375
|
+
# # Manually reload the operation.
|
376
|
+
# operation.reload!
|
377
|
+
#
|
378
|
+
# # Or block until the operation completes, triggering callbacks on
|
379
|
+
# # completion.
|
380
|
+
# operation.wait_until_done!
|
381
|
+
|
382
|
+
def update_cluster \
|
383
|
+
project_id,
|
384
|
+
region,
|
385
|
+
cluster_name,
|
386
|
+
cluster,
|
387
|
+
update_mask,
|
388
|
+
options: nil
|
389
|
+
req = {
|
390
|
+
project_id: project_id,
|
391
|
+
region: region,
|
392
|
+
cluster_name: cluster_name,
|
393
|
+
cluster: cluster,
|
394
|
+
update_mask: update_mask
|
395
|
+
}.delete_if { |_, v| v.nil? }
|
396
|
+
req = Google::Gax::to_proto(req, Google::Cloud::Dataproc::V1::UpdateClusterRequest)
|
397
|
+
operation = Google::Gax::Operation.new(
|
398
|
+
@update_cluster.call(req, options),
|
399
|
+
@operations_client,
|
400
|
+
Google::Cloud::Dataproc::V1::Cluster,
|
401
|
+
Google::Cloud::Dataproc::V1::ClusterOperationMetadata,
|
402
|
+
call_options: options
|
403
|
+
)
|
404
|
+
operation.on_done { |operation| yield(operation) } if block_given?
|
405
|
+
operation
|
406
|
+
end
|
407
|
+
|
408
|
+
# Deletes a cluster in a project.
|
409
|
+
#
|
410
|
+
# @param project_id [String]
|
411
|
+
# Required. The ID of the Google Cloud Platform project that the cluster
|
412
|
+
# belongs to.
|
413
|
+
# @param region [String]
|
414
|
+
# Required. The Cloud Dataproc region in which to handle the request.
|
415
|
+
# @param cluster_name [String]
|
416
|
+
# Required. The cluster name.
|
417
|
+
# @param options [Google::Gax::CallOptions]
|
418
|
+
# Overrides the default settings for this call, e.g, timeout,
|
419
|
+
# retries, etc.
|
420
|
+
# @return [Google::Gax::Operation]
|
421
|
+
# @raise [Google::Gax::GaxError] if the RPC is aborted.
|
422
|
+
# @example
|
423
|
+
# require "google/cloud/dataproc/v1"
|
424
|
+
#
|
425
|
+
# cluster_controller_client = Google::Cloud::Dataproc::V1::ClusterController.new
|
426
|
+
# project_id = ''
|
427
|
+
# region = ''
|
428
|
+
# cluster_name = ''
|
429
|
+
#
|
430
|
+
# # Register a callback during the method call.
|
431
|
+
# operation = cluster_controller_client.delete_cluster(project_id, region, cluster_name) do |op|
|
432
|
+
# raise op.results.message if op.error?
|
433
|
+
# op_results = op.results
|
434
|
+
# # Process the results.
|
435
|
+
#
|
436
|
+
# metadata = op.metadata
|
437
|
+
# # Process the metadata.
|
438
|
+
# end
|
439
|
+
#
|
440
|
+
# # Or use the return value to register a callback.
|
441
|
+
# operation.on_done do |op|
|
442
|
+
# raise op.results.message if op.error?
|
443
|
+
# op_results = op.results
|
444
|
+
# # Process the results.
|
445
|
+
#
|
446
|
+
# metadata = op.metadata
|
447
|
+
# # Process the metadata.
|
448
|
+
# end
|
449
|
+
#
|
450
|
+
# # Manually reload the operation.
|
451
|
+
# operation.reload!
|
452
|
+
#
|
453
|
+
# # Or block until the operation completes, triggering callbacks on
|
454
|
+
# # completion.
|
455
|
+
# operation.wait_until_done!
|
456
|
+
|
457
|
+
def delete_cluster \
|
458
|
+
project_id,
|
459
|
+
region,
|
460
|
+
cluster_name,
|
461
|
+
options: nil
|
462
|
+
req = {
|
463
|
+
project_id: project_id,
|
464
|
+
region: region,
|
465
|
+
cluster_name: cluster_name
|
466
|
+
}.delete_if { |_, v| v.nil? }
|
467
|
+
req = Google::Gax::to_proto(req, Google::Cloud::Dataproc::V1::DeleteClusterRequest)
|
468
|
+
operation = Google::Gax::Operation.new(
|
469
|
+
@delete_cluster.call(req, options),
|
470
|
+
@operations_client,
|
471
|
+
Google::Protobuf::Empty,
|
472
|
+
Google::Cloud::Dataproc::V1::ClusterOperationMetadata,
|
473
|
+
call_options: options
|
474
|
+
)
|
475
|
+
operation.on_done { |operation| yield(operation) } if block_given?
|
476
|
+
operation
|
477
|
+
end
|
478
|
+
|
479
|
+
# Gets the resource representation for a cluster in a project.
|
480
|
+
#
|
481
|
+
# @param project_id [String]
|
482
|
+
# Required. The ID of the Google Cloud Platform project that the cluster
|
483
|
+
# belongs to.
|
484
|
+
# @param region [String]
|
485
|
+
# Required. The Cloud Dataproc region in which to handle the request.
|
486
|
+
# @param cluster_name [String]
|
487
|
+
# Required. The cluster name.
|
488
|
+
# @param options [Google::Gax::CallOptions]
|
489
|
+
# Overrides the default settings for this call, e.g, timeout,
|
490
|
+
# retries, etc.
|
491
|
+
# @return [Google::Cloud::Dataproc::V1::Cluster]
|
492
|
+
# @raise [Google::Gax::GaxError] if the RPC is aborted.
|
493
|
+
# @example
|
494
|
+
# require "google/cloud/dataproc/v1"
|
495
|
+
#
|
496
|
+
# cluster_controller_client = Google::Cloud::Dataproc::V1::ClusterController.new
|
497
|
+
# project_id = ''
|
498
|
+
# region = ''
|
499
|
+
# cluster_name = ''
|
500
|
+
# response = cluster_controller_client.get_cluster(project_id, region, cluster_name)
|
501
|
+
|
502
|
+
def get_cluster \
|
503
|
+
project_id,
|
504
|
+
region,
|
505
|
+
cluster_name,
|
506
|
+
options: nil
|
507
|
+
req = {
|
508
|
+
project_id: project_id,
|
509
|
+
region: region,
|
510
|
+
cluster_name: cluster_name
|
511
|
+
}.delete_if { |_, v| v.nil? }
|
512
|
+
req = Google::Gax::to_proto(req, Google::Cloud::Dataproc::V1::GetClusterRequest)
|
513
|
+
@get_cluster.call(req, options)
|
514
|
+
end
|
515
|
+
|
516
|
+
# Lists all regions/{region}/clusters in a project.
|
517
|
+
#
|
518
|
+
# @param project_id [String]
|
519
|
+
# Required. The ID of the Google Cloud Platform project that the cluster
|
520
|
+
# belongs to.
|
521
|
+
# @param region [String]
|
522
|
+
# Required. The Cloud Dataproc region in which to handle the request.
|
523
|
+
# @param filter [String]
|
524
|
+
# Optional. A filter constraining the clusters to list. Filters are
|
525
|
+
# case-sensitive and have the following syntax:
|
526
|
+
#
|
527
|
+
# field = value [AND [field = value]] ...
|
528
|
+
#
|
529
|
+
# where **field** is one of +status.state+, +clusterName+, or +labels.[KEY]+,
|
530
|
+
# and +[KEY]+ is a label key. **value** can be +*+ to match all values.
|
531
|
+
# +status.state+ can be one of the following: +ACTIVE+, +INACTIVE+,
|
532
|
+
# +CREATING+, +RUNNING+, +ERROR+, +DELETING+, or +UPDATING+. +ACTIVE+
|
533
|
+
# contains the +CREATING+, +UPDATING+, and +RUNNING+ states. +INACTIVE+
|
534
|
+
# contains the +DELETING+ and +ERROR+ states.
|
535
|
+
# +clusterName+ is the name of the cluster provided at creation time.
|
536
|
+
# Only the logical +AND+ operator is supported; space-separated items are
|
537
|
+
# treated as having an implicit +AND+ operator.
|
538
|
+
#
|
539
|
+
# Example filter:
|
540
|
+
#
|
541
|
+
# status.state = ACTIVE AND clusterName = mycluster
|
542
|
+
# AND labels.env = staging AND labels.starred = *
|
543
|
+
# @param page_size [Integer]
|
544
|
+
# The maximum number of resources contained in the underlying API
|
545
|
+
# response. If page streaming is performed per-resource, this
|
546
|
+
# parameter does not affect the return value. If page streaming is
|
547
|
+
# performed per-page, this determines the maximum number of
|
548
|
+
# resources in a page.
|
549
|
+
# @param options [Google::Gax::CallOptions]
|
550
|
+
# Overrides the default settings for this call, e.g, timeout,
|
551
|
+
# retries, etc.
|
552
|
+
# @return [Google::Gax::PagedEnumerable<Google::Cloud::Dataproc::V1::Cluster>]
|
553
|
+
# An enumerable of Google::Cloud::Dataproc::V1::Cluster instances.
|
554
|
+
# See Google::Gax::PagedEnumerable documentation for other
|
555
|
+
# operations such as per-page iteration or access to the response
|
556
|
+
# object.
|
557
|
+
# @raise [Google::Gax::GaxError] if the RPC is aborted.
|
558
|
+
# @example
|
559
|
+
# require "google/cloud/dataproc/v1"
|
560
|
+
#
|
561
|
+
# cluster_controller_client = Google::Cloud::Dataproc::V1::ClusterController.new
|
562
|
+
# project_id = ''
|
563
|
+
# region = ''
|
564
|
+
#
|
565
|
+
# # Iterate over all results.
|
566
|
+
# cluster_controller_client.list_clusters(project_id, region).each do |element|
|
567
|
+
# # Process element.
|
568
|
+
# end
|
569
|
+
#
|
570
|
+
# # Or iterate over results one page at a time.
|
571
|
+
# cluster_controller_client.list_clusters(project_id, region).each_page do |page|
|
572
|
+
# # Process each page at a time.
|
573
|
+
# page.each do |element|
|
574
|
+
# # Process element.
|
575
|
+
# end
|
576
|
+
# end
|
577
|
+
|
578
|
+
def list_clusters \
|
579
|
+
project_id,
|
580
|
+
region,
|
581
|
+
filter: nil,
|
582
|
+
page_size: nil,
|
583
|
+
options: nil
|
584
|
+
req = {
|
585
|
+
project_id: project_id,
|
586
|
+
region: region,
|
587
|
+
filter: filter,
|
588
|
+
page_size: page_size
|
589
|
+
}.delete_if { |_, v| v.nil? }
|
590
|
+
req = Google::Gax::to_proto(req, Google::Cloud::Dataproc::V1::ListClustersRequest)
|
591
|
+
@list_clusters.call(req, options)
|
592
|
+
end
|
593
|
+
|
594
|
+
# Gets cluster diagnostic information.
|
595
|
+
# After the operation completes, the Operation.response field
|
596
|
+
# contains +DiagnoseClusterOutputLocation+.
|
597
|
+
#
|
598
|
+
# @param project_id [String]
|
599
|
+
# Required. The ID of the Google Cloud Platform project that the cluster
|
600
|
+
# belongs to.
|
601
|
+
# @param region [String]
|
602
|
+
# Required. The Cloud Dataproc region in which to handle the request.
|
603
|
+
# @param cluster_name [String]
|
604
|
+
# Required. The cluster name.
|
605
|
+
# @param options [Google::Gax::CallOptions]
|
606
|
+
# Overrides the default settings for this call, e.g, timeout,
|
607
|
+
# retries, etc.
|
608
|
+
# @return [Google::Gax::Operation]
|
609
|
+
# @raise [Google::Gax::GaxError] if the RPC is aborted.
|
610
|
+
# @example
|
611
|
+
# require "google/cloud/dataproc/v1"
|
612
|
+
#
|
613
|
+
# cluster_controller_client = Google::Cloud::Dataproc::V1::ClusterController.new
|
614
|
+
# project_id = ''
|
615
|
+
# region = ''
|
616
|
+
# cluster_name = ''
|
617
|
+
#
|
618
|
+
# # Register a callback during the method call.
|
619
|
+
# operation = cluster_controller_client.diagnose_cluster(project_id, region, cluster_name) do |op|
|
620
|
+
# raise op.results.message if op.error?
|
621
|
+
# op_results = op.results
|
622
|
+
# # Process the results.
|
623
|
+
#
|
624
|
+
# metadata = op.metadata
|
625
|
+
# # Process the metadata.
|
626
|
+
# end
|
627
|
+
#
|
628
|
+
# # Or use the return value to register a callback.
|
629
|
+
# operation.on_done do |op|
|
630
|
+
# raise op.results.message if op.error?
|
631
|
+
# op_results = op.results
|
632
|
+
# # Process the results.
|
633
|
+
#
|
634
|
+
# metadata = op.metadata
|
635
|
+
# # Process the metadata.
|
636
|
+
# end
|
637
|
+
#
|
638
|
+
# # Manually reload the operation.
|
639
|
+
# operation.reload!
|
640
|
+
#
|
641
|
+
# # Or block until the operation completes, triggering callbacks on
|
642
|
+
# # completion.
|
643
|
+
# operation.wait_until_done!
|
644
|
+
|
645
|
+
def diagnose_cluster \
|
646
|
+
project_id,
|
647
|
+
region,
|
648
|
+
cluster_name,
|
649
|
+
options: nil
|
650
|
+
req = {
|
651
|
+
project_id: project_id,
|
652
|
+
region: region,
|
653
|
+
cluster_name: cluster_name
|
654
|
+
}.delete_if { |_, v| v.nil? }
|
655
|
+
req = Google::Gax::to_proto(req, Google::Cloud::Dataproc::V1::DiagnoseClusterRequest)
|
656
|
+
operation = Google::Gax::Operation.new(
|
657
|
+
@diagnose_cluster.call(req, options),
|
658
|
+
@operations_client,
|
659
|
+
Google::Protobuf::Empty,
|
660
|
+
Google::Cloud::Dataproc::V1::DiagnoseClusterResults,
|
661
|
+
call_options: options
|
662
|
+
)
|
663
|
+
operation.on_done { |operation| yield(operation) } if block_given?
|
664
|
+
operation
|
665
|
+
end
|
666
|
+
end
|
667
|
+
end
|
668
|
+
end
|
669
|
+
end
|
670
|
+
end
|