google-cloud-dataproc 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,56 @@
1
+ {
2
+ "interfaces": {
3
+ "google.cloud.dataproc.v1.ClusterController": {
4
+ "retry_codes": {
5
+ "idempotent": [
6
+ "DEADLINE_EXCEEDED",
7
+ "UNAVAILABLE"
8
+ ],
9
+ "non_idempotent": []
10
+ },
11
+ "retry_params": {
12
+ "default": {
13
+ "initial_retry_delay_millis": 100,
14
+ "retry_delay_multiplier": 1.3,
15
+ "max_retry_delay_millis": 60000,
16
+ "initial_rpc_timeout_millis": 10000,
17
+ "rpc_timeout_multiplier": 1.0,
18
+ "max_rpc_timeout_millis": 10000,
19
+ "total_timeout_millis": 300000
20
+ }
21
+ },
22
+ "methods": {
23
+ "CreateCluster": {
24
+ "timeout_millis": 90000,
25
+ "retry_codes_name": "non_idempotent",
26
+ "retry_params_name": "default"
27
+ },
28
+ "UpdateCluster": {
29
+ "timeout_millis": 30000,
30
+ "retry_codes_name": "non_idempotent",
31
+ "retry_params_name": "default"
32
+ },
33
+ "DeleteCluster": {
34
+ "timeout_millis": 15000,
35
+ "retry_codes_name": "idempotent",
36
+ "retry_params_name": "default"
37
+ },
38
+ "GetCluster": {
39
+ "timeout_millis": 10000,
40
+ "retry_codes_name": "idempotent",
41
+ "retry_params_name": "default"
42
+ },
43
+ "ListClusters": {
44
+ "timeout_millis": 10000,
45
+ "retry_codes_name": "idempotent",
46
+ "retry_params_name": "default"
47
+ },
48
+ "DiagnoseCluster": {
49
+ "timeout_millis": 10000,
50
+ "retry_codes_name": "non_idempotent",
51
+ "retry_params_name": "default"
52
+ }
53
+ }
54
+ }
55
+ }
56
+ }
@@ -0,0 +1,166 @@
1
+ # Generated by the protocol buffer compiler. DO NOT EDIT!
2
+ # source: google/cloud/dataproc/v1/clusters.proto
3
+
4
+ require 'google/protobuf'
5
+
6
+ require 'google/api/annotations_pb'
7
+ require 'google/cloud/dataproc/v1/operations_pb'
8
+ require 'google/longrunning/operations_pb'
9
+ require 'google/protobuf/duration_pb'
10
+ require 'google/protobuf/field_mask_pb'
11
+ require 'google/protobuf/timestamp_pb'
12
+ Google::Protobuf::DescriptorPool.generated_pool.build do
13
+ add_message "google.cloud.dataproc.v1.Cluster" do
14
+ optional :project_id, :string, 1
15
+ optional :cluster_name, :string, 2
16
+ optional :config, :message, 3, "google.cloud.dataproc.v1.ClusterConfig"
17
+ map :labels, :string, :string, 8
18
+ optional :status, :message, 4, "google.cloud.dataproc.v1.ClusterStatus"
19
+ repeated :status_history, :message, 7, "google.cloud.dataproc.v1.ClusterStatus"
20
+ optional :cluster_uuid, :string, 6
21
+ optional :metrics, :message, 9, "google.cloud.dataproc.v1.ClusterMetrics"
22
+ end
23
+ add_message "google.cloud.dataproc.v1.ClusterConfig" do
24
+ optional :config_bucket, :string, 1
25
+ optional :gce_cluster_config, :message, 8, "google.cloud.dataproc.v1.GceClusterConfig"
26
+ optional :master_config, :message, 9, "google.cloud.dataproc.v1.InstanceGroupConfig"
27
+ optional :worker_config, :message, 10, "google.cloud.dataproc.v1.InstanceGroupConfig"
28
+ optional :secondary_worker_config, :message, 12, "google.cloud.dataproc.v1.InstanceGroupConfig"
29
+ optional :software_config, :message, 13, "google.cloud.dataproc.v1.SoftwareConfig"
30
+ repeated :initialization_actions, :message, 11, "google.cloud.dataproc.v1.NodeInitializationAction"
31
+ end
32
+ add_message "google.cloud.dataproc.v1.GceClusterConfig" do
33
+ optional :zone_uri, :string, 1
34
+ optional :network_uri, :string, 2
35
+ optional :subnetwork_uri, :string, 6
36
+ optional :internal_ip_only, :bool, 7
37
+ optional :service_account, :string, 8
38
+ repeated :service_account_scopes, :string, 3
39
+ repeated :tags, :string, 4
40
+ map :metadata, :string, :string, 5
41
+ end
42
+ add_message "google.cloud.dataproc.v1.InstanceGroupConfig" do
43
+ optional :num_instances, :int32, 1
44
+ repeated :instance_names, :string, 2
45
+ optional :image_uri, :string, 3
46
+ optional :machine_type_uri, :string, 4
47
+ optional :disk_config, :message, 5, "google.cloud.dataproc.v1.DiskConfig"
48
+ optional :is_preemptible, :bool, 6
49
+ optional :managed_group_config, :message, 7, "google.cloud.dataproc.v1.ManagedGroupConfig"
50
+ repeated :accelerators, :message, 8, "google.cloud.dataproc.v1.AcceleratorConfig"
51
+ end
52
+ add_message "google.cloud.dataproc.v1.ManagedGroupConfig" do
53
+ optional :instance_template_name, :string, 1
54
+ optional :instance_group_manager_name, :string, 2
55
+ end
56
+ add_message "google.cloud.dataproc.v1.AcceleratorConfig" do
57
+ optional :accelerator_type_uri, :string, 1
58
+ optional :accelerator_count, :int32, 2
59
+ end
60
+ add_message "google.cloud.dataproc.v1.DiskConfig" do
61
+ optional :boot_disk_size_gb, :int32, 1
62
+ optional :num_local_ssds, :int32, 2
63
+ end
64
+ add_message "google.cloud.dataproc.v1.NodeInitializationAction" do
65
+ optional :executable_file, :string, 1
66
+ optional :execution_timeout, :message, 2, "google.protobuf.Duration"
67
+ end
68
+ add_message "google.cloud.dataproc.v1.ClusterStatus" do
69
+ optional :state, :enum, 1, "google.cloud.dataproc.v1.ClusterStatus.State"
70
+ optional :detail, :string, 2
71
+ optional :state_start_time, :message, 3, "google.protobuf.Timestamp"
72
+ optional :substate, :enum, 4, "google.cloud.dataproc.v1.ClusterStatus.Substate"
73
+ end
74
+ add_enum "google.cloud.dataproc.v1.ClusterStatus.State" do
75
+ value :UNKNOWN, 0
76
+ value :CREATING, 1
77
+ value :RUNNING, 2
78
+ value :ERROR, 3
79
+ value :DELETING, 4
80
+ value :UPDATING, 5
81
+ end
82
+ add_enum "google.cloud.dataproc.v1.ClusterStatus.Substate" do
83
+ value :UNSPECIFIED, 0
84
+ value :UNHEALTHY, 1
85
+ value :STALE_STATUS, 2
86
+ end
87
+ add_message "google.cloud.dataproc.v1.SoftwareConfig" do
88
+ optional :image_version, :string, 1
89
+ map :properties, :string, :string, 2
90
+ end
91
+ add_message "google.cloud.dataproc.v1.ClusterMetrics" do
92
+ map :hdfs_metrics, :string, :int64, 1
93
+ map :yarn_metrics, :string, :int64, 2
94
+ end
95
+ add_message "google.cloud.dataproc.v1.CreateClusterRequest" do
96
+ optional :project_id, :string, 1
97
+ optional :region, :string, 3
98
+ optional :cluster, :message, 2, "google.cloud.dataproc.v1.Cluster"
99
+ end
100
+ add_message "google.cloud.dataproc.v1.UpdateClusterRequest" do
101
+ optional :project_id, :string, 1
102
+ optional :region, :string, 5
103
+ optional :cluster_name, :string, 2
104
+ optional :cluster, :message, 3, "google.cloud.dataproc.v1.Cluster"
105
+ optional :update_mask, :message, 4, "google.protobuf.FieldMask"
106
+ end
107
+ add_message "google.cloud.dataproc.v1.DeleteClusterRequest" do
108
+ optional :project_id, :string, 1
109
+ optional :region, :string, 3
110
+ optional :cluster_name, :string, 2
111
+ end
112
+ add_message "google.cloud.dataproc.v1.GetClusterRequest" do
113
+ optional :project_id, :string, 1
114
+ optional :region, :string, 3
115
+ optional :cluster_name, :string, 2
116
+ end
117
+ add_message "google.cloud.dataproc.v1.ListClustersRequest" do
118
+ optional :project_id, :string, 1
119
+ optional :region, :string, 4
120
+ optional :filter, :string, 5
121
+ optional :page_size, :int32, 2
122
+ optional :page_token, :string, 3
123
+ end
124
+ add_message "google.cloud.dataproc.v1.ListClustersResponse" do
125
+ repeated :clusters, :message, 1, "google.cloud.dataproc.v1.Cluster"
126
+ optional :next_page_token, :string, 2
127
+ end
128
+ add_message "google.cloud.dataproc.v1.DiagnoseClusterRequest" do
129
+ optional :project_id, :string, 1
130
+ optional :region, :string, 3
131
+ optional :cluster_name, :string, 2
132
+ end
133
+ add_message "google.cloud.dataproc.v1.DiagnoseClusterResults" do
134
+ optional :output_uri, :string, 1
135
+ end
136
+ end
137
+
138
+ module Google
139
+ module Cloud
140
+ module Dataproc
141
+ module V1
142
+ Cluster = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.Cluster").msgclass
143
+ ClusterConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.ClusterConfig").msgclass
144
+ GceClusterConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.GceClusterConfig").msgclass
145
+ InstanceGroupConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.InstanceGroupConfig").msgclass
146
+ ManagedGroupConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.ManagedGroupConfig").msgclass
147
+ AcceleratorConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.AcceleratorConfig").msgclass
148
+ DiskConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.DiskConfig").msgclass
149
+ NodeInitializationAction = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.NodeInitializationAction").msgclass
150
+ ClusterStatus = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.ClusterStatus").msgclass
151
+ ClusterStatus::State = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.ClusterStatus.State").enummodule
152
+ ClusterStatus::Substate = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.ClusterStatus.Substate").enummodule
153
+ SoftwareConfig = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.SoftwareConfig").msgclass
154
+ ClusterMetrics = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.ClusterMetrics").msgclass
155
+ CreateClusterRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.CreateClusterRequest").msgclass
156
+ UpdateClusterRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.UpdateClusterRequest").msgclass
157
+ DeleteClusterRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.DeleteClusterRequest").msgclass
158
+ GetClusterRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.GetClusterRequest").msgclass
159
+ ListClustersRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.ListClustersRequest").msgclass
160
+ ListClustersResponse = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.ListClustersResponse").msgclass
161
+ DiagnoseClusterRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.DiagnoseClusterRequest").msgclass
162
+ DiagnoseClusterResults = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1.DiagnoseClusterResults").msgclass
163
+ end
164
+ end
165
+ end
166
+ end
@@ -0,0 +1,58 @@
1
+ # Generated by the protocol buffer compiler. DO NOT EDIT!
2
+ # Source: google/cloud/dataproc/v1/clusters.proto for package 'google.cloud.dataproc.v1'
3
+ # Original file comments:
4
+ # Copyright 2017 Google Inc.
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ require 'grpc'
20
+ require 'google/cloud/dataproc/v1/clusters_pb'
21
+
22
+ module Google
23
+ module Cloud
24
+ module Dataproc
25
+ module V1
26
+ module ClusterController
27
+ # The ClusterControllerService provides methods to manage clusters
28
+ # of Google Compute Engine instances.
29
+ class Service
30
+
31
+ include GRPC::GenericService
32
+
33
+ self.marshal_class_method = :encode
34
+ self.unmarshal_class_method = :decode
35
+ self.service_name = 'google.cloud.dataproc.v1.ClusterController'
36
+
37
+ # Creates a cluster in a project.
38
+ rpc :CreateCluster, CreateClusterRequest, Google::Longrunning::Operation
39
+ # Updates a cluster in a project.
40
+ rpc :UpdateCluster, UpdateClusterRequest, Google::Longrunning::Operation
41
+ # Deletes a cluster in a project.
42
+ rpc :DeleteCluster, DeleteClusterRequest, Google::Longrunning::Operation
43
+ # Gets the resource representation for a cluster in a project.
44
+ rpc :GetCluster, GetClusterRequest, Cluster
45
+ # Lists all regions/{region}/clusters in a project.
46
+ rpc :ListClusters, ListClustersRequest, ListClustersResponse
47
+ # Gets cluster diagnostic information.
48
+ # After the operation completes, the Operation.response field
49
+ # contains `DiagnoseClusterOutputLocation`.
50
+ rpc :DiagnoseCluster, DiagnoseClusterRequest, Google::Longrunning::Operation
51
+ end
52
+
53
+ Stub = Service.rpc_stub_class
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,565 @@
1
+ # Copyright 2017 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module Google
16
+ module Cloud
17
+ module Dataproc
18
+ ##
19
+ # # Google Cloud Dataproc API Contents
20
+ #
21
+ # | Class | Description |
22
+ # | ----- | ----------- |
23
+ # | [ClusterControllerClient][] | Manages Hadoop-based clusters and jobs on Google Cloud Platform. |
24
+ # | [JobControllerClient][] | Manages Hadoop-based clusters and jobs on Google Cloud Platform. |
25
+ # | [Data Types][] | Data types for Google::Cloud::Dataproc::V1 |
26
+ #
27
+ # [ClusterControllerClient]: https://googlecloudplatform.github.io/google-cloud-ruby/#/docs/google-cloud-dataproc/latest/google/cloud/dataproc/v1/clustercontrollerclient
28
+ # [JobControllerClient]: https://googlecloudplatform.github.io/google-cloud-ruby/#/docs/google-cloud-dataproc/latest/google/cloud/dataproc/v1/jobcontrollerclient
29
+ # [Data Types]: https://googlecloudplatform.github.io/google-cloud-ruby/#/docs/google-cloud-dataproc/latest/google/cloud/dataproc/v1/datatypes
30
+ #
31
+ module V1
32
+ # Describes the identifying information, config, and status of
33
+ # a cluster of Google Compute Engine instances.
34
+ # @!attribute [rw] project_id
35
+ # @return [String]
36
+ # Required. The Google Cloud Platform project ID that the cluster belongs to.
37
+ # @!attribute [rw] cluster_name
38
+ # @return [String]
39
+ # Required. The cluster name. Cluster names within a project must be
40
+ # unique. Names of deleted clusters can be reused.
41
+ # @!attribute [rw] config
42
+ # @return [Google::Cloud::Dataproc::V1::ClusterConfig]
43
+ # Required. The cluster config. Note that Cloud Dataproc may set
44
+ # default values, and values may change when clusters are updated.
45
+ # @!attribute [rw] labels
46
+ # @return [Hash{String => String}]
47
+ # Optional. The labels to associate with this cluster.
48
+ # Label **keys** must contain 1 to 63 characters, and must conform to
49
+ # [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
50
+ # Label **values** may be empty, but, if present, must contain 1 to 63
51
+ # characters, and must conform to [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
52
+ # No more than 32 labels can be associated with a cluster.
53
+ # @!attribute [rw] status
54
+ # @return [Google::Cloud::Dataproc::V1::ClusterStatus]
55
+ # Output-only. Cluster status.
56
+ # @!attribute [rw] status_history
57
+ # @return [Array<Google::Cloud::Dataproc::V1::ClusterStatus>]
58
+ # Output-only. The previous cluster status.
59
+ # @!attribute [rw] cluster_uuid
60
+ # @return [String]
61
+ # Output-only. A cluster UUID (Unique Universal Identifier). Cloud Dataproc
62
+ # generates this value when it creates the cluster.
63
+ # @!attribute [rw] metrics
64
+ # @return [Google::Cloud::Dataproc::V1::ClusterMetrics]
65
+ # Contains cluster daemon metrics such as HDFS and YARN stats.
66
+ #
67
+ # **Beta Feature**: This report is available for testing purposes only. It may
68
+ # be changed before final release.
69
+ class Cluster; end
70
+
71
+ # The cluster config.
72
+ # @!attribute [rw] config_bucket
73
+ # @return [String]
74
+ # Optional. A Google Cloud Storage staging bucket used for sharing generated
75
+ # SSH keys and config. If you do not specify a staging bucket, Cloud
76
+ # Dataproc will determine an appropriate Cloud Storage location (US,
77
+ # ASIA, or EU) for your cluster's staging bucket according to the Google
78
+ # Compute Engine zone where your cluster is deployed, and then it will create
79
+ # and manage this project-level, per-location bucket for you.
80
+ # @!attribute [rw] gce_cluster_config
81
+ # @return [Google::Cloud::Dataproc::V1::GceClusterConfig]
82
+ # Required. The shared Google Compute Engine config settings for
83
+ # all instances in a cluster.
84
+ # @!attribute [rw] master_config
85
+ # @return [Google::Cloud::Dataproc::V1::InstanceGroupConfig]
86
+ # Optional. The Google Compute Engine config settings for
87
+ # the master instance in a cluster.
88
+ # @!attribute [rw] worker_config
89
+ # @return [Google::Cloud::Dataproc::V1::InstanceGroupConfig]
90
+ # Optional. The Google Compute Engine config settings for
91
+ # worker instances in a cluster.
92
+ # @!attribute [rw] secondary_worker_config
93
+ # @return [Google::Cloud::Dataproc::V1::InstanceGroupConfig]
94
+ # Optional. The Google Compute Engine config settings for
95
+ # additional worker instances in a cluster.
96
+ # @!attribute [rw] software_config
97
+ # @return [Google::Cloud::Dataproc::V1::SoftwareConfig]
98
+ # Optional. The config settings for software inside the cluster.
99
+ # @!attribute [rw] initialization_actions
100
+ # @return [Array<Google::Cloud::Dataproc::V1::NodeInitializationAction>]
101
+ # Optional. Commands to execute on each node after config is
102
+ # completed. By default, executables are run on master and all worker nodes.
103
+ # You can test a node's +role+ metadata to run an executable on
104
+ # a master or worker node, as shown below using +curl+ (you can also use +wget+):
105
+ #
106
+ # ROLE=$(curl -H Metadata-Flavor:Google http://metadata/computeMetadata/v1/instance/attributes/dataproc-role)
107
+ # if [[ "${ROLE}" == 'Master' ]]; then
108
+ # ... master specific actions ...
109
+ # else
110
+ # ... worker specific actions ...
111
+ # fi
112
+ class ClusterConfig; end
113
+
114
+ # Common config settings for resources of Google Compute Engine cluster
115
+ # instances, applicable to all instances in the cluster.
116
+ # @!attribute [rw] zone_uri
117
+ # @return [String]
118
+ # Optional. The zone where the Google Compute Engine cluster will be located.
119
+ # On a create request, it is required in the "global" region. If omitted
120
+ # in a non-global Cloud Dataproc region, the service will pick a zone in the
121
+ # corresponding Compute Engine region. On a get request, zone will
122
+ # always be present.
123
+ #
124
+ # A full URL, partial URI, or short name are valid. Examples:
125
+ #
126
+ # * +https://www.googleapis.com/compute/v1/projects/[project_id]/zones/[zone]+
127
+ # * +projects/[project_id]/zones/[zone]+
128
+ # * +us-central1-f+
129
+ # @!attribute [rw] network_uri
130
+ # @return [String]
131
+ # Optional. The Google Compute Engine network to be used for machine
132
+ # communications. Cannot be specified with subnetwork_uri. If neither
133
+ # +network_uri+ nor +subnetwork_uri+ is specified, the "default" network of
134
+ # the project is used, if it exists. Cannot be a "Custom Subnet Network" (see
135
+ # [Using Subnetworks](https://cloud.google.com/compute/docs/subnetworks) for more information).
136
+ #
137
+ # A full URL, partial URI, or short name are valid. Examples:
138
+ #
139
+ # * +https://www.googleapis.com/compute/v1/projects/[project_id]/regions/global/default+
140
+ # * +projects/[project_id]/regions/global/default+
141
+ # * +default+
142
+ # @!attribute [rw] subnetwork_uri
143
+ # @return [String]
144
+ # Optional. The Google Compute Engine subnetwork to be used for machine
145
+ # communications. Cannot be specified with network_uri.
146
+ #
147
+ # A full URL, partial URI, or short name are valid. Examples:
148
+ #
149
+ # * +https://www.googleapis.com/compute/v1/projects/[project_id]/regions/us-east1/sub0+
150
+ # * +projects/[project_id]/regions/us-east1/sub0+
151
+ # * +sub0+
152
+ # @!attribute [rw] internal_ip_only
153
+ # @return [true, false]
154
+ # Optional. If true, all instances in the cluster will only have internal IP
155
+ # addresses. By default, clusters are not restricted to internal IP addresses,
156
+ # and will have ephemeral external IP addresses assigned to each instance.
157
+ # This +internal_ip_only+ restriction can only be enabled for subnetwork
158
+ # enabled networks, and all off-cluster dependencies must be configured to be
159
+ # accessible without external IP addresses.
160
+ # @!attribute [rw] service_account
161
+ # @return [String]
162
+ # Optional. The service account of the instances. Defaults to the default
163
+ # Google Compute Engine service account. Custom service accounts need
164
+ # permissions equivalent to the folloing IAM roles:
165
+ #
166
+ # * roles/logging.logWriter
167
+ # * roles/storage.objectAdmin
168
+ #
169
+ # (see https://cloud.google.com/compute/docs/access/service-accounts#custom_service_accounts
170
+ # for more information).
171
+ # Example: +[account_id]@[project_id].iam.gserviceaccount.com+
172
+ # @!attribute [rw] service_account_scopes
173
+ # @return [Array<String>]
174
+ # Optional. The URIs of service account scopes to be included in Google
175
+ # Compute Engine instances. The following base set of scopes is always
176
+ # included:
177
+ #
178
+ # * https://www.googleapis.com/auth/cloud.useraccounts.readonly
179
+ # * https://www.googleapis.com/auth/devstorage.read_write
180
+ # * https://www.googleapis.com/auth/logging.write
181
+ #
182
+ # If no scopes are specified, the following defaults are also provided:
183
+ #
184
+ # * https://www.googleapis.com/auth/bigquery
185
+ # * https://www.googleapis.com/auth/bigtable.admin.table
186
+ # * https://www.googleapis.com/auth/bigtable.data
187
+ # * https://www.googleapis.com/auth/devstorage.full_control
188
+ # @!attribute [rw] tags
189
+ # @return [Array<String>]
190
+ # The Google Compute Engine tags to add to all instances (see
191
+ # [Tagging instances](https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).
192
+ # @!attribute [rw] metadata
193
+ # @return [Hash{String => String}]
194
+ # The Google Compute Engine metadata entries to add to all instances (see
195
+ # [Project and instance metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).
196
+ class GceClusterConfig; end
197
+
198
+ # Optional. The config settings for Google Compute Engine resources in
199
+ # an instance group, such as a master or worker group.
200
+ # @!attribute [rw] num_instances
201
+ # @return [Integer]
202
+ # Optional. The number of VM instances in the instance group.
203
+ # For master instance groups, must be set to 1.
204
+ # @!attribute [rw] instance_names
205
+ # @return [Array<String>]
206
+ # Optional. The list of instance names. Cloud Dataproc derives the names from
207
+ # +cluster_name+, +num_instances+, and the instance group if not set by user
208
+ # (recommended practice is to let Cloud Dataproc derive the name).
209
+ # @!attribute [rw] image_uri
210
+ # @return [String]
211
+ # Output-only. The Google Compute Engine image resource used for cluster
212
+ # instances. Inferred from +SoftwareConfig.image_version+.
213
+ # @!attribute [rw] machine_type_uri
214
+ # @return [String]
215
+ # Optional. The Google Compute Engine machine type used for cluster instances.
216
+ #
217
+ # A full URL, partial URI, or short name are valid. Examples:
218
+ #
219
+ # * +https://www.googleapis.com/compute/v1/projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2+
220
+ # * +projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2+
221
+ # * +n1-standard-2+
222
+ # @!attribute [rw] disk_config
223
+ # @return [Google::Cloud::Dataproc::V1::DiskConfig]
224
+ # Optional. Disk option config settings.
225
+ # @!attribute [rw] is_preemptible
226
+ # @return [true, false]
227
+ # Optional. Specifies that this instance group contains preemptible instances.
228
+ # @!attribute [rw] managed_group_config
229
+ # @return [Google::Cloud::Dataproc::V1::ManagedGroupConfig]
230
+ # Output-only. The config for Google Compute Engine Instance Group
231
+ # Manager that manages this group.
232
+ # This is only used for preemptible instance groups.
233
+ # @!attribute [rw] accelerators
234
+ # @return [Array<Google::Cloud::Dataproc::V1::AcceleratorConfig>]
235
+ # Optional. The Google Compute Engine accelerator configuration for these
236
+ # instances.
237
+ #
238
+ # **Beta Feature**: This feature is still under development. It may be
239
+ # changed before final release.
240
+ class InstanceGroupConfig; end
241
+
242
+ # Specifies the resources used to actively manage an instance group.
243
+ # @!attribute [rw] instance_template_name
244
+ # @return [String]
245
+ # Output-only. The name of the Instance Template used for the Managed
246
+ # Instance Group.
247
+ # @!attribute [rw] instance_group_manager_name
248
+ # @return [String]
249
+ # Output-only. The name of the Instance Group Manager for this group.
250
+ class ManagedGroupConfig; end
251
+
252
+ # Specifies the type and number of accelerator cards attached to the instances
253
+ # of an instance group (see [GPUs on Compute Engine](https://cloud.google.com/compute/docs/gpus/)).
254
+ # @!attribute [rw] accelerator_type_uri
255
+ # @return [String]
256
+ # Full URL, partial URI, or short name of the accelerator type resource to
257
+ # expose to this instance. See [Google Compute Engine AcceleratorTypes](
258
+ # /compute/docs/reference/beta/acceleratorTypes)
259
+ #
260
+ # Examples
261
+ # * +https://www.googleapis.com/compute/beta/projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80+
262
+ # * +projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80+
263
+ # * +nvidia-tesla-k80+
264
+ # @!attribute [rw] accelerator_count
265
+ # @return [Integer]
266
+ # The number of the accelerator cards of this type exposed to this instance.
267
+ class AcceleratorConfig; end
268
+
269
+ # Specifies the config of disk options for a group of VM instances.
270
+ # @!attribute [rw] boot_disk_size_gb
271
+ # @return [Integer]
272
+ # Optional. Size in GB of the boot disk (default is 500GB).
273
+ # @!attribute [rw] num_local_ssds
274
+ # @return [Integer]
275
+ # Optional. Number of attached SSDs, from 0 to 4 (default is 0).
276
+ # If SSDs are not attached, the boot disk is used to store runtime logs and
277
+ # [HDFS](https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data.
278
+ # If one or more SSDs are attached, this runtime bulk
279
+ # data is spread across them, and the boot disk contains only basic
280
+ # config and installed binaries.
281
+ class DiskConfig; end
282
+
283
+ # Specifies an executable to run on a fully configured node and a
284
+ # timeout period for executable completion.
285
+ # @!attribute [rw] executable_file
286
+ # @return [String]
287
+ # Required. Google Cloud Storage URI of executable file.
288
+ # @!attribute [rw] execution_timeout
289
+ # @return [Google::Protobuf::Duration]
290
+ # Optional. Amount of time executable has to complete. Default is
291
+ # 10 minutes. Cluster creation fails with an explanatory error message (the
292
+ # name of the executable that caused the error and the exceeded timeout
293
+ # period) if the executable is not completed at end of the timeout period.
294
+ class NodeInitializationAction; end
295
+
296
+ # The status of a cluster and its instances.
297
+ # @!attribute [rw] state
298
+ # @return [Google::Cloud::Dataproc::V1::ClusterStatus::State]
299
+ # Output-only. The cluster's state.
300
+ # @!attribute [rw] detail
301
+ # @return [String]
302
+ # Output-only. Optional details of cluster's state.
303
+ # @!attribute [rw] state_start_time
304
+ # @return [Google::Protobuf::Timestamp]
305
+ # Output-only. Time when this state was entered.
306
+ # @!attribute [rw] substate
307
+ # @return [Google::Cloud::Dataproc::V1::ClusterStatus::Substate]
308
+ # Output-only. Additional state information that includes
309
+ # status reported by the agent.
310
+ class ClusterStatus
311
+ # The cluster state.
312
+ module State
313
+ # The cluster state is unknown.
314
+ UNKNOWN = 0
315
+
316
+ # The cluster is being created and set up. It is not ready for use.
317
+ CREATING = 1
318
+
319
+ # The cluster is currently running and healthy. It is ready for use.
320
+ RUNNING = 2
321
+
322
+ # The cluster encountered an error. It is not ready for use.
323
+ ERROR = 3
324
+
325
+ # The cluster is being deleted. It cannot be used.
326
+ DELETING = 4
327
+
328
+ # The cluster is being updated. It continues to accept and process jobs.
329
+ UPDATING = 5
330
+ end
331
+
332
+ module Substate
333
+ UNSPECIFIED = 0
334
+
335
+ # The cluster is known to be in an unhealthy state
336
+ # (for example, critical daemons are not running or HDFS capacity is
337
+ # exhausted).
338
+ #
339
+ # Applies to RUNNING state.
340
+ UNHEALTHY = 1
341
+
342
+ # The agent-reported status is out of date (may occur if
343
+ # Cloud Dataproc loses communication with Agent).
344
+ #
345
+ # Applies to RUNNING state.
346
+ STALE_STATUS = 2
347
+ end
348
+ end
349
+
350
+ # Specifies the selection and config of software inside the cluster.
351
+ # @!attribute [rw] image_version
352
+ # @return [String]
353
+ # Optional. The version of software inside the cluster. It must match the
354
+ # regular expression +[0-9]+\.[0-9]++. If unspecified, it defaults to the
355
+ # latest version (see [Cloud Dataproc Versioning](https://cloud.google.com/dataproc/versioning)).
356
+ # @!attribute [rw] properties
357
+ # @return [Hash{String => String}]
358
+ # Optional. The properties to set on daemon config files.
359
+ #
360
+ # Property keys are specified in +prefix:property+ format, such as
361
+ # +core:fs.defaultFS+. The following are supported prefixes
362
+ # and their mappings:
363
+ #
364
+ # * capacity-scheduler: +capacity-scheduler.xml+
365
+ # * core: +core-site.xml+
366
+ # * distcp: +distcp-default.xml+
367
+ # * hdfs: +hdfs-site.xml+
368
+ # * hive: +hive-site.xml+
369
+ # * mapred: +mapred-site.xml+
370
+ # * pig: +pig.properties+
371
+ # * spark: +spark-defaults.conf+
372
+ # * yarn: +yarn-site.xml+
373
+ #
374
+ # For more information, see
375
+ # [Cluster properties](https://cloud.google.com/dataproc/docs/concepts/cluster-properties).
376
+ class SoftwareConfig; end
377
+
378
+ # Contains cluster daemon metrics, such as HDFS and YARN stats.
379
+ #
380
+ # **Beta Feature**: This report is available for testing purposes only. It may
381
+ # be changed before final release.
382
+ # @!attribute [rw] hdfs_metrics
383
+ # @return [Hash{String => Integer}]
384
+ # The HDFS metrics.
385
+ # @!attribute [rw] yarn_metrics
386
+ # @return [Hash{String => Integer}]
387
+ # The YARN metrics.
388
+ class ClusterMetrics; end
389
+
390
+ # A request to create a cluster.
391
+ # @!attribute [rw] project_id
392
+ # @return [String]
393
+ # Required. The ID of the Google Cloud Platform project that the cluster
394
+ # belongs to.
395
+ # @!attribute [rw] region
396
+ # @return [String]
397
+ # Required. The Cloud Dataproc region in which to handle the request.
398
+ # @!attribute [rw] cluster
399
+ # @return [Google::Cloud::Dataproc::V1::Cluster]
400
+ # Required. The cluster to create.
401
+ class CreateClusterRequest; end
402
+
403
+ # A request to update a cluster.
404
+ # @!attribute [rw] project_id
405
+ # @return [String]
406
+ # Required. The ID of the Google Cloud Platform project the
407
+ # cluster belongs to.
408
+ # @!attribute [rw] region
409
+ # @return [String]
410
+ # Required. The Cloud Dataproc region in which to handle the request.
411
+ # @!attribute [rw] cluster_name
412
+ # @return [String]
413
+ # Required. The cluster name.
414
+ # @!attribute [rw] cluster
415
+ # @return [Google::Cloud::Dataproc::V1::Cluster]
416
+ # Required. The changes to the cluster.
417
+ # @!attribute [rw] update_mask
418
+ # @return [Google::Protobuf::FieldMask]
419
+ # Required. Specifies the path, relative to +Cluster+, of
420
+ # the field to update. For example, to change the number of workers
421
+ # in a cluster to 5, the +update_mask+ parameter would be
422
+ # specified as +config.worker_config.num_instances+,
423
+ # and the +PATCH+ request body would specify the new value, as follows:
424
+ #
425
+ # {
426
+ # "config":{
427
+ # "workerConfig":{
428
+ # "numInstances":"5"
429
+ # }
430
+ # }
431
+ # }
432
+ # Similarly, to change the number of preemptible workers in a cluster to 5,
433
+ # the +update_mask+ parameter would be
434
+ # +config.secondary_worker_config.num_instances+, and the +PATCH+ request
435
+ # body would be set as follows:
436
+ #
437
+ # {
438
+ # "config":{
439
+ # "secondaryWorkerConfig":{
440
+ # "numInstances":"5"
441
+ # }
442
+ # }
443
+ # }
444
+ # <strong>Note:</strong> Currently, only the following fields can be updated:
445
+ #
446
+ # <table>
447
+ # <tbody>
448
+ # <tr>
449
+ # <td><strong>Mask</strong></td>
450
+ # <td><strong>Purpose</strong></td>
451
+ # </tr>
452
+ # <tr>
453
+ # <td><strong><em>labels</em></strong></td>
454
+ # <td>Update labels</td>
455
+ # </tr>
456
+ # <tr>
457
+ # <td><strong><em>config.worker_config.num_instances</em></strong></td>
458
+ # <td>Resize primary worker group</td>
459
+ # </tr>
460
+ # <tr>
461
+ # <td><strong><em>config.secondary_worker_config.num_instances</em></strong></td>
462
+ # <td>Resize secondary worker group</td>
463
+ # </tr>
464
+ # </tbody>
465
+ # </table>
466
+ class UpdateClusterRequest; end
467
+
468
+ # A request to delete a cluster.
469
+ # @!attribute [rw] project_id
470
+ # @return [String]
471
+ # Required. The ID of the Google Cloud Platform project that the cluster
472
+ # belongs to.
473
+ # @!attribute [rw] region
474
+ # @return [String]
475
+ # Required. The Cloud Dataproc region in which to handle the request.
476
+ # @!attribute [rw] cluster_name
477
+ # @return [String]
478
+ # Required. The cluster name.
479
+ class DeleteClusterRequest; end
480
+
481
+ # Request to get the resource representation for a cluster in a project.
482
+ # @!attribute [rw] project_id
483
+ # @return [String]
484
+ # Required. The ID of the Google Cloud Platform project that the cluster
485
+ # belongs to.
486
+ # @!attribute [rw] region
487
+ # @return [String]
488
+ # Required. The Cloud Dataproc region in which to handle the request.
489
+ # @!attribute [rw] cluster_name
490
+ # @return [String]
491
+ # Required. The cluster name.
492
+ class GetClusterRequest; end
493
+
494
+ # A request to list the clusters in a project.
495
+ # @!attribute [rw] project_id
496
+ # @return [String]
497
+ # Required. The ID of the Google Cloud Platform project that the cluster
498
+ # belongs to.
499
+ # @!attribute [rw] region
500
+ # @return [String]
501
+ # Required. The Cloud Dataproc region in which to handle the request.
502
+ # @!attribute [rw] filter
503
+ # @return [String]
504
+ # Optional. A filter constraining the clusters to list. Filters are
505
+ # case-sensitive and have the following syntax:
506
+ #
507
+ # field = value [AND [field = value]] ...
508
+ #
509
+ # where **field** is one of +status.state+, +clusterName+, or +labels.[KEY]+,
510
+ # and +[KEY]+ is a label key. **value** can be +*+ to match all values.
511
+ # +status.state+ can be one of the following: +ACTIVE+, +INACTIVE+,
512
+ # +CREATING+, +RUNNING+, +ERROR+, +DELETING+, or +UPDATING+. +ACTIVE+
513
+ # contains the +CREATING+, +UPDATING+, and +RUNNING+ states. +INACTIVE+
514
+ # contains the +DELETING+ and +ERROR+ states.
515
+ # +clusterName+ is the name of the cluster provided at creation time.
516
+ # Only the logical +AND+ operator is supported; space-separated items are
517
+ # treated as having an implicit +AND+ operator.
518
+ #
519
+ # Example filter:
520
+ #
521
+ # status.state = ACTIVE AND clusterName = mycluster
522
+ # AND labels.env = staging AND labels.starred = *
523
+ # @!attribute [rw] page_size
524
+ # @return [Integer]
525
+ # Optional. The standard List page size.
526
+ # @!attribute [rw] page_token
527
+ # @return [String]
528
+ # Optional. The standard List page token.
529
+ class ListClustersRequest; end
530
+
531
+ # The list of all clusters in a project.
532
+ # @!attribute [rw] clusters
533
+ # @return [Array<Google::Cloud::Dataproc::V1::Cluster>]
534
+ # Output-only. The clusters in the project.
535
+ # @!attribute [rw] next_page_token
536
+ # @return [String]
537
+ # Output-only. This token is included in the response if there are more
538
+ # results to fetch. To fetch additional results, provide this value as the
539
+ # +page_token+ in a subsequent +ListClustersRequest+.
540
+ class ListClustersResponse; end
541
+
542
+ # A request to collect cluster diagnostic information.
543
+ # @!attribute [rw] project_id
544
+ # @return [String]
545
+ # Required. The ID of the Google Cloud Platform project that the cluster
546
+ # belongs to.
547
+ # @!attribute [rw] region
548
+ # @return [String]
549
+ # Required. The Cloud Dataproc region in which to handle the request.
550
+ # @!attribute [rw] cluster_name
551
+ # @return [String]
552
+ # Required. The cluster name.
553
+ class DiagnoseClusterRequest; end
554
+
555
+ # The location of diagnostic output.
556
+ # @!attribute [rw] output_uri
557
+ # @return [String]
558
+ # Output-only. The Google Cloud Storage URI of the diagnostic output.
559
+ # The output report is a plain text file with a summary of collected
560
+ # diagnostics.
561
+ class DiagnoseClusterResults; end
562
+ end
563
+ end
564
+ end
565
+ end