google-cloud-dataproc 0.3.1 → 0.4.0

data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/jobs.rb

@@ -289,6 +289,38 @@ module Google
         #     Optional. The runtime log config for job execution.
         class PigJob; end
 
+        # A Cloud Dataproc job for running
+        # [Apache SparkR](https://spark.apache.org/docs/latest/sparkr.html)
+        # applications on YARN.
+        # @!attribute [rw] main_r_file_uri
+        #   @return [String]
+        #     Required. The HCFS URI of the main R file to use as the driver.
+        #     Must be a .R file.
+        # @!attribute [rw] args
+        #   @return [Array<String>]
+        #     Optional. The arguments to pass to the driver.  Do not include arguments,
+        #     such as `--conf`, that can be set as job properties, since a collision may
+        #     occur that causes an incorrect job submission.
+        # @!attribute [rw] file_uris
+        #   @return [Array<String>]
+        #     Optional. HCFS URIs of files to be copied to the working directory of
+        #     R drivers and distributed tasks. Useful for naively parallel tasks.
+        # @!attribute [rw] archive_uris
+        #   @return [Array<String>]
+        #     Optional. HCFS URIs of archives to be extracted in the working directory of
+        #     Spark drivers and tasks. Supported file types:
+        #     .jar, .tar, .tar.gz, .tgz, and .zip.
+        # @!attribute [rw] properties
+        #   @return [Hash{String => String}]
+        #     Optional. A mapping of property names to values, used to configure SparkR.
+        #     Properties that conflict with values set by the Cloud Dataproc API may be
+        #     overwritten. Can include properties set in
+        #     /etc/spark/conf/spark-defaults.conf and classes in user code.
+        # @!attribute [rw] logging_config
+        #   @return [Google::Cloud::Dataproc::V1beta2::LoggingConfig]
+        #     Optional. The runtime log config for job execution.
+        class SparkRJob; end
+
         # Cloud Dataproc job config.
         # @!attribute [rw] cluster_name
         #   @return [String]
@@ -386,11 +418,12 @@ module Google
         #     belongs to.
         # @!attribute [rw] job_id
         #   @return [String]
-        #     Optional. The job ID, which must be unique within the project. The job ID
-        #     is generated by the server upon job submission or provided by the user as a
-        #     means to perform retries without creating duplicate jobs. The ID must
-        #     contain only letters (a-z, A-Z), numbers (0-9), underscores (_), or
-        #     hyphens (-). The maximum length is 100 characters.
+        #     Optional. The job ID, which must be unique within the project.
+        #
+        #     The ID must contain only letters (a-z, A-Z), numbers (0-9),
+        #     underscores (_), or hyphens (-). The maximum length is 100 characters.
+        #
+        #     If not specified by the caller, the job ID will be provided by the server.
         class JobReference; end
 
         # A YARN application created by a job. Application information is a subset of
@@ -472,6 +505,9 @@ module Google
         # @!attribute [rw] pig_job
         #   @return [Google::Cloud::Dataproc::V1beta2::PigJob]
         #     Job is a Pig job.
+        # @!attribute [rw] spark_r_job
+        #   @return [Google::Cloud::Dataproc::V1beta2::SparkRJob]
+        #     Job is a SparkR job.
         # @!attribute [rw] spark_sql_job
         #   @return [Google::Cloud::Dataproc::V1beta2::SparkSqlJob]
         #     Job is a SparkSql job.
@@ -548,10 +584,9 @@ module Google
         # @!attribute [rw] request_id
         #   @return [String]
         #     Optional. A unique id used to identify the request. If the server
-        #     receives two
-        #     {Google::Cloud::Dataproc::V1beta2::SubmitJobRequest SubmitJobRequest} requests
-        #     with the same id, then the second request will be ignored and the first
-        #     {Google::Cloud::Dataproc::V1beta2::Job Job} created and stored in the backend
+        #     receives two {Google::Cloud::Dataproc::V1beta2::SubmitJobRequest SubmitJobRequest} requests  with the same
+        #     id, then the second request will be ignored and the
+        #     first {Google::Cloud::Dataproc::V1beta2::Job Job} created and stored in the backend
         #     is returned.
         #
         #     It is recommended to always set this value to a

data/lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/workflow_templates.rb

@@ -25,6 +25,8 @@ module Google
         #     The id must contain only letters (a-z, A-Z), numbers (0-9),
         #     underscores (_), and hyphens (-). Cannot begin or end with underscore
         #     or hyphen. Must consist of between 3 and 50 characters.
+        #
+        #     .
         # @!attribute [rw] name
         #   @return [String]
         #     Output only. The "resource name" of the template, as described
@@ -136,8 +138,8 @@ module Google
         #
         #     The step id is used as prefix for job id, as job
         #     `goog-dataproc-workflow-step-id` label, and in
-        #     {Google::Cloud::Dataproc::V1beta2::OrderedJob#prerequisite_step_ids prerequisiteStepIds}
-        #     field from other steps.
+        #     {Google::Cloud::Dataproc::V1beta2::OrderedJob#prerequisite_step_ids prerequisiteStepIds} field from other
+        #     steps.
         #
         #     The id must contain only letters (a-z, A-Z), numbers (0-9),
         #     underscores (_), and hyphens (-). Cannot begin or end with underscore
@@ -205,10 +207,10 @@ module Google
         #     A field is allowed to appear in at most one parameter's list of field
         #     paths.
         #
-        #     A field path is similar in syntax to a
-        #     {Google::Protobuf::FieldMask}. For example, a
-        #     field path that references the zone field of a workflow template's cluster
-        #     selector would be specified as `placement.clusterSelector.zone`.
+        #     A field path is similar in syntax to a {Google::Protobuf::FieldMask}.
+        #     For example, a field path that references the zone field of a workflow
+        #     template's cluster selector would be specified as
+        #     `placement.clusterSelector.zone`.
         #
         #     Also, field paths can reference fields using the following syntax:
         #

data/lib/google/cloud/dataproc/v1beta2/job_controller_client.rb

@@ -27,6 +27,7 @@ require "google/gax"
 
 require "google/cloud/dataproc/v1beta2/jobs_pb"
 require "google/cloud/dataproc/v1beta2/credentials"
+require "google/cloud/dataproc/version"
 
 module Google
   module Cloud
@@ -129,7 +130,7 @@ module Google
               updater_proc = credentials.updater_proc
             end
 
-            package_version = Gem.loaded_specs['google-cloud-dataproc'].version.version
+            package_version = Google::Cloud::Dataproc::VERSION
 
             google_api_client = "gl-ruby/#{RUBY_VERSION}"
             google_api_client << " #{lib_name}/#{lib_version}" if lib_name
@@ -217,10 +218,9 @@ module Google
           #   can also be provided.
           # @param request_id [String]
           #   Optional. A unique id used to identify the request. If the server
-          #   receives two
-          #   {Google::Cloud::Dataproc::V1beta2::SubmitJobRequest SubmitJobRequest} requests
-          #   with the same id, then the second request will be ignored and the first
-          #   {Google::Cloud::Dataproc::V1beta2::Job Job} created and stored in the backend
+          #   receives two {Google::Cloud::Dataproc::V1beta2::SubmitJobRequest SubmitJobRequest} requests  with the same
+          #   id, then the second request will be ignored and the
+          #   first {Google::Cloud::Dataproc::V1beta2::Job Job} created and stored in the backend
           #   is returned.
           #
           #   It is recommended to always set this value to a

data/lib/google/cloud/dataproc/v1beta2/jobs_pb.rb

@@ -91,6 +91,14 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
       optional :query_list, :message, 2, "google.cloud.dataproc.v1beta2.QueryList"
     end
   end
+  add_message "google.cloud.dataproc.v1beta2.SparkRJob" do
+    optional :main_r_file_uri, :string, 1
+    repeated :args, :string, 2
+    repeated :file_uris, :string, 3
+    repeated :archive_uris, :string, 4
+    map :properties, :string, :string, 5
+    optional :logging_config, :message, 6, "google.cloud.dataproc.v1beta2.LoggingConfig"
+  end
   add_message "google.cloud.dataproc.v1beta2.JobPlacement" do
     optional :cluster_name, :string, 1
     optional :cluster_uuid, :string, 2
@@ -158,6 +166,7 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
       optional :pyspark_job, :message, 5, "google.cloud.dataproc.v1beta2.PySparkJob"
       optional :hive_job, :message, 6, "google.cloud.dataproc.v1beta2.HiveJob"
       optional :pig_job, :message, 7, "google.cloud.dataproc.v1beta2.PigJob"
+      optional :spark_r_job, :message, 21, "google.cloud.dataproc.v1beta2.SparkRJob"
       optional :spark_sql_job, :message, 12, "google.cloud.dataproc.v1beta2.SparkSqlJob"
     end
   end
@@ -225,6 +234,7 @@ module Google
         HiveJob = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.HiveJob").msgclass
         SparkSqlJob = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.SparkSqlJob").msgclass
         PigJob = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.PigJob").msgclass
+        SparkRJob = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.SparkRJob").msgclass
         JobPlacement = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.JobPlacement").msgclass
         JobStatus = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.JobStatus").msgclass
         JobStatus::State = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.JobStatus.State").enummodule

data/lib/google/cloud/dataproc/v1beta2/jobs_services_pb.rb

@@ -1,7 +1,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # Source: google/cloud/dataproc/v1beta2/jobs.proto for package 'google.cloud.dataproc.v1beta2'
 # Original file comments:
-# Copyright 2018 Google LLC.
+# Copyright 2019 Google LLC.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/google/cloud/dataproc/v1beta2/shared_pb.rb

@@ -6,12 +6,24 @@ require 'google/protobuf'
 
 require 'google/api/annotations_pb'
 Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_enum "google.cloud.dataproc.v1beta2.Component" do
+    value :COMPONENT_UNSPECIFIED, 0
+    value :ANACONDA, 5
+    value :DRUID, 9
+    value :HIVE_WEBHCAT, 3
+    value :JUPYTER, 1
+    value :KERBEROS, 7
+    value :PRESTO, 6
+    value :ZEPPELIN, 4
+    value :ZOOKEEPER, 8
+  end
 end
 
 module Google
   module Cloud
     module Dataproc
       module V1beta2
+        Component = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.cloud.dataproc.v1beta2.Component").enummodule
       end
     end
   end

data/lib/google/cloud/dataproc/v1beta2/workflow_template_service_client.rb

@@ -29,6 +29,7 @@ require "google/longrunning/operations_client"
 
 require "google/cloud/dataproc/v1beta2/workflow_templates_pb"
 require "google/cloud/dataproc/v1beta2/credentials"
+require "google/cloud/dataproc/version"
 
 module Google
   module Cloud
@@ -181,7 +182,7 @@ module Google
               updater_proc = credentials.updater_proc
             end
 
-            package_version = Gem.loaded_specs['google-cloud-dataproc'].version.version
+            package_version = Google::Cloud::Dataproc::VERSION
 
             google_api_client = "gl-ruby/#{RUBY_VERSION}"
             google_api_client << " #{lib_name}/#{lib_version}" if lib_name
@@ -477,8 +478,7 @@ module Google
           # Instantiates a template and begins execution.
           #
           # This method is equivalent to executing the sequence
-          # {Google::Cloud::Dataproc::V1beta2::WorkflowTemplateService::CreateWorkflowTemplate CreateWorkflowTemplate},
-          # {Google::Cloud::Dataproc::V1beta2::WorkflowTemplateService::InstantiateWorkflowTemplate InstantiateWorkflowTemplate},
+          # {Google::Cloud::Dataproc::V1beta2::WorkflowTemplateService::CreateWorkflowTemplate CreateWorkflowTemplate}, {Google::Cloud::Dataproc::V1beta2::WorkflowTemplateService::InstantiateWorkflowTemplate InstantiateWorkflowTemplate},
           # {Google::Cloud::Dataproc::V1beta2::WorkflowTemplateService::DeleteWorkflowTemplate DeleteWorkflowTemplate}.
           #
           # The returned Operation can be used to track execution of

data/lib/google/cloud/dataproc/v1beta2/workflow_templates_services_pb.rb

@@ -1,7 +1,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # Source: google/cloud/dataproc/v1beta2/workflow_templates.proto for package 'google.cloud.dataproc.v1beta2'
 # Original file comments:
-# Copyright 2018 Google LLC.
+# Copyright 2019 Google LLC.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -65,8 +65,7 @@ module Google
             # Instantiates a template and begins execution.
             #
             # This method is equivalent to executing the sequence
-            # [CreateWorkflowTemplate][google.cloud.dataproc.v1beta2.WorkflowTemplateService.CreateWorkflowTemplate],
-            # [InstantiateWorkflowTemplate][google.cloud.dataproc.v1beta2.WorkflowTemplateService.InstantiateWorkflowTemplate],
+            # [CreateWorkflowTemplate][google.cloud.dataproc.v1beta2.WorkflowTemplateService.CreateWorkflowTemplate], [InstantiateWorkflowTemplate][google.cloud.dataproc.v1beta2.WorkflowTemplateService.InstantiateWorkflowTemplate],
             # [DeleteWorkflowTemplate][google.cloud.dataproc.v1beta2.WorkflowTemplateService.DeleteWorkflowTemplate].
             #
             # The returned Operation can be used to track execution of

data/lib/google/cloud/dataproc/version.rb

@@ -0,0 +1,22 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+module Google
+  module Cloud
+    module Dataproc
+      VERSION = "0.4.0".freeze
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: google-cloud-dataproc
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Google LLC
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-29 00:00:00.000000000 Z
+date: 2019-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-gax
@@ -127,16 +127,22 @@ files:
 - lib/google/cloud/dataproc/v1/jobs_pb.rb
 - lib/google/cloud/dataproc/v1/jobs_services_pb.rb
 - lib/google/cloud/dataproc/v1/operations_pb.rb
+- lib/google/cloud/dataproc/v1/shared_pb.rb
 - lib/google/cloud/dataproc/v1/workflow_template_service_client.rb
 - lib/google/cloud/dataproc/v1/workflow_template_service_client_config.json
 - lib/google/cloud/dataproc/v1/workflow_templates_pb.rb
 - lib/google/cloud/dataproc/v1/workflow_templates_services_pb.rb
 - lib/google/cloud/dataproc/v1beta2.rb
+- lib/google/cloud/dataproc/v1beta2/autoscaling_policies_pb.rb
+- lib/google/cloud/dataproc/v1beta2/autoscaling_policies_services_pb.rb
+- lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service_client.rb
+- lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service_client_config.json
 - lib/google/cloud/dataproc/v1beta2/cluster_controller_client.rb
 - lib/google/cloud/dataproc/v1beta2/cluster_controller_client_config.json
 - lib/google/cloud/dataproc/v1beta2/clusters_pb.rb
 - lib/google/cloud/dataproc/v1beta2/clusters_services_pb.rb
 - lib/google/cloud/dataproc/v1beta2/credentials.rb
+- lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/autoscaling_policies.rb
 - lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/clusters.rb
 - lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/jobs.rb
 - lib/google/cloud/dataproc/v1beta2/doc/google/cloud/dataproc/v1beta2/workflow_templates.rb
@@ -157,6 +163,7 @@ files:
 - lib/google/cloud/dataproc/v1beta2/workflow_template_service_client_config.json
 - lib/google/cloud/dataproc/v1beta2/workflow_templates_pb.rb
 - lib/google/cloud/dataproc/v1beta2/workflow_templates_services_pb.rb
+- lib/google/cloud/dataproc/version.rb
 homepage: https://github.com/googleapis/google-cloud-ruby/tree/master/google-cloud-dataproc
 licenses:
 - Apache-2.0