mortar-api-ruby 0.8.11 → 0.8.12

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZGQ4MDJlY2NjODJlYzFiZmVhNzc0MWQ1ZjhmN2NlN2JmYzRhYzA2NQ==
+  data.tar.gz: !binary |-
+    ZjA0ODA2ZWM2MWM0YTY3NzIxYmE5NThlYmZhYzkwNGYwMzdkNGU2Mw==
+SHA512:
+  metadata.gz: !binary |-
+    OTJmZGIxOWY2MzVkODA5MzRjMGE1MmE2NTNhYzFkM2U5MjIwNjBhZjQ0OWNk
+    YjBlMTc3YmE3MDFjODRkMmNlMGZhMDE1YzU5ZTYxMGU2NjhjZGM1Njk2OGUy
+    MDYzZjQ4ZDliNjY0ZTQ2NDE2YjM0NDY5NTkxMWZiZjA5YjY3MmU=
+  data.tar.gz: !binary |-
+    MDQyMWUxYzliNWRhMjg3OTUwYWNlOTExZmM2Yjc4ODA2MjI0OTY5ZDM3ZGEx
+    YTdjMWExOGRlYmU5ODk4Y2MyMmFhNDNhZTRjZmZlNTlhM2MxODg4ZTM0YWZj
+    ZjhkNWQ2OGIxMjRhZTJmOTI5YmQ1ZTc1NmZhMWViM2IxNDEwMWI=

data/lib/mortar/api/clusters.rb

@@ -28,12 +28,13 @@ module Mortar
     end
     
     # GET /vX/clusters
-    def get_clusters()
+    def get_clusters(cluster_backend=Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1)
       request(
         :expects  => 200,
         :idempotent => true,
         :method   => :get,
-        :path     => versioned_path("/clusters")
+        :path     => "#{versioned_path("/clusters")}",
+        :query    => {:cluster_backend => cluster_backend}
       )
     end 
     

data/lib/mortar/api/jobs.rb

@@ -40,6 +40,11 @@ module Mortar
       LUIGI_JOB_STATUS__STOPPED           = 'stopped'
       LUIGI_JOB_STATUS__STOPPING          = 'stopping'
 
+      SPARK_JOB_STATUS__PENDING  = "pending"
+      SPARK_JOB_STATUS__STARTING = "starting"
+      SPARK_JOB_STATUS__RUNNING  = "running"
+      SPARK_JOB_STATUS__FINISHED = "finished"
+
 
       STATUSES_IN_PROGRESS    = Set.new([STATUS_STARTING,
                                          STATUS_GATEWAY_STARTING,
@@ -50,7 +55,10 @@ module Mortar
                                          LUIGI_JOB_STATUS__PENDING,
                                          LUIGI_JOB_STATUS__STARTING,
                                          LUIGI_JOB_STATUS__RUNNING,
-                                         LUIGI_JOB_STATUS__STOPPING])
+                                         LUIGI_JOB_STATUS__STOPPING,
+                                         SPARK_JOB_STATUS__PENDING,
+                                         SPARK_JOB_STATUS__STARTING,
+                                         SPARK_JOB_STATUS__RUNNING])
 
       STATUSES_COMPLETE       = Set.new([STATUS_SCRIPT_ERROR, 
                                         STATUS_PLAN_ERROR,
@@ -59,15 +67,21 @@ module Mortar
                                         STATUS_SERVICE_ERROR,
                                         STATUS_STOPPED,
                                         LUIGI_JOB_STATUS__FINISHED,
-                                        LUIGI_JOB_STATUS__STOPPED])
+                                        LUIGI_JOB_STATUS__STOPPED,
+                                        SPARK_JOB_STATUS__FINISHED])
 
       CLUSTER_TYPE__SINGLE_JOB = 'single_job'
       CLUSTER_TYPE__PERSISTENT = 'persistent'
       CLUSTER_TYPE__PERMANENT = 'permanent'
 
+      CLUSTER_BACKEND__EMR_HADOOP_1 = "EMR_HADOOP_1"
+      CLUSTER_BACKEND__EMR_HADOOP_2 = "EMR_HADOOP_2"
+      CLUSTER_BACKEND__ALL          = "ALL"
+
       JOB_TYPE_ALL   = 'all'
       JOB_TYPE_PIG   = 'pig'
       JOB_TYPE_LUIGI = 'luigi'
+      JOB_TYPE_SPARK = 'spark'
     end
     
     
@@ -119,6 +133,7 @@ module Mortar
         "git_ref" => git_ref,
         "cluster_size" => cluster_size,
         "cluster_type" => cluster_type,
+        "cluster_backend" => Jobs::CLUSTER_BACKEND__EMR_HADOOP_1,
         "parameters" => parameters,
         "notify_on_job_finish" => notify_on_job_finish,
         "job_type" => Jobs::JOB_TYPE_PIG,
@@ -165,6 +180,56 @@ module Mortar
         :body     => json_encode(body))
     end
 
+    def post_spark_job_new_cluster(project_name, sparkscript_name, git_ref, cluster_size, options={})
+      script_arguments = options[:script_arguments] || ""
+      use_spot_instances = options[:use_spot_instances].nil? ? false : options[:use_spot_instances]
+      cluster_type = options[:cluster_type].nil? ? Jobs::CLUSTER_TYPE__PERSISTENT : options[:cluster_type]
+
+
+      body = { "project_name" => project_name,
+        "git_ref" => git_ref,
+        "cluster_size" => cluster_size,
+        "cluster_type" => cluster_type,
+        "cluster_backend" => Jobs::CLUSTER_BACKEND__EMR_HADOOP_2,
+        "script_arguments" => script_arguments,
+        "job_type" => Jobs::JOB_TYPE_SPARK,
+        "use_spot_instances" => use_spot_instances,
+        "sparkscript_name" => sparkscript_name
+      }
+
+      unless options[:project_script_path].nil?
+        body["project_script_path"] = options[:project_script_path]
+      end
+
+      request(
+        :expects  => 200,
+        :method   => :post,
+        :path     => versioned_path("/jobs"),
+        :body     => json_encode(body))
+    end
+
+    def post_spark_job_existing_cluster(project_name, sparkscript_name, git_ref, cluster_id, options={})
+      script_arguments = options[:script_arguments] || ""      
+
+      body = {"project_name" => project_name,
+        "git_ref" => git_ref,
+        "cluster_id" => cluster_id,
+        "script_arguments" => script_arguments,
+        "job_type" => Jobs::JOB_TYPE_SPARK,
+        "sparkscript_name" => sparkscript_name
+      }
+
+      unless options[:project_script_path].nil?
+        body["project_script_path"] = options[:project_script_path]
+      end
+
+      request(
+        :expects  => 200,
+        :method   => :post,
+        :path     => versioned_path("/jobs"),
+        :body     => json_encode(body))
+    end
+
     # GET /vX/jobs
     def get_jobs(skip, limit, job_type=Jobs::JOB_TYPE_PIG)
       request(

data/lib/mortar/api/version.rb

@@ -18,7 +18,7 @@ module Mortar
   class API
     # client version
     # see http://semver.org/
-    VERSION = "0.8.11"
+    VERSION = "0.8.12"
     
     SERVER_API_VERSION = "2"
   end

data/spec/mortar/api/clusters_spec.rb

@@ -30,8 +30,8 @@ describe Mortar::API do
   
   context "clusters" do
 
-    it "gets recent and running clusters" do
-      Excon.stub({:method => :get, :path => "/v2/clusters"}) do |params|
+    it "gets recent and running clusters - no cluster backend" do
+      Excon.stub({:method => :get, :path => "/v2/clusters", :query => {:cluster_backend => Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1}}) do |params|
         {:body => Mortar::API::OkJson.encode({"clusters" => [{'cluster_id' => '1', 'status_code' => 'running'}, {'cluster_id' => '2', 'status_code' => 'running'}]}), :status => 200}
       end
       response = @api.get_clusters()
@@ -40,6 +40,16 @@ describe Mortar::API do
       clusters.length.should == 2
     end
     
+    it "gets recent and running clusters for hadoop 2 clusters" do
+      Excon.stub({:method => :get, :path => "/v2/clusters", :query => {:cluster_backend => Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_2}}) do |params|
+        {:body => Mortar::API::OkJson.encode({"clusters" => [{'cluster_id' => '1', 'status_code' => 'running'}, {'cluster_id' => '2', 'status_code' => 'running'}]}), :status => 200}
+      end
+      response = @api.get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_2)
+      clusters = response.body["clusters"]
+      clusters.nil?.should be_false
+      clusters.length.should == 2
+    end
+
     it "stops a running cluster" do
       cluster_id = "1234abc342221abc"
       Excon.stub({:method => :delete, :path => "/v2/clusters/#{cluster_id}"}) do |params|

data/spec/mortar/api/jobs_spec.rb

@@ -106,6 +106,7 @@ describe Mortar::API do
                                          "git_ref" => git_ref,
                                          "cluster_size" => cluster_size,
                                          "cluster_type" => cluster_type,
+                                         "cluster_backend" => Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1,
                                          "parameters" => {},
                                          "notify_on_job_finish" => true,
                                          "job_type" => "pig",
@@ -133,6 +134,7 @@ describe Mortar::API do
                                          "git_ref" => git_ref,
                                          "cluster_size" => cluster_size,
                                          "cluster_type" => cluster_type,
+                                         "cluster_backend" => Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1,
                                          "parameters" => {},
                                          "notify_on_job_finish" => true,
                                          "job_type" => "pig",
@@ -162,6 +164,7 @@ describe Mortar::API do
                                          "git_ref" => git_ref,
                                          "cluster_size" => cluster_size,
                                          "cluster_type" => cluster_type,
+                                         "cluster_backend" => Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1,
                                          "parameters" => {},
                                          "notify_on_job_finish" => false,
                                          "job_type" => "pig",
@@ -196,6 +199,7 @@ describe Mortar::API do
                                          "git_ref" => git_ref,
                                          "cluster_size" => cluster_size,
                                          "cluster_type" => cluster_type,
+                                         "cluster_backend" => Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1,
                                          "parameters" => {},
                                          "notify_on_job_finish" => true,
                                          "job_type" => "pig",
@@ -225,6 +229,7 @@ describe Mortar::API do
                                          "git_ref" => git_ref,
                                          "cluster_size" => cluster_size,
                                          "cluster_type" => cluster_type,
+                                         "cluster_backend" => Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1,
                                          "parameters" => {},
                                          "notify_on_job_finish" => true,
                                          "job_type" => "pig",
@@ -286,5 +291,57 @@ describe Mortar::API do
       response = @api.stop_job(job_id)
       response.body["success"].should be_true
     end
+
+    it "posts a spark job for a new cluster, defaulting cluster_type to persistent" do
+      job_id = "7b93e4d3ab034188a0c2be418d3d24ed"
+      project_name = "my_project"
+      sparkscript_name = "my_sparkscript"
+      git_ref = "e20395b8b06fbf52e86665b0660209673f311d1a"
+      cluster_size = 5
+      cluster_type = Mortar::API::Jobs::CLUSTER_TYPE__PERSISTENT
+      body = Mortar::API::OkJson.encode({"project_name" => project_name,
+                                         "git_ref" => git_ref,
+                                         "cluster_size" => cluster_size,
+                                         "cluster_type" => cluster_type,
+                                         "cluster_backend" => Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_2,
+                                         "script_arguments" => "",
+                                         "job_type" => "spark",
+                                         "use_spot_instances" => false,
+                                         "sparkscript_name" => sparkscript_name})
+      Excon.stub({:method => :post, :path => "/v2/jobs", :body => body}) do |params|
+        {:body => Mortar::API::OkJson.encode({'job_id' => job_id}), :status => 200}
+      end
+      response = @api.post_spark_job_new_cluster(project_name, sparkscript_name, git_ref, cluster_size)
+      response.body['job_id'].should == job_id
+    end
+
+    it "posts a spark job for an existing cluster" do
+      job_id = "7b93e4d3ab034188a0c2be418d3d24ed"
+      project_name = "my_project"
+      sparkscript_name = "my_sparkscript"
+      project_script_path = "sparkscripts/"
+      git_ref = "e20395b8b06fbf52e86665b0660209673f311d1a"
+      cluster_id = "f82c774f7ccd429e91db996838cb6c4a"
+      script_arguments = "some script args"
+      body = Mortar::API::OkJson.encode({"project_name" => project_name,
+                                         "git_ref" => git_ref,
+                                         "cluster_id" => cluster_id,
+                                         "script_arguments" => script_arguments,
+                                         "job_type" => "spark",
+                                         "sparkscript_name" => sparkscript_name,
+                                         "project_script_path" => project_script_path})
+      Excon.stub({:method => :post, :path => "/v2/jobs", :body => body}) do |params|
+        {:body => Mortar::API::OkJson.encode({'job_id' => job_id}), :status => 200}
+      end
+
+      response = @api.post_spark_job_existing_cluster(
+        project_name, 
+        sparkscript_name, 
+        git_ref, 
+        cluster_id, 
+        :script_arguments => script_arguments, 
+        :project_script_path => project_script_path)
+      response.body['job_id'].should == job_id
+    end
   end
 end

metadata

@@ -1,106 +1,91 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: mortar-api-ruby
-version: !ruby/object:Gem::Version 
-  hash: 41
-  prerelease: 
-  segments: 
-  - 0
-  - 8
-  - 11
-  version: 0.8.11
+version: !ruby/object:Gem::Version
+  version: 0.8.12
 platform: ruby
-authors: 
+authors:
 - Mortar Data
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2014-10-24 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2015-01-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: excon
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 51
-        segments: 
-        - 0
-        - 28
-        version: "0.28"
+      - !ruby/object:Gem::Version
+        version: '0.28'
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: gem-release
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.28'
+- !ruby/object:Gem::Dependency
+  name: gem-release
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: rake
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 73
-        segments: 
-        - 10
-        - 1
-        - 1
+      - !ruby/object:Gem::Version
         version: 10.1.1
   type: :development
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: rr
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 13
-        segments: 
-        - 1
-        - 1
-        version: "1.1"
+      - !ruby/object:Gem::Version
+        version: 10.1.1
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
   type: :development
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: rspec
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 2
-        - 0
-        version: "2.0"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
   type: :development
-  version_requirements: *id005
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
 description: Client for Mortar API.
 email: support@mortardata.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
 - .travis.yml
 - Gemfile
@@ -142,38 +127,40 @@ files:
 - spec/spec_helper.rb
 homepage: http://mortardata.com/
 licenses: []
-
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 57
-      segments: 
-      - 1
-      - 8
-      - 7
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
       version: 1.8.7
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Client for Mortar API
-test_files: []
-
+test_files:
+- spec/mortar/api/basicauth_spec.rb
+- spec/mortar/api/clusters_spec.rb
+- spec/mortar/api/config_spec.rb
+- spec/mortar/api/describe_spec.rb
+- spec/mortar/api/errors_spec.rb
+- spec/mortar/api/illustrate_spec.rb
+- spec/mortar/api/jobs_spec.rb
+- spec/mortar/api/login_spec.rb
+- spec/mortar/api/projects_spec.rb
+- spec/mortar/api/tasks_spec.rb
+- spec/mortar/api/user_spec.rb
+- spec/mortar/api/validate_spec.rb
+- spec/mortar/api_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb