mortar 0.15.38 → 0.15.39

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: ffc0b8718edcb166d6efb97aa313f93f62cbe05a
-  data.tar.gz: e50a708d63e897a055528aeb61b0a2a023c6cb78
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    N2RjODE0YjhlMmJjNDAyMjAxNmZhM2MxM2M2YTI1MjZlMjY2YWU4Yg==
+  data.tar.gz: !binary |-
+    MzI0Y2NiMWEwNTI2MWRjNGIxYjRiMDhhZGJiNTZjNzY5ZDZkMDAwOQ==
 SHA512:
-  metadata.gz: 0c57ea3f3a078fd0896b92973d5bd731bdedd2024c2ddfae3f74121dce10fddef75b374c4b6f6618345f724ce79963cf501065553b7494c919627aa64815b715
-  data.tar.gz: b62dc6bb899b601ea4c6162a2bcdf068d47e23d13d1acf75cfefb40bcefdd757d0475d0dd91d6325b7031352ea4a3097e96a733ec65123eff2d08f4ca0d96cc0
+  metadata.gz: !binary |-
+    MjQ0MWRiMWUxZGJlNTQ2YjI5ODljNzJkZjg3ZjUwZDAyYjVhZmY4OGU5NzUy
+    OWZkMzQ5MWEzYWMxZDg2ZmZkN2VhNmFjZjgyYWI3MGE5YzM3Nzg1MDY2ZmEy
+    NDdmZjljYzMwNjI1YTAyZTUxYWUxYjM3M2FmMzk0MTg2MzEzZWQ=
+  data.tar.gz: !binary |-
+    N2NhNjE5MThkNDQ2MjVmZTc1ZWEwMWNmYTJjZmM2OWE3NzdjMWI0NGYyODFk
+    NmZmYjgzYjNkMzAxNmI4MjliMDM0MzMzYTM3NTk3NmFjMGJiZjI5ZWNmYjkw
+    ZDc1MjI1YzQ5NmU5NzdlM2ZjMDU0ZmFjMmM0MzFmM2I5ZmY4MzY=

data/lib/mortar/command/jobs.rb

@@ -60,6 +60,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
   #
   # -c, --clusterid CLUSTERID   # Run job on an existing cluster with ID of CLUSTERID (optional)
   # -s, --clustersize NUMNODES  # Run job on a new cluster, with NUMNODES nodes (optional; must be >= 2 if provided)
+  # -t, --clustertags A,B,C     # Run job on an existing cluster with specified tags
   # -1, --singlejobcluster      # Stop the cluster after job completes.  (Default: false--cluster can be used for other jobs, and will shut down after 1 hour of inactivity)
   # -2, --permanentcluster      # Don't automatically stop the cluster after it has been idle for an hour (Default: false--cluster will be shut down after 1 hour of inactivity)
   # -3, --spot                  # Use spot instances for this cluster (Default: false, only applicable to new clusters)
@@ -112,21 +113,42 @@ class Mortar::Command::Jobs < Mortar::Command::Base
         error "Unknown Script Type"
       end
     end
-    
+
+    if options[:clustertags]
+      cluster_tags = options[:clustertags].split(',')
+    else
+      cluster_tags = []
+    end
+
     unless options[:clusterid] || options[:clustersize]
       clusters = api.get_clusters(pig_version.cluster_backend).body['clusters']
 
-      largest_free_cluster = clusters.select{ |c| \
-        c['running_jobs'].length == 0 && c['status_code'] == Mortar::API::Clusters::STATUS_RUNNING }.
-        max_by{|c| c['size']}
+      if cluster_tags.length > 0
+        tagged_clusters = clusters.select{
+          |c| c['status_code'] == Mortar::API::Clusters::STATUS_RUNNING && (cluster_tags - c['tags']).empty?
+        }
+        if tagged_clusters.length == 0
+          error "There're no clusters with tags [" + options[:clustertags] + "]"
+        end
 
-      if largest_free_cluster.nil?
-        options[:clustersize] = 2
-        display("Defaulting to running job on new cluster of size 2")
+        largest_cluster = tagged_clusters.max_by{|c| c['size']}
+
+        options[:clusterid] = largest_cluster['cluster_id']
+        display("Running job on the cluster with tags [" + options[:clustertags] + "], id = " + largest_cluster['cluster_id'] +
+          ", size = " + largest_cluster['size'].to_s)
       else
-        options[:clusterid] = largest_free_cluster['cluster_id']
-        display("Defaulting to running job on largest existing free cluster, id = " + 
-                largest_free_cluster['cluster_id'] + ", size = " + largest_free_cluster['size'].to_s)
+        largest_free_cluster = clusters.select{ |c| \
+          c['running_jobs'].length == 0 && c['status_code'] == Mortar::API::Clusters::STATUS_RUNNING }.
+          max_by{|c| c['size']}
+
+        if largest_free_cluster.nil?
+          options[:clustersize] = 2
+          display("Defaulting to running job on new cluster of size 2")
+        else
+          options[:clusterid] = largest_free_cluster['cluster_id']
+          display("Defaulting to running job on largest existing free cluster, id = " +
+                  largest_free_cluster['cluster_id'] + ", size = " + largest_free_cluster['size'].to_s)
+        end
       end
     end
       
@@ -137,7 +159,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
         end
       end
     end
- 
+
     if options[:project]
       if options[:branch]
         git_ref = options[:branch]

data/lib/mortar/command/spark.rb

@@ -29,6 +29,7 @@ class Mortar::Command::Spark < Mortar::Command::Base
   #
   # -c, --clusterid CLUSTERID   # Run job on an existing cluster with ID of CLUSTERID (Default: runs on an existing available cluster)
   # -s, --clustersize NUMNODES  # Run job with NUMNODES nodes (optional; must be >= 2 if provided)
+  # -t, --clustertags A,B,C     # Run job on an existing cluster with specified tags
   # -3, --spot                  # Use spot instances for this cluster (Default: true)
   # -P, --project PROJECTNAME   # Use a project that is not checked out in the current directory.  Runs code from project's master branch in GitHub rather than snapshotting local code.
   # -B, --branch BRANCHNAME     # Used with --project to specify a non-master branch
@@ -65,20 +66,45 @@ class Mortar::Command::Spark < Mortar::Command::Base
       git_ref = sync_code_with_cloud()
     end
 
+    if options[:clustertags]
+      cluster_tags = options[:clustertags].split(',')
+    else
+      cluster_tags = []
+    end
+
     unless options[:clusterid] || options[:clustersize]
       clusters = api.get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_SPARK_JOBSERVER).body['clusters']
 
-      largest_free_cluster = clusters.select{ |c| \
-        c['running_jobs'].length == 0 && c['status_code'] == Mortar::API::Clusters::STATUS_RUNNING }.
-        max_by{|c| c['size']}
-      
-      if largest_free_cluster.nil?
-        error('No running clusters with Spark Job Server detected, please, launch a SparkJobServer cluster first')
-      end
+      if cluster_tags.length > 0
+        tagged_clusters = clusters.select{
+          |c| c['status_code'] == Mortar::API::Clusters::STATUS_RUNNING && (cluster_tags - c['tags']).empty?
+        }
+        if tagged_clusters.length > 1
+          display(tagged_clusters)
+          error "There're " + tagged_clusters.length + " clusters with tags [" + options[:clustertags] +
+            "]. Please, select one cluster."
+        elsif tagged_clusters.length == 0
+          error "There're no clusters with tags [" + options[:clustertags] + "]"
+        end
+
+        largest_cluster = tagged_clusters.max_by{|c| c['size']}
 
-      options[:clusterid] = largest_free_cluster['cluster_id']
-      display("Defaulting to running job on largest existing free cluster, id = " +
-              largest_free_cluster['cluster_id'] + ", size = " + largest_free_cluster['size'].to_s)
+        options[:clusterid] = largest_cluster['cluster_id']
+        display("Running job on the cluster with tags [" + options[:clustertags] + "], id = " + largest_cluster['cluster_id'] +
+          ", size = " + largest_cluster['size'].to_s)
+      else
+        largest_free_cluster = clusters.select{ |c| \
+          c['running_jobs'].length == 0 && c['status_code'] == Mortar::API::Clusters::STATUS_RUNNING }.
+          max_by{|c| c['size']}
+
+        if largest_free_cluster.nil?
+          error('No running clusters with Spark Job Server detected, please, launch a SparkJobServer cluster first')
+        end
+
+        options[:clusterid] = largest_free_cluster['cluster_id']
+        display("Defaulting to running job on largest existing free cluster, id = " +
+                largest_free_cluster['cluster_id'] + ", size = " + largest_free_cluster['size'].to_s)
+      end
     end
 
     response = action("Requesting job execution") do

data/lib/mortar/project.rb

@@ -85,6 +85,10 @@ module Mortar
         @luigiscripts
       end
 
+      def sparkjobs_path
+        File.join(@root_path, "src/main/scala")
+      end
+        
       def sparkscripts_path
         File.join(@root_path, "sparkscripts")
       end

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.15.38"
+  VERSION = "0.15.39"
 end

data/spec/mortar/command/jobs_spec.rb

@@ -439,6 +439,65 @@ STDOUT
         end
       end
 
+      it "runs a job by default on the largest existing running cluster using cluster tags" do
+        with_git_initialized_project do |p|
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
+
+          small_cluster_id = '510beb6b3004860820ab6538'
+          small_cluster_size = 2
+          small_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
+          large_cluster_id = '510bf0db3004860820ab6590'
+          large_cluster_size = 5
+          large_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
+          starting_cluster_id = '510bf0db3004860820abaaaa'
+          starting_cluster_size = 10
+          starting_cluster_status = Mortar::API::Clusters::STATUS_STARTING
+          huge_busy_cluster_id = '510bf0db3004860820ab6621'
+          huge_busy_cluster_size = 20
+          huge_busy_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
+
+
+          mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_2) {
+            Excon::Response.new(:body => {
+              'clusters' => [
+                { 'cluster_id' => small_cluster_id, 'size' => small_cluster_size, 'running_jobs' => [], 'status_code' => small_cluster_status, 'tags' => ['small'] },
+                { 'cluster_id' => large_cluster_id, 'size' => large_cluster_size, 'running_jobs' => [], 'status_code' => large_cluster_status, 'tags' => ['large', 'cool'] },
+                { 'cluster_id' => starting_cluster_id, 'size' => starting_cluster_size, 'running_jobs' => [], 'status_code' => starting_cluster_status, 'tags' => ['starting'] },
+                { 'cluster_id' => huge_busy_cluster_id, 'size' => huge_busy_cluster_size, 'tags' => ['huge'],
+                  'running_jobs' => [ { 'job_id' => 'c571a8c7f76a4fd4a67c103d753e2dd5',
+                    'job_name' => "", 'start_timestamp' => ""} ], 'status_code' => huge_busy_cluster_status  }
+              ]})
+          }
+          mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), large_cluster_id,
+            :pig_version => "0.12-Hadoop-2",
+            :project_script_path => be_a_kind_of(String),
+            :parameters => [],
+            :notify_on_job_finish => true,
+            :is_control_script => false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+
+          write_file(File.join(p.pigscripts_path, "my_script.pig"))
+          stderr, stdout = execute("jobs:run pigscripts/my_script.pig --clustertags large,cool", p, @git)
+          stdout.should == <<-STDOUT
+Running job on the cluster with tags [large,cool], id = 510bf0db3004860820ab6590, size = 5
+Taking code snapshot... done
+Sending code snapshot to Mortar... done
+Requesting job execution... done
+job_id: c571a8c7f76a4fd4a67c103d753e2dd5
+
+Job status can be viewed on the web at:
+
+ http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
+
+Or by running:
+
+  mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 --poll
+
+          STDOUT
+
+        end
+      end
+
       it "runs a job by default on the largest existing running cluster for Hadoop 2" do
         with_git_initialized_project do |p|
           job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"

data/spec/mortar/command/spark_spec.rb

@@ -0,0 +1,205 @@
+#
+# Copyright 2014 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'spec_helper'
+require 'fakefs/spec_helpers'
+require 'mortar/command/spark'
+require 'mortar/api/jobs'
+
+module Mortar::Command
+  describe Spark do
+
+    before(:each) do
+      stub_core
+      @git = Mortar::Git::Git.new
+    end
+
+    context("index") do
+      it "shows help when user adds help argument" do
+        with_git_initialized_project do |p|
+          stderr_dash_h, stdout_dash_h = execute("spark -h", p, @git)
+          stderr_help, stdout_help = execute("spark help", p, @git)
+          stdout_dash_h.should == stdout_help
+          stderr_dash_h.should == stderr_help
+        end
+      end
+
+      it "runs a spark job with no arguments new cluster" do
+        with_git_initialized_project do |p|
+          # stub api requests
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          job_url = "http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
+          cluster_id = nil
+          mock(Mortar::Auth.api).post_spark_job_on_jobserver("myproject", "com.mortar.Job", is_a(String), cluster_id,
+            :script_arguments => '',
+            :clustersize => '4'
+          ) { Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url}) }
+
+          write_file(File.join(p.sparkjobs_path, "com/mortar/Job.scala"))
+          stderr, stdout = execute("spark com.mortar.Job --clustersize 4", p, @git)
+          puts stderr
+          stdout.should == <<-STDOUT
+Taking code snapshot... done
+Sending code snapshot to Mortar... done
+Requesting job execution... done
+job_id: c571a8c7f76a4fd4a67c103d753e2dd5
+
+Job status can be viewed on the web at:
+
+ http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
+
+          STDOUT
+        end
+      end
+
+      it "runs a spark job with script_arguments existing cluster" do
+        with_git_initialized_project do |p|
+          # stub api requests
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          cluster_id = "c571a8c7f76a4fd4a67c103d753e2dd7"
+          job_url = "http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
+          script_arguments = "--arg1 v1 --arg2 v2 --arg3"
+          mock(Mortar::Auth.api).post_spark_job_on_jobserver("myproject", "com.mortar.Job", is_a(String), cluster_id,
+            :script_arguments => script_arguments
+          ) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+
+          write_file(File.join(p.sparkjobs_path, "com/mortar/Job.scala"))
+          stderr, stdout = execute("spark com.mortar.Job --clusterid #{cluster_id} #{script_arguments}", p, @git)
+          stdout.should == <<-STDOUT
+Taking code snapshot... done
+Sending code snapshot to Mortar... done
+Requesting job execution... done
+job_id: c571a8c7f76a4fd4a67c103d753e2dd5
+
+Job status can be viewed on the web at:
+
+ http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
+
+          STDOUT
+        end
+      end
+
+      it "runs a spark job with on free cluster" do
+        with_git_initialized_project do |p|
+          # stub api requests
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          job_url = "http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
+          script_arguments = "arg1 arg2 arg3"
+
+          small_cluster_id = '510beb6b3004860820ab6538'
+          small_cluster_size = 2
+          small_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
+          large_cluster_id = '510bf0db3004860820ab6590'
+          large_cluster_size = 5
+          large_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
+          starting_cluster_id = '510bf0db3004860820abaaaa'
+          starting_cluster_size = 10
+          starting_cluster_status = Mortar::API::Clusters::STATUS_STARTING
+          huge_busy_cluster_id = '510bf0db3004860820ab6621'
+          huge_busy_cluster_size = 20
+          huge_busy_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
+
+
+          mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_SPARK_JOBSERVER) {
+            Excon::Response.new(:body => {
+              'clusters' => [
+                { 'cluster_id' => small_cluster_id, 'size' => small_cluster_size, 'running_jobs' => [], 'status_code' => small_cluster_status },
+                { 'cluster_id' => large_cluster_id, 'size' => large_cluster_size, 'running_jobs' => [], 'status_code' => large_cluster_status },
+                { 'cluster_id' => starting_cluster_id, 'size' => starting_cluster_size, 'running_jobs' => [], 'status_code' => starting_cluster_status },
+                { 'cluster_id' => huge_busy_cluster_id, 'size' => huge_busy_cluster_size,
+                  'running_jobs' => [ { 'job_id' => 'c571a8c7f76a4fd4a67c103d753e2dd5',
+                    'job_name' => "", 'start_timestamp' => ""} ], 'status_code' => huge_busy_cluster_status  }
+              ]})
+          }
+
+          mock(Mortar::Auth.api).post_spark_job_on_jobserver("myproject", "com.mortar.Job", is_a(String), large_cluster_id,
+            :script_arguments => script_arguments
+          ) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+
+          write_file(File.join(p.sparkjobs_path, "com/mortar/Job.scala"))
+          stderr, stdout = execute("spark com.mortar.Job #{script_arguments}", p, @git)
+          stdout.should == <<-STDOUT
+Taking code snapshot... done
+Sending code snapshot to Mortar... done
+Defaulting to running job on largest existing free cluster, id = 510bf0db3004860820ab6590, size = 5
+Requesting job execution... done
+job_id: c571a8c7f76a4fd4a67c103d753e2dd5
+
+Job status can be viewed on the web at:
+
+ http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
+
+          STDOUT
+        end
+      end
+
+      it "runs a spark job using cluster tags" do
+        with_git_initialized_project do |p|
+          # stub api requests
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          job_url = "http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
+          script_arguments = "arg1 arg2 arg3"
+
+          small_cluster_id = '510beb6b3004860820ab6538'
+          small_cluster_size = 2
+          small_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
+          large_cluster_id = '510bf0db3004860820ab6590'
+          large_cluster_size = 5
+          large_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
+          starting_cluster_id = '510bf0db3004860820abaaaa'
+          starting_cluster_size = 10
+          starting_cluster_status = Mortar::API::Clusters::STATUS_STARTING
+          huge_busy_cluster_id = '510bf0db3004860820ab6621'
+          huge_busy_cluster_size = 20
+          huge_busy_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
+
+
+          mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_SPARK_JOBSERVER) {
+            Excon::Response.new(:body => {
+              'clusters' => [
+                { 'cluster_id' => small_cluster_id, 'size' => small_cluster_size, 'running_jobs' => [], 'status_code' => small_cluster_status, 'tags' => ['small'] },
+                { 'cluster_id' => large_cluster_id, 'size' => large_cluster_size, 'running_jobs' => [], 'status_code' => large_cluster_status, 'tags' => ['large', 'cool'] },
+                { 'cluster_id' => starting_cluster_id, 'size' => starting_cluster_size, 'running_jobs' => [], 'status_code' => starting_cluster_status, 'tags' => ['starting'] },
+                { 'cluster_id' => huge_busy_cluster_id, 'size' => huge_busy_cluster_size, 'tags' => ['huge'],
+                  'running_jobs' => [ { 'job_id' => 'c571a8c7f76a4fd4a67c103d753e2dd5',
+                    'job_name' => "", 'start_timestamp' => ""} ], 'status_code' => huge_busy_cluster_status  }
+              ]})
+          }
+
+          mock(Mortar::Auth.api).post_spark_job_on_jobserver("myproject", "com.mortar.Job", is_a(String), large_cluster_id,
+            :script_arguments => script_arguments
+          ) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+
+          write_file(File.join(p.sparkjobs_path, "com/mortar/Job.scala"))
+          stderr, stdout = execute("spark com.mortar.Job #{script_arguments} --clustertags large,cool", p, @git)
+          stdout.should == <<-STDOUT
+Taking code snapshot... done
+Sending code snapshot to Mortar... done
+Running job on the cluster with tags [large,cool], id = 510bf0db3004860820ab6590, size = 5
+Requesting job execution... done
+job_id: c571a8c7f76a4fd4a67c103d753e2dd5
+
+Job status can be viewed on the web at:
+
+ http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
+
+          STDOUT
+        end
+      end
+
+    end
+  end
+end

metadata

@@ -1,195 +1,195 @@
 --- !ruby/object:Gem::Specification
 name: mortar
 version: !ruby/object:Gem::Version
-  version: 0.15.38
+  version: 0.15.39
 platform: ruby
 authors:
 - Mortar Data
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-16 00:00:00.000000000 Z
+date: 2016-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 4.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 4.0.0
 - !ruby/object:Gem::Dependency
   name: mortar-api-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.8.15
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.8.15
 - !ruby/object:Gem::Dependency
   name: netrc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.7'
 - !ruby/object:Gem::Dependency
   name: launchy
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '2.1'
 - !ruby/object:Gem::Dependency
   name: parseconfig
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.0.2
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.5.0
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.28'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.28'
 - !ruby/object:Gem::Dependency
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.4.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.4.2
 - !ruby/object:Gem::Dependency
   name: gem-release
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 10.1.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 10.1.1
 - !ruby/object:Gem::Dependency
   name: rr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '2.0'
 description: Client library and command-line tool to interact with the Mortar service.
@@ -322,6 +322,7 @@ files:
 - spec/mortar/command/pigscripts_spec.rb
 - spec/mortar/command/projects_spec.rb
 - spec/mortar/command/s3_spec.rb
+- spec/mortar/command/spark_spec.rb
 - spec/mortar/command/validate_spec.rb
 - spec/mortar/command/version_spec.rb
 - spec/mortar/command_spec.rb
@@ -351,19 +352,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: 1.8.7
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: Client library and CLI to interact with the Mortar service.
 test_files: []
-has_rdoc: