mortar 0.15.32 → 0.15.33

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    M2EzNDc4OGVmZTc1MGE2MWI5MWQ2MjBmZTBmZjBiNzIwMWRiNGQ4Nw==
+  data.tar.gz: !binary |-
+    MWY2Yzk3MTkxNzk2MmM3MTg2NjZhODA4YTZmNWQ1Y2QwMTRkNWQyMQ==
+SHA512:
+  metadata.gz: !binary |-
+    Njg5ZTc0NTQxOTg0NTg2ZWJiZTJjNTg3Njg3OGI2NzU1ZDNjYzdiNTJjNjBi
+    NzU2MTlhOTJmZWUyN2IyMDE3MzI2YjY5ZWJlNzdiZTU5ZDM4NmExNWNhYmVm
+    MGZlODNhYzY4ZTNhYjFiOGRmOTUwMTAzM2NmOGYzNzdkMjQwMzU=
+  data.tar.gz: !binary |-
+    OTU4YmJiYWUxODhkYjliNDMwNzllNGU3Y2ExNGZjMTRjMzRkNDQ5MjAzNjkw
+    ZWM2NzdiNjU4ZWRiZWE5OTJkMDRmYjM5MjkxNjczYjQwZWIwMWQ4NWU4NDdl
+    ZTdhMGUwYTIzZDk5YzExNWU0MzExNmMzYTNlOGRiOTYzMmJmNzc=

data/lib/mortar/command/base.rb

@@ -103,6 +103,10 @@ class Mortar::Command::Base
     end
     param_list
   end
+
+  def spark_script_arguments
+    invalid_arguments.join(" ")
+  end
   
   def luigi_parameters
     parameters = []
@@ -447,6 +451,18 @@ protected
     pigscript or controlscript
   end
 
+  def validate_sparkscript!(sparkscript_name)
+    shortened_script_name = File.basename(sparkscript_name)
+
+    unless sparkscript = project.sparkscripts[shortened_script_name]
+      available_scripts = project.sparkscripts.none? ? "No sparkscripts found" : "Available sparkscripts:\n#{project.sparkscripts.collect{|k,v| v.executable_path}.sort.join("\n")}"
+      error("Unable to find sparkscript #{sparkscript_name}\n#{available_scripts}")
+    end
+    #While validating we can load the defaults that are relevant to this script.
+    load_defaults(sparkscript_name)
+    sparkscript
+  end    
+
   def validate_luigiscript!(luigiscript_name)
     shortened_script_name = File.basename(luigiscript_name, ".*")
     unless luigiscript = project.luigiscripts[shortened_script_name]

data/lib/mortar/command/clusters.rb

@@ -27,11 +27,11 @@ class Mortar::Command::Clusters < Mortar::Command::Base
   def index
     validate_arguments!
     
-    clusters = api.get_clusters().body['clusters']
+    clusters = api.get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__ALL).body['clusters']
     if not clusters.empty?
       display_table(clusters,
-      %w( cluster_id size status_description cluster_type_description start_timestamp duration),
-      ['cluster_id', 'Size (# of Nodes)', 'Status', 'Type', 'Start Timestamp', 'Elapsed Time'])
+      %w( cluster_id size status_description cluster_type_description start_timestamp duration cluster_backend_description),
+      ['cluster_id', 'Size (# of Nodes)', 'Status', 'Type', 'Start Timestamp', 'Elapsed Time', 'Cluster Backend'])
     else
       display("There are no running or recent clusters")
     end

data/lib/mortar/command/jobs.rb

@@ -114,7 +114,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
     end
     
     unless options[:clusterid] || options[:clustersize]
-      clusters = api.get_clusters().body['clusters']
+      clusters = api.get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1).body['clusters']
 
       largest_free_cluster = clusters.select{ |c| \
         c['running_jobs'].length == 0 && c['status_code'] == Mortar::API::Clusters::STATUS_RUNNING }.

data/lib/mortar/command/spark.rb

@@ -0,0 +1,132 @@
+#
+# Copyright 2015 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mortar/command/base"
+require "time"
+
+# run spark jobs
+#
+class Mortar::Command::Spark < Mortar::Command::Base
+
+  include Mortar::Git
+
+  # spark SCRIPT
+  #
+  # Run a spark job.
+  #
+  # -c, --clusterid CLUSTERID   # Run job on an existing cluster with ID of CLUSTERID (optional)
+  # -s, --clustersize NUMNODES  # Run job on a new cluster, with NUMNODES nodes (optional; must be >= 2 if provided)
+  # -1, --singlejobcluster      # Stop the cluster after job completes.  (Default: false--cluster can be used for other jobs, and will shut down after 1 hour of inactivity)
+  # -2, --permanentcluster      # Don't automatically stop the cluster after it has been idle for an hour (Default: false--cluster will be shut down after 1 hour of inactivity)
+  # -3, --spot                  # Use spot instances for this cluster (Default: false, only applicable to new clusters)
+  # -P, --project PROJECTNAME   # Use a project that is not checked out in the current directory.  Runs code from project's master branch in GitHub rather than snapshotting local code.
+  # -B, --branch BRANCHNAME     # Used with --project to specify a non-master branch
+  #
+  # Examples:
+  #
+  #    Run the classify_text sparkscript:
+  #        $ mortar spark sparkscripts/classify_text.py
+  #
+  #    Run the classify_text sparkscript with 3 script arguments (input location, output location, tuning parameter):
+  #        $ mortar spark sparkscripts/classify_text.py s3://your-bucket/input s3://your-bucket/output 100
+  #
+  def index
+    script_name = shift_argument
+    unless script_name
+      error("Usage: mortar spark SCRIPT\nMust specify SCRIPT.")
+    end
+    
+    if options[:project]
+      project_name = options[:project]
+    else
+      project_name = project.name
+      script = validate_sparkscript!(script_name)
+      script_name = script.name
+    end
+
+    script_arguments = spark_script_arguments()
+
+    if options[:clusterid]
+      [:clustersize, :singlejobcluster, :permanentcluster].each do |opt|
+        unless options[opt].nil?
+          error("Option #{opt.to_s} cannot be set when running a job on an existing cluster (with --clusterid option)")
+        end
+      end
+    end
+
+    if options[:project]
+      if options[:branch]
+        git_ref = options[:branch]
+      else
+        git_ref = "master"
+      end
+    else
+      git_ref = sync_code_with_cloud()
+    end
+
+    unless options[:clusterid] || options[:clustersize]
+      clusters = api.get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_2).body['clusters']
+
+      largest_free_cluster = clusters.select{ |c| \
+        c['running_jobs'].length == 0 && c['status_code'] == Mortar::API::Clusters::STATUS_RUNNING }.
+        max_by{|c| c['size']}
+
+      if largest_free_cluster.nil?
+        options[:clustersize] = 2
+        display("Defaulting to running job on new cluster of size 2")
+      else
+        options[:clusterid] = largest_free_cluster['cluster_id']
+        display("Defaulting to running job on largest existing free cluster, id = " + 
+                largest_free_cluster['cluster_id'] + ", size = " + largest_free_cluster['size'].to_s)
+      end
+    end
+
+    response = action("Requesting job execution") do
+      if options[:clustersize]
+        if options[:singlejobcluster] && options[:permanentcluster]
+          error("Cannot declare cluster as both --singlejobcluster and --permanentcluster")
+        end
+        cluster_size = options[:clustersize].to_i
+        cluster_type = Mortar::API::Jobs::CLUSTER_TYPE__PERSISTENT
+        if options[:singlejobcluster]
+          cluster_type = Mortar::API::Jobs::CLUSTER_TYPE__SINGLE_JOB
+        elsif options[:permanentcluster]
+          cluster_type = Mortar::API::Jobs::CLUSTER_TYPE__PERMANENT
+        end
+        use_spot_instances = options[:spot] || false
+        api.post_spark_job_new_cluster(project_name, script_name, git_ref, cluster_size, 
+          :project_script_path => script.rel_path,
+          :script_arguments => script_arguments,
+          :cluster_type => cluster_type,
+          :use_spot_instances => use_spot_instances).body
+      else
+        cluster_id = options[:clusterid]
+        api.post_spark_job_existing_cluster(project_name, script_name, git_ref, cluster_id,
+          :project_script_path => script.rel_path,
+          :script_arguments => script_arguments).body
+      end
+    end
+    
+    display("job_id: #{response['job_id']}")
+    display
+    display("Job status can be viewed on the web at:\n\n #{response['web_job_url']}")
+    display
+
+    response['job_id']
+  end
+  
+  alias_command "spark:run", "spark"
+end

data/lib/mortar/generators/project_generator.rb

@@ -76,6 +76,13 @@ module Mortar
             generate_file "client.cfg.template", "client.cfg.template"
           end
 
+          mkdir "sparkscripts"
+
+          inside "sparkscripts" do
+            copy_file "README", "README"
+            generate_file "sparkscript.py", "#{project_name}_pi.py"
+          end
+
           mkdir "lib"
           inside "lib" do
             copy_file "README", "README"

data/lib/mortar/git.rb

@@ -202,6 +202,7 @@ module Mortar
       #
       def ensure_valid_mortar_project_manifest()
         if File.exists? project_manifest_name
+          ensure_sparkscripts_in_project_manifest()
           ensure_luigiscripts_in_project_manifest()
           ensure_gitignore_in_project_manifest()
           add_newline_to_file(project_manifest_name)
@@ -210,6 +211,18 @@ module Mortar
         end
       end
 
+      #
+      # Ensure that the sparkscripts directory,
+      # which was added after some project manifests were
+      # created, is in the manifest (if sparkscripts exists).
+      #
+      def ensure_sparkscripts_in_project_manifest
+        sparkscripts_path = "sparkscripts"
+        if File.directory? sparkscripts_path
+          add_entry_to_mortar_project_manifest(project_manifest_name, sparkscripts_path)
+        end
+      end
+
       #
       # Ensure that the luigiscripts directory,
       # which was added after some project manifests were
@@ -250,6 +263,10 @@ module Mortar
             manifest.puts "luigiscripts"
           end
 
+          if File.directory? "#{path}/sparkscripts"
+            manifest.puts "sparkscripts"
+          end
+
           if File.exists? "#{path}/.gitignore"
             manifest.puts ".gitignore"
           end

data/lib/mortar/project.rb

@@ -85,6 +85,19 @@ module Mortar
         @luigiscripts
       end
 
+      def sparkscripts_path
+        File.join(@root_path, "sparkscripts")
+      end
+
+      def sparkscripts
+        @sparkscripts ||= SparkScripts.new(
+          sparkscripts_path,
+          "sparkscripts",
+          "",
+          :optional => true)
+        @sparkscripts
+      end
+
       def tmp_path
         path = File.join(@root_path, "tmp")
         unless File.directory? path
@@ -138,7 +151,6 @@ module Mortar
         if File.directory? @path
           # get {script_name => full_path}
           file_paths = Dir[File.join(@path, "**", "*#{@filename_extension}")]
-
           scripts = file_paths.collect{|element_path| [element_name(element_path), element(element_name(element_path), element_path)]}
 
           #Check for duplicates.
@@ -179,6 +191,12 @@ module Mortar
       end
     end
 
+    class SparkScripts < ProjectEntity
+      def element(name, path)
+        SparkScript.new(name, path)
+      end
+    end
+
     class PythonUDFs < ProjectEntity
       def element(name, path)
         Script.new(name, path)
@@ -219,6 +237,14 @@ module Mortar
       end
       
     end
+
+    class SparkScript < Script
+      
+      def executable_path
+        "sparkscripts/#{self.name}"
+      end
+      
+    end
     
     class ControlScript < Script
       

data/lib/mortar/templates/project/project.manifest

@@ -8,6 +8,7 @@
 
 lib
 luigiscripts
+sparkscripts
 macros
 pigscripts
 udfs

data/lib/mortar/templates/project/sparkscripts/README

@@ -0,0 +1 @@
+The sparkscripts directory is where your spark scripts should be stored.

data/lib/mortar/templates/project/sparkscripts/sparkscript.py

@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+This is an example Spark script to get you started. This script will run
+a simple Spark script that will calculate the value of pi.
+
+To Run:
+mortar spark sparkscripts/<%= project_name %>_pi.py [partitions]"
+"""
+
+import sys
+from random import random
+from operator import add
+
+from pyspark import SparkContext
+
+
+if __name__ == "__main__":
+    sc = SparkContext(appName="PythonPi")
+    partitions = int(sys.argv[1]) if len(sys.argv) > 1 else 2
+    n = 100000 * partitions
+
+    def f(_):
+        x = random() * 2 - 1
+        y = random() * 2 - 1
+        return 1 if x ** 2 + y ** 2 < 1 else 0
+
+    count = sc.parallelize(xrange(1, n + 1), partitions).map(f).reduce(add)
+    print "Pi is roughly %f" % (4.0 * count / n)
+
+    sc.stop()

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.15.32"
+  VERSION = "0.15.33"
 end

data/spec/mortar/command/clusters_spec.rb

@@ -32,24 +32,26 @@ module Mortar::Command
                      "size" => 2,
                      "status_description" => "Running",
                      "start_timestamp" => "2012-08-27T21:27:15.669000+00:00",
-                     "duration" => "2 mins"},
+                     "duration" => "2 mins",
+                     "cluster_backend_description" => "Hadoop 1 (Pig)" },
                     {"cluster_id" => "50fbe5a23004292547fc2225", 
-                                 "size" => 10,
-                                 "status_description" => "Shut Down",
-                                 "start_timestamp" => "2011-08-27T21:27:15.669000+00:00",
-                                 "duration" => "20 mins"}]
-         mock(Mortar::Auth.api).get_clusters().returns(Excon::Response.new(:body => {"clusters" => clusters}))
+                     "size" => 10,
+                     "status_description" => "Shut Down",
+                     "start_timestamp" => "2011-08-27T21:27:15.669000+00:00",
+                     "duration" => "20 mins",
+                     "cluster_backend_description" => "Hadoop 2 (Spark)" }]
+         mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__ALL).returns(Excon::Response.new(:body => {"clusters" => clusters}))
          stderr, stdout = execute("clusters", nil, nil)
          stdout.should == <<-STDOUT
-cluster_id                Size (# of Nodes)  Status     Type  Start Timestamp                   Elapsed Time
-------------------------  -----------------  ---------  ----  --------------------------------  ------------
-50fbe5a23004292547fc2224                  2  Running          2012-08-27T21:27:15.669000+00:00  2 mins
-50fbe5a23004292547fc2225                 10  Shut Down        2011-08-27T21:27:15.669000+00:00  20 mins
+cluster_id                Size (# of Nodes)  Status     Type  Start Timestamp                   Elapsed Time  Cluster Backend
+------------------------  -----------------  ---------  ----  --------------------------------  ------------  ----------------
+50fbe5a23004292547fc2224                  2  Running          2012-08-27T21:27:15.669000+00:00  2 mins        Hadoop 1 (Pig)
+50fbe5a23004292547fc2225                 10  Shut Down        2011-08-27T21:27:15.669000+00:00  20 mins       Hadoop 2 (Spark)
 STDOUT
       end
 
       it "handles no clusters running" do
-        mock(Mortar::Auth.api).get_clusters().returns(Excon::Response.new(:body => {"clusters" => []}))
+        mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__ALL).returns(Excon::Response.new(:body => {"clusters" => []}))
          stderr, stdout = execute("clusters", nil, nil)
          stdout.should == <<-STDOUT
 There are no running or recent clusters

data/spec/mortar/command/generate_spec.rb

@@ -40,6 +40,7 @@ describe Mortar::Command::Generate do
       File.exists?("Test/pigscripts/Test.pig").should be_true
       File.exists?("Test/udfs/python/Test.py").should be_true
       File.exists?("Test/luigiscripts/README").should be_true
+      File.exists?("Test/sparkscripts/README").should be_true
 
       File.read("Test/pigscripts/Test.pig").each_line { |line| line.match(/<%.*%>/).should be_nil }
     end

data/spec/mortar/command/jobs_spec.rb

@@ -281,7 +281,7 @@ STDOUT
           job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_size = 2
 
-          mock(Mortar::Auth.api).get_clusters() {Excon::Response.new(:body => {'clusters' => []})}
+          mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1) {Excon::Response.new(:body => {'clusters' => []})}
           mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :pig_version => "0.9", 
             :project_script_path => be_a_kind_of(String),
@@ -319,7 +319,7 @@ STDOUT
           job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_size = 2
 
-          mock(Mortar::Auth.api).get_clusters() {Excon::Response.new(:body => {'clusters' => []})}
+          mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1) {Excon::Response.new(:body => {'clusters' => []})}
           mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :pig_version => "0.9", 
             :project_script_path => be_a_kind_of(String),
@@ -399,7 +399,7 @@ STDOUT
           huge_busy_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
           
 
-          mock(Mortar::Auth.api).get_clusters() {
+          mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1) {
             Excon::Response.new(:body => { 
               'clusters' => [
                   { 'cluster_id' => small_cluster_id, 'size' => small_cluster_size, 'running_jobs' => [], 'status_code' => small_cluster_status }, 

data/spec/mortar/command/projects_spec.rb

@@ -154,11 +154,15 @@ STDOUT
         File.exists?("luigiscripts/README").should be_true
         File.exists?("luigiscripts/some_new_project_luigi.py").should be_true
         File.exists?("luigiscripts/client.cfg.template").should be_true
+        File.exists?("sparkscripts/README").should be_true
+        File.exists?("sparkscripts/some_new_project_pi.py").should be_true
         File.exists?("lib/README").should be_true
         File.exists?("params/README").should be_true
 
         File.read("pigscripts/some_new_project.pig").each_line { |line| line.match(/<%.*%>/).should be_nil }
         File.read("luigiscripts/some_new_project_luigi.py").each_line { |line| line.match(/<%.*%>/).should be_nil }
+        File.read("sparkscripts/some_new_project_pi.py").each_line { |line| line.match(/<%.*%>/).should be_nil }
+
 
         stdout.should == <<-STDOUT
 \r\e[0KVerifying GitHub username: /\r\e[0KVerifying GitHub username: -\r\e[0KVerifying GitHub username:  Done!
@@ -185,6 +189,9 @@ Sending request to register project: some_new_project... done
 \e[1;32m      create\e[0m  luigiscripts/README
 \e[1;32m      create\e[0m  luigiscripts/some_new_project_luigi.py
 \e[1;32m      create\e[0m  luigiscripts/client.cfg.template
+\e[1;32m      create\e[0m  sparkscripts
+\e[1;32m      create\e[0m  sparkscripts/README
+\e[1;32m      create\e[0m  sparkscripts/some_new_project_pi.py
 \e[1;32m      create\e[0m  lib
 \e[1;32m      create\e[0m  lib/README
 \e[1;32m      create\e[0m  params

data/spec/mortar/command/spark_spec.rb

@@ -0,0 +1,154 @@
+#
+# Copyright 2014 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'spec_helper'
+require 'fakefs/spec_helpers'
+require 'mortar/command/spark'
+require 'mortar/api/jobs'
+
+module Mortar::Command
+  describe Spark do
+    
+    before(:each) do
+      stub_core      
+      @git = Mortar::Git::Git.new
+    end
+    
+    context("index") do
+      it "shows help when user adds help argument" do
+        with_git_initialized_project do |p|
+          stderr_dash_h, stdout_dash_h = execute("spark -h", p, @git) 
+          stderr_help, stdout_help = execute("spark help", p, @git)
+          stdout_dash_h.should == stdout_help
+          stderr_dash_h.should == stderr_help
+        end
+      end
+
+      it "runs a spark job with no arguments new cluster" do
+        with_git_initialized_project do |p|
+          # stub api requests
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          job_url = "http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
+          mock(Mortar::Auth.api).post_spark_job_new_cluster("myproject", "my_script.py", is_a(String), 4,
+            :project_script_path => be_a_kind_of(String),
+            :script_arguments => "",
+            :cluster_type=>"persistent",
+            :use_spot_instances=>false
+          ) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+
+          write_file(File.join(p.sparkscripts_path, "my_script.py"))
+          stderr, stdout = execute("spark sparkscripts/my_script.py --clustersize 4", p, @git)
+          puts stderr
+          stdout.should == <<-STDOUT
+Taking code snapshot... done
+Sending code snapshot to Mortar... done
+Requesting job execution... done
+job_id: c571a8c7f76a4fd4a67c103d753e2dd5
+
+Job status can be viewed on the web at:
+
+ http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
+
+STDOUT
+        end
+      end
+
+      it "runs a spark job with script_arguments existing cluster" do
+        with_git_initialized_project do |p|
+          # stub api requests
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          cluster_id = "c571a8c7f76a4fd4a67c103d753e2dd7"
+          job_url = "http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
+          script_arguments = "arg1 arg2 arg3"
+          mock(Mortar::Auth.api).post_spark_job_existing_cluster("myproject", "my_script.py", is_a(String), cluster_id,
+            :project_script_path => be_a_kind_of(String),
+            :script_arguments => script_arguments
+          ) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+
+          write_file(File.join(p.sparkscripts_path, "my_script.py"))
+          stderr, stdout = execute("spark sparkscripts/my_script.py --clusterid #{cluster_id} #{script_arguments}", p, @git)
+          stdout.should == <<-STDOUT
+Taking code snapshot... done
+Sending code snapshot to Mortar... done
+Requesting job execution... done
+job_id: c571a8c7f76a4fd4a67c103d753e2dd5
+
+Job status can be viewed on the web at:
+
+ http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
+
+STDOUT
+        end
+      end
+
+      it "runs a spark job with on free cluster" do
+        with_git_initialized_project do |p|
+          # stub api requests
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          job_url = "http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
+          script_arguments = "arg1 arg2 arg3"
+
+          small_cluster_id = '510beb6b3004860820ab6538'
+          small_cluster_size = 2
+          small_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
+          large_cluster_id = '510bf0db3004860820ab6590'
+          large_cluster_size = 5
+          large_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
+          starting_cluster_id = '510bf0db3004860820abaaaa'
+          starting_cluster_size = 10
+          starting_cluster_status = Mortar::API::Clusters::STATUS_STARTING
+          huge_busy_cluster_id = '510bf0db3004860820ab6621'
+          huge_busy_cluster_size = 20
+          huge_busy_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
+          
+
+          mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_2) {
+            Excon::Response.new(:body => { 
+              'clusters' => [
+                  { 'cluster_id' => small_cluster_id, 'size' => small_cluster_size, 'running_jobs' => [], 'status_code' => small_cluster_status }, 
+                  { 'cluster_id' => large_cluster_id, 'size' => large_cluster_size, 'running_jobs' => [], 'status_code' => large_cluster_status },
+                  { 'cluster_id' => starting_cluster_id, 'size' => starting_cluster_size, 'running_jobs' => [], 'status_code' => starting_cluster_status },
+                  { 'cluster_id' => huge_busy_cluster_id, 'size' => huge_busy_cluster_size, 
+                    'running_jobs' => [ { 'job_id' => 'c571a8c7f76a4fd4a67c103d753e2dd5',
+                       'job_name' => "", 'start_timestamp' => ""} ], 'status_code' => huge_busy_cluster_status  }
+              ]})
+          }
+
+          mock(Mortar::Auth.api).post_spark_job_existing_cluster("myproject", "my_script.py", is_a(String), large_cluster_id,
+            :project_script_path => be_a_kind_of(String),
+            :script_arguments => script_arguments
+          ) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+
+          write_file(File.join(p.sparkscripts_path, "my_script.py"))
+          stderr, stdout = execute("spark sparkscripts/my_script.py #{script_arguments}", p, @git)
+          stdout.should == <<-STDOUT
+Taking code snapshot... done
+Sending code snapshot to Mortar... done
+Defaulting to running job on largest existing free cluster, id = 510bf0db3004860820ab6590, size = 5
+Requesting job execution... done
+job_id: c571a8c7f76a4fd4a67c103d753e2dd5
+
+Job status can be viewed on the web at:
+
+ http://127.0.0.1:5000/jobs/spark_job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
+
+STDOUT
+        end
+      end
+
+    end
+  end
+end

data/spec/mortar/git_spec.rb

@@ -154,6 +154,7 @@ macros
 pigscripts
 udfs
 luigiscripts
+sparkscripts
 MANIFEST0
           manifest_path = File.join(p.root_path, "project.manifest")
           write_file(manifest_path, manifest_without_gitignore)
@@ -170,6 +171,7 @@ macros
 pigscripts
 udfs
 luigiscripts
+sparkscripts
 .gitignore
 MANIFEST0AFTER
         end
@@ -190,6 +192,7 @@ macros
 pigscripts
 udfs
 luigiscripts
+sparkscripts
 MANIFEST1
           manifest_path = File.join(p.root_path, "project.manifest")
           write_file(manifest_path, manifest_without_gitignore)
@@ -206,6 +209,7 @@ macros
 pigscripts
 udfs
 luigiscripts
+sparkscripts
 MANIFEST1AFTER
         end
       end
@@ -277,6 +281,73 @@ MANIFEST1AFTER
         end
       end
 
+      it "adds sparkscripts if the directory exists and manifest does not have it" do
+        with_git_initialized_project do |p|
+          # ensure luigiscripts path exists
+          sparkscripts_path = File.join(p.root_path, "sparkscripts")
+          unless File.directory? sparkscripts_path
+            FileUtils.mkdir_p(sparkscripts_path)
+          end
+
+          # remove it from manifest
+          manifest_without_sparkscripts = <<-MANIFEST0
+lib
+macros
+pigscripts
+udfs
+MANIFEST0
+          manifest_path = File.join(p.root_path, "project.manifest")
+          write_file(manifest_path, manifest_without_sparkscripts)
+
+          project_manifest_before = File.open(manifest_path, "r").read
+          project_manifest_before.include?("sparkscripts").should be_false
+
+          @git.ensure_sparkscripts_in_project_manifest()
+
+          project_manifest_after = File.open(manifest_path, "r").read
+          project_manifest_after.should == <<-MANIFEST0AFTER
+lib
+macros
+pigscripts
+udfs
+sparkscripts
+MANIFEST0AFTER
+        end
+      end
+
+      it "does not add sparkscripts if the directory does not exist" do
+        with_git_initialized_project do |p|
+          # ensure sparkscripts path does not exist
+          sparkscripts_path = File.join(p.root_path, "sparkscripts")
+          if File.directory? sparkscripts_path
+            FileUtils.rm_rf(sparkscripts_path)
+          end
+
+          # remove it from manifest
+          manifest_without_sparkscripts = <<-MANIFEST1
+lib
+macros
+pigscripts
+udfs
+MANIFEST1
+          manifest_path = File.join(p.root_path, "project.manifest")
+          write_file(manifest_path, manifest_without_sparkscripts)
+
+          project_manifest_before = File.open(manifest_path, "r").read
+          project_manifest_before.include?("sparkscripts").should be_false
+
+          @git.ensure_sparkscripts_in_project_manifest()
+
+          project_manifest_after = File.open(manifest_path, "r").read
+          project_manifest_after.should == <<-MANIFEST1AFTER
+lib
+macros
+pigscripts
+udfs
+MANIFEST1AFTER
+        end
+      end
+
       it "handles manifest with no trailing newline" do
         with_git_initialized_project do |p|
           # ensure luigiscripts path exists

data/spec/spec_helper.rb

@@ -180,6 +180,7 @@ def with_blank_project_with_name(name, &block)
   FileUtils.mkdir_p(project_path)
   
   # setup project subdirectories
+  FileUtils.mkdir_p(File.join(project_path, "sparkscripts"))
   FileUtils.mkdir_p(File.join(project_path, "luigiscripts"))
   FileUtils.mkdir_p(File.join(project_path, "pigscripts"))
   FileUtils.mkdir_p(File.join(project_path, "macros"))

metadata

@@ -1,230 +1,204 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: mortar
-version: !ruby/object:Gem::Version 
-  hash: 99
-  prerelease: 
-  segments: 
-  - 0
-  - 15
-  - 32
-  version: 0.15.32
+version: !ruby/object:Gem::Version
+  version: 0.15.33
 platform: ruby
-authors: 
+authors:
 - Mortar Data
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2014-11-11 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2015-01-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rdoc
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 63
-        segments: 
-        - 4
-        - 0
-        - 0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 4.0.0
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: mortar-api-ruby
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: mortar-api-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 41
-        segments: 
-        - 0
-        - 8
-        - 11
-        version: 0.8.11
+      - !ruby/object:Gem::Version
+        version: 0.8.12
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: netrc
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 5
-        segments: 
-        - 0
-        - 7
-        version: "0.7"
+      - !ruby/object:Gem::Version
+        version: 0.8.12
+- !ruby/object:Gem::Dependency
+  name: netrc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
   type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: launchy
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 1
-        segments: 
-        - 2
-        - 1
-        version: "2.1"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: launchy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
   type: :runtime
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: parseconfig
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 19
-        segments: 
-        - 1
-        - 0
-        - 2
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: parseconfig
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
         version: 1.0.2
   type: :runtime
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: aws-sdk
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 15
-        segments: 
-        - 1
-        - 0
-        version: "1.0"
+      - !ruby/object:Gem::Version
+        version: '1.0'
   type: :runtime
-  version_requirements: *id006
-- !ruby/object:Gem::Dependency 
-  name: nokogiri
   prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 1
-        - 5
-        - 0
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
         version: 1.5.0
   type: :runtime
-  version_requirements: *id007
-- !ruby/object:Gem::Dependency 
-  name: excon
   prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 51
-        segments: 
-        - 0
-        - 28
-        version: "0.28"
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: excon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.28'
   type: :development
-  version_requirements: *id008
-- !ruby/object:Gem::Dependency 
-  name: fakefs
   prerelease: false
-  requirement: &id009 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.28'
+- !ruby/object:Gem::Dependency
+  name: fakefs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 0
-        - 4
-        - 2
+      - !ruby/object:Gem::Version
         version: 0.4.2
   type: :development
-  version_requirements: *id009
-- !ruby/object:Gem::Dependency 
-  name: gem-release
   prerelease: false
-  requirement: &id010 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+- !ruby/object:Gem::Dependency
+  name: gem-release
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id010
-- !ruby/object:Gem::Dependency 
-  name: rake
   prerelease: false
-  requirement: &id011 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 73
-        segments: 
-        - 10
-        - 1
-        - 1
+      - !ruby/object:Gem::Version
         version: 10.1.1
   type: :development
-  version_requirements: *id011
-- !ruby/object:Gem::Dependency 
-  name: rr
   prerelease: false
-  requirement: &id012 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 10.1.1
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id012
-- !ruby/object:Gem::Dependency 
-  name: rspec
   prerelease: false
-  requirement: &id013 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 2
-        - 0
-        version: "2.0"
+      - !ruby/object:Gem::Version
+        version: '2.0'
   type: :development
-  version_requirements: *id013
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
 description: Client library and command-line tool to interact with the Mortar service.
 email: support@mortardata.com
-executables: 
+executables:
 - mortar
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - README.md
 - bin/mortar
 - css/illustrate.css
@@ -254,6 +228,7 @@ files:
 - lib/mortar/command/plugins.rb
 - lib/mortar/command/projects.rb
 - lib/mortar/command/s3.rb
+- lib/mortar/command/spark.rb
 - lib/mortar/command/validate.rb
 - lib/mortar/command/version.rb
 - lib/mortar/conf/luigi/logging.ini
@@ -314,6 +289,8 @@ files:
 - lib/mortar/templates/project/project.manifest
 - lib/mortar/templates/project/project.properties
 - lib/mortar/templates/project/requirements.txt
+- lib/mortar/templates/project/sparkscripts/README
+- lib/mortar/templates/project/sparkscripts/sparkscript.py
 - lib/mortar/templates/project/udfs/java/gitkeep
 - lib/mortar/templates/project/udfs/jython/gitkeep
 - lib/mortar/templates/project/udfs/python/python_udf.py
@@ -347,6 +324,7 @@ files:
 - spec/mortar/command/pigscripts_spec.rb
 - spec/mortar/command/projects_spec.rb
 - spec/mortar/command/s3_spec.rb
+- spec/mortar/command/spark_spec.rb
 - spec/mortar/command/validate_spec.rb
 - spec/mortar/command/version_spec.rb
 - spec/mortar/command_spec.rb
@@ -370,38 +348,25 @@ files:
 - spec/support/display_message_matcher.rb
 homepage: http://mortardata.com/
 licenses: []
-
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 57
-      segments: 
-      - 1
-      - 8
-      - 7
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
       version: 1.8.7
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Client library and CLI to interact with the Mortar service.
 test_files: []
-