mortar 0.7.2 → 0.7.3

data/lib/mortar/command/jobs.rb

@@ -23,6 +23,10 @@ class Mortar::Command::Jobs < Mortar::Command::Base
 
   include Mortar::Git
 
+  CLUSTER_TYPE__SINGLE_JOB = 'single_job'
+  CLUSTER_TYPE__PERSISTENT = 'persistent'
+  CLUSTER_TYPE__PERMANENT = 'permanent'
+
   # jobs
   #
   # Show recent and running jobs.
@@ -57,6 +61,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
   # -c, --clusterid CLUSTERID   # Run job on an existing cluster with ID of CLUSTERID (optional)
   # -s, --clustersize NUMNODES  # Run job on a new cluster, with NUMNODES nodes (optional; must be >= 2 if provided)
   # -1, --singlejobcluster      # Stop the cluster after job completes.  (Default: false—-cluster can be used for other jobs, and will shut down after 1 hour of inactivity)
+  # -2, --permanentcluster      # Don't automatically stop the cluster after it has been idle for an hour (Default: false--cluster will be shut down after 1 hour of inactivity)
   # -p, --parameter NAME=VALUE  # Set a pig parameter value in your script.
   # -f, --param-file PARAMFILE  # Load pig parameter values from a file.
   # -d, --donotnotify           # Don't send an email on job completion.  (Default: false--an email will be sent to you once the job completes)
@@ -102,7 +107,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
     end
       
     if options[:clusterid]
-      [:clustersize, :singlejobcluster].each do |opt|
+      [:clustersize, :singlejobcluster, :permanentcluster].each do |opt|
         unless options[opt].nil?
           error("Option #{opt.to_s} cannot be set when running a job on an existing cluster (with --clusterid option)")
         end
@@ -116,11 +121,19 @@ class Mortar::Command::Jobs < Mortar::Command::Base
     # post job to API    
     response = action("Requesting job execution") do
       if options[:clustersize]
+        if options[:singlejobcluster] && options[:permanentcluster]
+          error("Cannot declare cluster as both --singlejobcluster and --permanentcluster")
+        end
         cluster_size = options[:clustersize].to_i
-        keepalive = ! options[:singlejobcluster]
+        cluster_type = CLUSTER_TYPE__PERSISTENT
+        if options[:singlejobcluster]
+          cluster_type = CLUSTER_TYPE__SINGLE_JOB
+        elsif options[:permanentcluster]
+          cluster_type = CLUSTER_TYPE__PERMANENT
+        end
         api.post_job_new_cluster(project.name, script.name, git_ref, cluster_size, 
           :parameters => pig_parameters,
-          :keepalive => keepalive,
+          :cluster_type => cluster_type,
           :notify_on_job_finish => notify_on_job_finish,
           :is_control_script => is_control_script).body
       else

data/lib/mortar/local/controller.rb

@@ -22,7 +22,7 @@ require "mortar/local/jython"
 
 
 class Mortar::Local::Controller
-  include Mortar::Helpers
+  include Mortar::Local::InstallUtil
 
   NO_JAVA_ERROR_MESSAGE = <<EOF
 A suitable java installation could not be found.  If you already have java installed
@@ -42,8 +42,8 @@ https://pypi.python.org/pypi/virtualenv
 EOF
 
   NO_AWS_KEYS_ERROR_MESSAGE = <<EOF
-Please specify your aws access key via enviroment variable AWS_ACCESS_KEY
-and your aws secret key via enviroment variable AWS_SECRET_KEY"
+Please specify your aws access key via environment variable AWS_ACCESS_KEY
+and your aws secret key via environment variable AWS_SECRET_KEY"
 EOF
 
 
@@ -91,6 +91,19 @@ EOF
 
     jy = Mortar::Local::Jython.new()
     jy.install_or_update()
+
+    ensure_local_install_dir_in_gitignore
+  end
+
+  def ensure_local_install_dir_in_gitignore()
+    if File.exists? local_project_gitignore
+      open(local_project_gitignore, 'r+') do |gitignore|
+        unless gitignore.read().include? local_install_directory_name
+          gitignore.seek(0, IO::SEEK_END)
+          gitignore.puts local_install_directory_name
+        end
+      end
+    end
   end
 
   # Main entry point for user running a pig script

data/lib/mortar/local/installutil.rb

@@ -28,10 +28,26 @@ module Mortar
 
       include Mortar::Helpers
 
-      def local_install_directory
+      def local_install_directory_name
+        ".mortar-local"
+      end
+
+      def project_root
         # note: assumes that CWD is the project root, is
         # this a safe assumption?
-        File.join(Dir.getwd, ".mortar-local")
+        Dir.getwd
+      end
+
+      def local_install_directory
+        File.join(project_root, local_install_directory_name)
+      end
+
+      def local_pig_logfile
+        project_root + "/local-pig.log"
+      end
+
+      def local_project_gitignore
+        project_root + "/.gitignore"
       end
 
       def jython_directory
@@ -42,6 +58,10 @@ module Mortar
         jython_directory + "/cachedir"
       end
 
+      def gitignore_template_path
+        File.expand_path("../../templates/project/gitignore", __FILE__)
+      end
+
       # Drops a marker file for an installed package, used
       # to help determine if updates should be performed
       def note_install(subdirectory)

data/lib/mortar/local/pig.rb

@@ -224,6 +224,7 @@ class Mortar::Local::Pig
   # get it to do something interesting, such as '-f some-file.pig'
   def run_pig_command(cmd, parameters = nil, jython_output = true)
     unset_hadoop_env_vars
+    delete_local_log_file
     # Generate the script for running the command, then
     # write it to a temp script which will be exectued
     script_text = script_for_command(cmd, parameters)
@@ -243,6 +244,12 @@ class Mortar::Local::Pig
     ENV['HADOOP_CONF_DIR'] = ''
   end
 
+  def delete_local_log_file
+    if File.exists? local_pig_logfile
+      FileUtils.rm local_pig_logfile
+    end
+  end
+
   # Generates a bash script which sets up the necessary environment and
   # then runs the pig command
   def script_for_command(cmd, parameters, jython_output = true)
@@ -278,6 +285,7 @@ class Mortar::Local::Pig
     opts['fs.s3n.awsAccessKeyId'] = ENV['AWS_ACCESS_KEY']
     opts['fs.s3n.awsSecretAccessKey'] = ENV['AWS_SECRET_KEY']
     opts['pig.events.logformat'] = PIG_LOG_FORMAT
+    opts['pig.logfile'] = local_pig_logfile
     opts['python.verbose'] = 'error'
     opts['jython.output'] = true
     opts['python.home'] = jython_directory

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.7.2"
+  VERSION = "0.7.3"
 end

data/spec/mortar/command/jobs_spec.rb

@@ -48,7 +48,7 @@ module Mortar::Command
 
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]), 
-            :keepalive => false,
+            :cluster_type => Jobs::CLUSTER_TYPE__SINGLE_JOB,
             :notify_on_job_finish => true,
             :is_control_script=> false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
 
@@ -71,7 +71,64 @@ Or by running:
 STDOUT
         end
       end
-      
+
+      it "handles permanentcluster parameter" do
+        with_git_initialized_project do |p|
+          # stub api requests
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
+          cluster_size = 5
+
+          mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
+                                                      :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
+                                                      :cluster_type => Jobs::CLUSTER_TYPE__PERMANENT,
+                                                      :notify_on_job_finish => true,
+                                                      :is_control_script=> false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+
+          write_file(File.join(p.pigscripts_path, "my_script.pig"))
+          stderr, stdout = execute("jobs:run my_script -2 --clustersize 5 -p FIRST_PARAM=FOO -p SECOND_PARAM=BAR", p, @git)
+          stdout.should == <<-STDOUT
+Taking code snapshot... done
+Sending code snapshot to Mortar... done
+Requesting job execution... done
+job_id: c571a8c7f76a4fd4a67c103d753e2dd5
+
+Job status can be viewed on the web at:
+
+ http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
+
+Or by running:
+
+  mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 --poll
+
+          STDOUT
+        end
+      end
+
+
+      it "throws error on singlejobcluster and permanentcluster" do
+        with_git_initialized_project do |p|
+
+          write_file(File.join(p.pigscripts_path, "my_script.pig"))
+          stderr, stdout = execute("jobs:run my_script -2 -1 --clustersize 5 -p FIRST_PARAM=FOO -p SECOND_PARAM=BAR", p, @git)
+          stderr.should == <<-STDERR
+ !    Cannot declare cluster as both --singlejobcluster and --permanentcluster
+STDERR
+        end
+      end
+
+      it "throws error on clusterid and permanentcluster" do
+        with_git_initialized_project do |p|
+          cluster_id = "e2790e7e8c7d48e39157238d58191346"
+
+          write_file(File.join(p.pigscripts_path, "my_script.pig"))
+          stderr, stdout = execute("jobs:run my_script -2 --clusterid e2790e7e8c7d48e39157238d58191346 -p FIRST_PARAM=FOO -p SECOND_PARAM=BAR", p, @git)
+          stderr.should == <<-STDERR
+ !    Option permanentcluster cannot be set when running a job on an existing cluster (with --clusterid option)
+          STDERR
+        end
+      end
+
       it "runs a job on a new cluster" do
         with_git_initialized_project do |p|
           # stub api requests
@@ -81,7 +138,7 @@ STDOUT
 
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]), 
-            :keepalive => true,
+            :cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
             :notify_on_job_finish => true,
             :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
 
@@ -146,7 +203,7 @@ STDOUT
           mock(Mortar::Auth.api).get_clusters() {Excon::Response.new(:body => {'clusters' => []})}
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => [], 
-            :keepalive => true,
+            :cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
             :notify_on_job_finish => true,
             :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
 
@@ -262,11 +319,10 @@ STDOUT
         with_git_initialized_project do |p|
           job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_size = 5
-          keepalive = true
 
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR\n"}]), 
-            :keepalive => true,
+            :cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
             :notify_on_job_finish => true,
             :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id})}
 
@@ -288,11 +344,10 @@ PARAMS
         with_git_initialized_project do |p|
           job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_size = 5
-          keepalive = true
 
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR\n"}]), 
-            :keepalive => true,
+            :cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
             :notify_on_job_finish => true,
             :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id})}
 
@@ -315,7 +370,6 @@ PARAMS
         with_git_initialized_project do |p|
           job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_size = 5
-          keepalive = true
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
 

data/spec/mortar/command/local_spec.rb

@@ -140,6 +140,9 @@ STDERR
         any_instance_of(Mortar::Local::Jython) do |j|
           mock(j).install_or_update.returns(true)
         end
+        any_instance_of(Mortar::Local::Controller) do |j|
+          mock(j).ensure_local_install_dir_in_gitignore.returns(true)
+        end
         stderr, stdout = execute("local:configure")
         stderr.should == ""
       end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mortar
 version: !ruby/object:Gem::Version 
-  hash: 7
+  hash: 5
   prerelease: 
   segments: 
   - 0
   - 7
-  - 2
-  version: 0.7.2
+  - 3
+  version: 0.7.3
 platform: ruby
 authors: 
 - Mortar Data
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-03-26 00:00:00 Z
+date: 2013-03-29 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mortar-api-ruby
@@ -25,12 +25,12 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 7
+        hash: 1
         segments: 
         - 0
         - 6
-        - 0
-        version: 0.6.0
+        - 3
+        version: 0.6.3
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency