mortar 0.4.4 → 0.5.0

data/README.md

@@ -1,6 +1,6 @@
 # Mortar Development Framework
 
-Mortar is a platform as a service for Hadoop. With Mortar, you can run jobs on Hadoop using Apache Pig and Python without any special training.
+[Mortar](http://www.mortardata.com/) is a platform as a service for Hadoop. With Mortar, you can run jobs on Hadoop using Apache Pig and Python without any special training.
 
 The Mortar Development Framework lets you develop Mortar Hadoop jobs directly on your local computer without installing any Hadoop libraries.  Lots more info can be found on the [Mortar help site](http://help.mortardata.com).  
 

data/lib/mortar/command/clusters.rb

@@ -37,4 +37,23 @@ class Mortar::Command::Clusters < Mortar::Command::Base
     end
     
   end
+  
+   # clusters: stop CLUSTER_ID
+   #
+   # Stop a running cluster.
+   #
+   def stop
+     cluster_id = shift_argument
+     unless cluster_id
+       error("Usage: mortar clusters:stop CLUSTER_ID\nMust specify CLUSTER_ID.")
+     end   
+         
+     response = api.stop_cluster(cluster_id) .body  
+
+     if response['message'].nil?
+        display("Stopping cluster #{cluster_id}.")
+     else
+       display(response['message'])
+     end
+   end
 end

data/lib/mortar/command/illustrate.rb

@@ -40,9 +40,12 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
     pigscript_name = shift_argument
     alias_name = shift_argument
     skip_pruning = options[:skippruning] ||= false
+    
+    # TODO: When illustrating without alias works, remove the `&& alias_name` to re-enable the feature on CLI
     unless pigscript_name && alias_name
       error("Usage: mortar illustrate PIGSCRIPT ALIAS\nMust specify PIGSCRIPT and ALIAS.")
     end
+    
     validate_arguments!
     validate_git_based_project!
     pigscript = validate_pigscript!(pigscript_name)

data/lib/mortar/command/jobs.rb

@@ -60,6 +60,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
   # -1, --singlejobcluster      # Stop the cluster after job completes.  (Default: false—-cluster can be used for other jobs, and will shut down after 1 hour of inactivity)
   # -p, --parameter NAME=VALUE  # Set a pig parameter value in your script.
   # -f, --param-file PARAMFILE  # Load pig parameter values from a file.
+  # -d, --donotnotify           # Don't send an email on job completion.  (Default: false--an email will be sent to you once the job completes)
   #
   #Examples:
   #
@@ -74,7 +75,8 @@ class Mortar::Command::Jobs < Mortar::Command::Base
     validate_arguments!
 
     unless options[:clusterid] || options[:clustersize]
-      error("Please provide either the --clustersize option to run job on a new cluster, or --clusterid to run on an existing one.")
+      options[:clustersize] = 2
+      display("Defaulting to running job on new cluster of size 2")
     end
       
     if options[:clusterid]
@@ -88,6 +90,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
     validate_git_based_project!
     pigscript = validate_pigscript!(pigscript_name)
     git_ref = create_and_push_snapshot_branch(git, project)
+    notify_on_job_finish = ! options[:donotnotify]
     
     # post job to API
     response = action("Requesting job execution") do
@@ -96,11 +99,13 @@ class Mortar::Command::Jobs < Mortar::Command::Base
         keepalive = ! options[:singlejobcluster]
         api.post_job_new_cluster(project.name, pigscript.name, git_ref, cluster_size, 
           :parameters => pig_parameters,
-          :keepalive => keepalive).body
+          :keepalive => keepalive,
+          :notify_on_job_finish => notify_on_job_finish).body
       else
         cluster_id = options[:clusterid]
         api.post_job_existing_cluster(project.name, pigscript.name, git_ref, cluster_id,
-          :parameters => pig_parameters).body
+          :parameters => pig_parameters,
+          :notify_on_job_finish => notify_on_job_finish).body
       end
     end
 

data/lib/mortar/templates/pigscript/pigscript.pig

@@ -1,11 +1,13 @@
 /**
  * <%= script_name %>
- *
- * Required parameters:
- *
- * - INPUT_PATH Input path for script data (e.g. s3n://hawk-example-data/tutorial/excite.log.bz2)
- * - OUTPUT_PATH Output path for script data (e.g. s3n://my-output-bucket/<%= script_name %>)
  */
+ 
+/** 
+ * Parameters - set default values here; you can override with -p on the command-line.
+ */
+ 
+<%= '%default'%> INPUT_PATH 's3n://hawk-example-data/tutorial/excite.log.bz2'
+<%= '%default'%> OUTPUT_PATH 's3n://my-output-bucket/$MORTAR_EMAIL_S3_ESCAPED/<%= script_name %>'
 
 <% if not options[:skip_udf] %>
 /**

data/lib/mortar/templates/project/pigscripts/pigscript.pig

@@ -1,11 +1,13 @@
 /**
  * <%= project_name %>
- *
- * Required parameters:
- *
- * - INPUT_PATH Input path for script data (e.g. s3n://hawk-example-data/tutorial/excite.log.bz2)
- * - OUTPUT_PATH Output path for script data (e.g. s3n://my-output-bucket/<%= project_name %>)
  */
+ 
+/** 
+ * Parameters - set default values here; you can override with -p on the command-line.
+ */
+ 
+<%= '%default'%> INPUT_PATH 's3n://hawk-example-data/tutorial/excite.log.bz2'
+<%= '%default'%> OUTPUT_PATH 's3n://my-output-bucket/$MORTAR_EMAIL_S3_ESCAPED/<%= project_name %>'
 
 /**
  * User-Defined Functions (UDFs)

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.4.4"
+  VERSION = "0.5.0"
 end

data/spec/mortar/command/clusters_spec.rb

@@ -56,5 +56,28 @@ There are no running or recent clusters
 STDOUT
       end
     end
+    
+     context("stop") do
+        it "Stops a running cluster with default message" do
+          cluster_id = "1234abcd"
+          mock(Mortar::Auth.api).stop_cluster(cluster_id) {Excon::Response.new(:body => {"success" => true})}
+
+          stderr, stdout = execute("clusters:stop #{cluster_id}")
+          stdout.should == <<-STDOUT
+Stopping cluster #{cluster_id}.
+STDOUT
+      end
+
+        it "Stops a running cluster with server message" do
+          cluster_id = "1234abcd"
+          message = "some awesome message"
+          mock(Mortar::Auth.api).stop_cluster(cluster_id) {Excon::Response.new(:body => {"success" => true, "message" => message})}
+
+          stderr, stdout = execute("clusters:stop #{cluster_id}")
+          stdout.should == "#{message}\n"
+        end
+      end
+    
   end
+    
 end

data/spec/mortar/command/jobs_spec.rb

@@ -48,7 +48,8 @@ module Mortar::Command
 
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]), 
-            :keepalive => false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+            :keepalive => false,
+            :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
           stderr, stdout = execute("jobs:run my_script -1 --clustersize 5 -p FIRST_PARAM=FOO -p SECOND_PARAM=BAR", p, @git)
@@ -79,7 +80,8 @@ STDOUT
 
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]), 
-            :keepalive => true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+            :keepalive => true,
+            :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
           stderr, stdout = execute("jobs:run my_script --clustersize 5 -p FIRST_PARAM=FOO -p SECOND_PARAM=BAR", p, @git)
@@ -101,6 +103,39 @@ STDOUT
         end
       end
 
+      it "runs a job with no cluster defined" do
+        with_git_initialized_project do |p|
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
+          cluster_size = 2
+
+          mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
+            :parameters => [], 
+            :keepalive => true,
+            :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+
+          write_file(File.join(p.pigscripts_path, "my_script.pig"))
+          stderr, stdout = execute("jobs:run my_script ", p, @git)
+          stdout.should == <<-STDOUT
+Defaulting to running job on new cluster of size 2
+Taking code snapshot... done
+Sending code snapshot to Mortar... done
+Requesting job execution... done
+job_id: c571a8c7f76a4fd4a67c103d753e2dd5
+
+Job status can be viewed on the web at:
+
+ http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
+
+Or by running:
+
+  mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 --poll
+
+STDOUT
+
+        end
+      end
+
       it "runs a job on an existing cluster" do
         with_git_initialized_project do |p|
           # stub api requests
@@ -108,10 +143,10 @@ STDOUT
           job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_id = "e2790e7e8c7d48e39157238d58191346"
 
-          mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :parameters => []) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+          mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :parameters => [], :notify_on_job_finish => false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
-          stderr, stdout = execute("jobs:run my_script --clusterid e2790e7e8c7d48e39157238d58191346", p, @git)
+          stderr, stdout = execute("jobs:run my_script --clusterid e2790e7e8c7d48e39157238d58191346 -d", p, @git)
           stdout.should == <<-STDOUT
 Taking code snapshot... done
 Sending code snapshot to Mortar... done
@@ -138,7 +173,8 @@ STDOUT
 
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR\n"}]), 
-            :keepalive => true) {Excon::Response.new(:body => {"job_id" => job_id})}
+            :keepalive => true,
+            :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id})}
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
 
@@ -162,7 +198,8 @@ PARAMS
 
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR\n"}]), 
-            :keepalive => true) {Excon::Response.new(:body => {"job_id" => job_id})}
+            :keepalive => true,
+            :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id})}
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mortar
 version: !ruby/object:Gem::Version 
-  hash: 7
+  hash: 11
   prerelease: 
   segments: 
   - 0
-  - 4
-  - 4
-  version: 0.4.4
+  - 5
+  - 0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Mortar Data
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-11-28 00:00:00 Z
+date: 2013-01-17 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mortar-api-ruby
@@ -25,12 +25,12 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 13
+        hash: 11
         segments: 
         - 0
-        - 4
-        - 1
-        version: 0.4.1
+        - 5
+        - 0
+        version: 0.5.0
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency