mortar 0.15.27 → 0.15.28

data/lib/mortar/command/jobs.rb

@@ -208,9 +208,8 @@ class Mortar::Command::Jobs < Mortar::Command::Base
       error("Usage: mortar jobs:status JOB_ID\nMust specify JOB_ID.")
     end
     validate_arguments!
-    
-    # Inner function to display the hash table when the job is complte
-    def display_job_status(job_status)
+
+    def pig_job_display_entries(job_status)
       job_display_entries = {
         "status" => job_status["status_description"],
         "cluster_id" => job_status["cluster_id"],
@@ -220,18 +219,35 @@ class Mortar::Command::Jobs < Mortar::Command::Base
         "job running for" => job_status["duration"],
         "job run with parameters" => job_status["parameters"],
       }
+    end
 
-      
-      unless job_status["error"].nil? || job_status["error"]["message"].nil?
-        error_context = get_error_message_context(job_status["error"]["message"])
-        unless error_context == ""
-          job_status["error"]["help"] = error_context
-        end
-        job_status["error"].each_pair do |key, value|
-          job_display_entries["error - #{key}"] = value
-        end
+    def luigi_job_display_entries(job_status)
+      status = job_status['status_description']
+      if !job_status['status_detail_description'].nil?
+        status += " - #{job_status['status_detail_description']}"
       end
-      
+
+      job_display_entries = {
+        "status" => status,
+        "job submitted at" => job_status["start_timestamp"],
+        "job began running at" => job_status["running_timestamp"],
+        "job finished at" => job_status["stop_timestamp"],
+        "job running for" => job_status["duration"],
+        "job run with parameters" => job_status["parameters"],
+      }
+    end
+
+    def add_pig_job_fields(job_status, job_display_entries)
+     if job_status["outputs"] && job_status["outputs"].length > 0
+        job_display_entries["outputs"] = Hash.new { |h,k| h[k] = [] }
+        job_status["outputs"].select{|o| o["alias"]}.collect{ |output|
+          output_hash = {}
+          output_hash["location"] = output["location"] if output["location"]
+          output_hash["records"] = output["records"] if output["records"]
+          [output['alias'], output_hash]
+        }.each{ |k,v| job_display_entries["outputs"][k] << v }
+      end
+
       if job_status["num_hadoop_jobs"] && job_status["num_hadoop_jobs_succeeded"]
         job_display_entries["progress"] = "#{job_status["progress"]}%"
         job_display_entries["hadoop jobs complete"] = 
@@ -241,24 +257,28 @@ class Mortar::Command::Jobs < Mortar::Command::Base
       else
         job_display_entries["progress"] = "#{job_status["progress"]}%"
       end
-      
-      if job_status["outputs"] && job_status["outputs"].length > 0
-        job_display_entries["outputs"] = Hash.new { |h,k| h[k] = [] }
-        job_status["outputs"].select{|o| o["alias"]}.collect{ |output|
-          output_hash = {}
-          output_hash["location"] = output["location"] if output["location"]
-          output_hash["records"] = output["records"] if output["records"]
-          [output['alias'], output_hash]
-        }.each{ |k,v| job_display_entries["outputs"][k] << v }
+    end
+    
+    # Inner function to display the hash table when the job is complte
+    def display_job_status(job_status)
+      if (job_status['job_type'] == Mortar::API::Jobs::JOB_TYPE_LUIGI)
+        job_display_entries = luigi_job_display_entries(job_status)
+      else
+        job_display_entries = pig_job_display_entries(job_status)
+        add_pig_job_fields(job_status, job_display_entries)
       end
       
-      if job_status["controlscript_name"]
-        script_name = job_status["controlscript_name"]
-      else 
-        script_name = job_status["pigscript_name"]
+      unless job_status["error"].nil? || job_status["error"]["message"].nil?
+        error_context = get_error_message_context(job_status["error"]["message"])
+        unless error_context == ""
+          job_status["error"]["help"] = error_context
+        end
+        job_status["error"].each_pair do |key, value|
+          job_display_entries["error - #{key}"] = value
+        end
       end
 
-      styled_header("#{job_status["project_name"]}: #{script_name} (job_id: #{job_status["job_id"]})")
+      styled_header("#{job_status["display_name"]} (job_id: #{job_status["job_id"]})")
       styled_hash(job_display_entries)
     end
     
@@ -274,8 +294,8 @@ class Mortar::Command::Jobs < Mortar::Command::Base
           break
         end
 
-        # If the job is running show the progress bar
-        if job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING && job_status["num_hadoop_jobs"]
+        # If not a Luigi job and job is running show the progress bar
+        if job_status['job_type'] != Mortar::API::Jobs::JOB_TYPE_LUIGI and job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING && job_status["num_hadoop_jobs"]
           progressbar = "=" + ("=" * (job_status["progress"].to_i / 5)) + ">"
 
           if job_status["num_hadoop_jobs"] && job_status["num_hadoop_jobs_succeeded"]
@@ -285,7 +305,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
 
           printf("\r[#{spinner(ticks)}] Status: [%-22s] %s%% Complete (%s MapReduce jobs finished)", progressbar, job_status["progress"], hadoop_jobs_ratio_complete)
 
-        elsif job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING
+        elsif job_status['job_type'] != Mortar::API::Jobs::JOB_TYPE_LUIGI and job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING
           jobs_complete = '%0.2f' % job_status["num_hadoop_jobs_succeeded"]
           printf("\r[#{spinner(ticks)}] #{jobs_complete} MapReduce Jobs complete.")
 
@@ -295,6 +315,9 @@ class Mortar::Command::Jobs < Mortar::Command::Base
           if !job_status['status_details'].nil?
             job_display_status += " - #{job_status['status_details']}"
           end
+          if !job_status['status_detail_description'].nil?
+            job_display_status += " - #{job_status['status_detail_description']}"
+          end
           redisplay("[#{spinner(ticks)}] Status: #{job_display_status}")
         end
       end

data/lib/mortar/templates/project/luigiscripts/luigiscript.py

@@ -20,11 +20,11 @@ from mortar.luigi import mortartask
   https://help.mortardata.com/technologies/luigi
 
   To Run:
-    mortar luigi luigiscripts/<%= project_name_alias %>_luigi.py \
-      --output-base-path "s3://mortar-example-output-data/<your_username_here>/<%= project_name_alias %>"
+    mortar luigi luigiscripts/<%= project_name %>_luigi.py \
+    --output-base-path "s3://mortar-example-output-data/<your_username_here>/<%= project_name %>"
 """
 
-MORTAR_PROJECT = '<%= project_name_alias %>'
+MORTAR_PROJECT = '<%= project_name %>'
 
 # helper function
 def create_full_path(base_path, sub_path):
@@ -97,7 +97,7 @@ class RunMyExamplePigScript(mortartask.MortarProjectPigscriptTask):
     """
     Name of Pig script to run.
     """
-    return '<%= project_name_alias %>'
+    return '<%= project_name %>'
 
 
 class ShutdownClusters(mortartask.MortarClusterShutdownTask):
@@ -131,4 +131,4 @@ if __name__ == "__main__":
   need to be run should be called in the main method. In this case ShutdownClusters
   is being called.
   """
-  luigi.run(main_task_cls= ShutdownClusters)
+  luigi.run(main_task_cls= ShutdownClusters)

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.15.27"
+  VERSION = "0.15.28"
 end

data/spec/mortar/command/jobs_spec.rb

@@ -568,6 +568,7 @@ STDERR
           mock(Mortar::Auth.api).get_job(job_id) {Excon::Response.new(:body => {"job_id" => job_id,
             "pigscript_name" => pigscript_name,
             "project_name" => project_name,
+            "display_name" => "#{project_name}: #{pigscript_name}",
             "status_code" => status_code,
             "status_description" => "Success",
             "progress" => progress,
@@ -621,6 +622,7 @@ STDOUT
           mock(Mortar::Auth.api).get_job(job_id) {Excon::Response.new(:body => {"job_id" => job_id,
             "pigscript_name" => pigscript_name,
             "project_name" => project_name,
+            "display_name" => "#{project_name}: #{pigscript_name}",
             "status_code" => status_code,
             "status_description" => "Running",
             "progress" => progress,
@@ -668,6 +670,7 @@ STDOUT
           mock(Mortar::Auth.api).get_job(job_id) {Excon::Response.new(:body => {"job_id" => job_id,
             "pigscript_name" => pigscript_name,
             "project_name" => project_name,
+            "display_name" => "#{project_name}: #{pigscript_name}",
             "status_code" => status_code,
             "status_description" => "Execution error",
             "progress" => progress,
@@ -719,6 +722,7 @@ STDOUT
           mock(Mortar::Auth.api).get_job(job_id).returns(Excon::Response.new(:body => {"job_id" => job_id,
             "controlscript_name" => controlscript_name,
             "project_name" => project_name,
+            "display_name" => "#{project_name}: #{controlscript_name}",
             "status_code" => status_code,
             "status_description" => "Execution error",
             "progress" => progress,
@@ -744,6 +748,7 @@ STDOUT
           mock(Mortar::Auth.api).get_job(job_id).returns(Excon::Response.new(:body => {"job_id" => job_id,
             "controlscript_name" => controlscript_name,
             "project_name" => project_name,
+            "display_name" => "#{project_name}: #{controlscript_name}",
             "status_code" => status_code,
             "status_description" => "Success",
             "progress" => progress,
@@ -798,6 +803,7 @@ STDOUT
           mock(Mortar::Auth.api).get_job(job_id).returns(Excon::Response.new(:body => {"job_id" => job_id,
             "pigscript_name" => pigscript_name,
             "project_name" => project_name,
+            "display_name" => "#{project_name}: #{pigscript_name}",
             "status_code" => status_code,
             "status_description" => "Execution error",
             "progress" => progress,
@@ -824,6 +830,7 @@ STDOUT
           mock(Mortar::Auth.api).get_job(job_id).returns(Excon::Response.new(:body => {"job_id" => job_id,
             "pigscript_name" => pigscript_name,
             "project_name" => project_name,
+            "display_name" => "#{project_name}: #{pigscript_name}",
             "status_code" => status_code,
             "status_description" => "Success",
             "progress" => progress,
@@ -862,28 +869,97 @@ STDOUT
         end
       end
 
-      context("stop") do
-        it "Stops a running job with default message" do
-          job_id = "1234abcd"
-          mock(Mortar::Auth.api).stop_job(job_id) {Excon::Response.new(:body => {"success" => true})}
+      it "gets status for a running pipeline job using polling" do
+        #{"project_name"=>"jkarn-mortar-recsys", "status_code"=>"finished", "status_detail_description"=>"Success", "status_detail"=>"success", "duration"=>"< 1 min", "status_description"=>"Finished", "script_variables"=>[], "start_timestamp"=>"2014-10-03T12:34:40.994000+00:00", "display_name"=>"jkarn-mortar-recsys: simple-retail-generate-signals", "job_id"=>"542e97cea40a865ab7000dfe", "request_timestamp"=>"2014-10-03T12:34:22.439000+00:00", "luigiscript_name"=>"simple-retail-generate-signals", "job_type"=>"luigi", "error"=>nil, "git_ref"=>"015690ad2c951c67366e36847be59646d441c5b1", "stop_timestamp"=>"2014-10-03T12:35:20.241000+00:00"}
+        with_git_initialized_project do |p|
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          luigiscript_name = "my_luigi_script"
+          project_name = "myproject"
+          display_name = "#{project_name}: #{luigiscript_name}"
+          status_code = Mortar::API::Jobs::LUIGI_JOB_STATUS__STARTING
+          status_description = "#{status_code} description"
+          status_detail = nil
+          status_detail_description = "#{status_detail} description"
+          start_timestamp = "2012-02-28T03:35:42.831000+00:00"
+          stop_timestamp = "2012-02-28T03:44:52.613000+00:00"
+          running_timestamp = "2012-02-28T03:41:52.613000+00:00"
+          duration = "< 1 min"
 
-          stderr, stdout = execute("jobs:stop #{job_id}")
-          stdout.should == <<-STDOUT
-Stopping job #{job_id}.
+          mock(Mortar::Auth.api).get_job(job_id).returns(Excon::Response.new(:body => {"job_id" => job_id,
+            "project_name"=>project_name,
+            "status_code"=>status_code,
+            "status_detail_description"=>status_detail_description,
+            "status_detail"=>status_detail,
+            "duration"=>duration,
+            "status_description"=>status_description,
+            "script_variables"=>[],
+            "start_timestamp"=>start_timestamp,
+            "display_name"=>display_name,
+            "job_id"=>job_id,
+            "request_timestamp"=>running_timestamp,
+            "luigiscript_name"=>luigiscript_name, 
+            "job_type"=>Mortar::API::Jobs::JOB_TYPE_LUIGI, 
+            "error"=>nil, 
+            "git_ref"=>"015690ad2c951c67366e36847be59646d441c5b1", 
+            "stop_timestamp"=>stop_timestamp
+          }))
+
+          status_code = Mortar::API::Jobs::LUIGI_JOB_STATUS__FINISHED
+          status_detail = "success"
+
+          status_description = "#{status_code} description"
+          status_detail_description = "#{status_detail} description"
+
+        mock(Mortar::Auth.api).get_job(job_id).returns(Excon::Response.new(:body => {"job_id" => job_id,
+            "project_name"=>project_name,
+            "status_code"=>status_code,
+            "status_detail_description"=>"#{status_detail} description",
+            "status_detail"=>status_detail,
+            "duration"=>duration,
+            "status_description"=>"#{status_code} description",
+            "script_variables"=>[],
+            "start_timestamp"=>start_timestamp,
+            "display_name"=>display_name,
+            "job_id"=>job_id,
+            "request_timestamp"=>running_timestamp,
+            "luigiscript_name"=>luigiscript_name, 
+            "job_type"=>Mortar::API::Jobs::JOB_TYPE_LUIGI, 
+            "error"=>nil, 
+            "git_ref"=>"015690ad2c951c67366e36847be59646d441c5b1", 
+            "stop_timestamp"=>stop_timestamp
+          }))
+        stderr, stdout = execute("jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 -p --polling_interval 0.05", p, @git)
+        stdout.should == <<-STDOUT
+\r\e[0K[/] Status: starting description -  description\r\e[0K=== #{display_name} (job_id: #{job_id})
+job finished at:         #{stop_timestamp}
+job running for:         #{duration}
+job submitted at:        #{start_timestamp}
+status:                  #{status_description} - #{status_detail_description}
 STDOUT
         end
+      end
+    end
 
-        it "Stops a running job with server message" do
-          job_id = "1234abcd"
-          message = "some awesome message"
-          mock(Mortar::Auth.api).stop_job(job_id) {Excon::Response.new(:body => {"success" => true, "message" => message})}
+    context("stop") do
+      it "Stops a running job with default message" do
+        job_id = "1234abcd"
+        mock(Mortar::Auth.api).stop_job(job_id) {Excon::Response.new(:body => {"success" => true})}
 
-          stderr, stdout = execute("jobs:stop #{job_id}")
-          stdout.should == "#{message}\n"
-        end
+        stderr, stdout = execute("jobs:stop #{job_id}")
+        stdout.should == <<-STDOUT
+Stopping job #{job_id}.
+STDOUT
+      end
 
+      it "Stops a running job with server message" do
+        job_id = "1234abcd"
+        message = "some awesome message"
+        mock(Mortar::Auth.api).stop_job(job_id) {Excon::Response.new(:body => {"success" => true, "message" => message})}
 
+        stderr, stdout = execute("jobs:stop #{job_id}")
+        stdout.should == "#{message}\n"
       end
+
     end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mortar
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 27
   prerelease: 
   segments: 
   - 0
   - 15
-  - 27
-  version: 0.15.27
+  - 28
+  version: 0.15.28
 platform: ruby
 authors: 
 - Mortar Data
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2014-10-06 00:00:00 Z
+date: 2014-10-07 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rdoc
@@ -41,12 +41,12 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 47
+        hash: 45
         segments: 
         - 0
         - 8
-        - 8
-        version: 0.8.8
+        - 9
+        version: 0.8.9
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency