mortar 0.2.0 → 0.2.1

data/lib/mortar/command.rb

@@ -128,6 +128,7 @@ module Mortar
     def self.prepare_run(cmd, args=[])
       command = parse(cmd)
 
+
       if args.include?('-h') || args.include?('--help')
         args.unshift(cmd) unless cmd =~ /^-.*/
         cmd = 'help'
@@ -138,6 +139,15 @@ module Mortar
         if %w( -v --version ).include?(cmd)
           cmd = 'version'
           command = parse(cmd)
+        # Check if the command tried matches a command file. If it does, the command exists, but doesn't have an index action
+        # Otherwise it would have been picked up by the original parse command.
+        elsif Dir[File.join(File.dirname(__FILE__), "command", "*.rb")].find { |file| file.include?(cmd) }
+          display "#{cmd} command requires arguments"
+          display
+          # Display the command's help message
+          args.unshift(cmd) unless cmd =~ /^-.*/
+          cmd = 'help'
+          command = parse('help')
         else
           error([
             "`#{cmd}` is not a mortar command.",

data/lib/mortar/command/clusters.rb

@@ -36,8 +36,13 @@ class Mortar::Command::Clusters < Mortar::Command::Base
     validate_arguments!
     
     clusters = api.get_clusters().body['clusters']
-    display_table(clusters,
+    if not clusters.empty?
+      display_table(clusters,
       %w( cluster_id size status_description cluster_type_description start_timestamp duration),
       ['cluster_id', 'Size (# of Nodes)', 'Status', 'Type', 'Start Timestamp', 'Elapsed Time'])
+    else
+      display("There are no running or recent clusters")
+    end
+    
   end
 end

data/lib/mortar/command/describe.rb

@@ -66,9 +66,7 @@ class Mortar::Command::Describe < Mortar::Command::Base
       is_finished =
         Mortar::API::Describe::STATUSES_COMPLETE.include?(describe_result["status_code"])
         
-      redisplay("Status: %s %s" % [
-        describe_result['status_description'] + (is_finished ? "" : "..."),
-        is_finished ? " " : spinner(ticks)],
+      redisplay("[#{spinner(ticks)}] Calculating schema for #{alias_name} and ancestors...",
         is_finished) # only display newline on last message
       if is_finished
         display

data/lib/mortar/command/jobs.rb

@@ -124,7 +124,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
     display
     display("Job status can be viewed on the web at:\n\n #{response['web_job_url']}")
     display
-    display("Or by running:\n\n  mortar jobs:status #{response['job_id']}")
+    display("Or by running:\n\n  mortar jobs:status #{response['job_id']} --poll")
     display
   end
   
@@ -135,6 +135,9 @@ class Mortar::Command::Jobs < Mortar::Command::Base
   #
   # Check the status of a job.
   #
+  # -p, --poll      # Poll the status of a job
+  #
+  #
   #Examples:
   #
   # $ mortar jobs:status 2000cbbba40a860a6f000000
@@ -149,43 +152,80 @@ class Mortar::Command::Jobs < Mortar::Command::Base
       error("Usage: mortar jobs:status JOB_ID\nMust specify JOB_ID.")
     end
     
-    job_status = api.get_job(job_id).body
-
-    job_display_entries = {
-      "status" => job_status["status_description"],
-      "progress" => "#{job_status["progress"]}%",
-      "cluster_id" => job_status["cluster_id"],
-      "job submitted at" => job_status["start_timestamp"],
-      "job began running at" => job_status["running_timestamp"],
-      "job finished at" => job_status["stop_timestamp"],
-      "job running for" => job_status["duration"],
-      "job run with parameters" => job_status["parameters"],
-      "error" => job_status["error"]
-    }
-    
-    unless job_status["error"].nil? || job_status["error"]["message"].nil?
-      error_context = get_error_message_context(job_status["error"]["message"])
-      unless error_context == ""
-        job_status["error"]["help"] = error_context
+    # Inner function to display the hash table when the job is complte
+    def display_job_status(job_status)
+      job_display_entries = {
+        "status" => job_status["status_description"],
+        "progress" => "#{job_status["progress"]}%",
+        "cluster_id" => job_status["cluster_id"],
+        "job submitted at" => job_status["start_timestamp"],
+        "job began running at" => job_status["running_timestamp"],
+        "job finished at" => job_status["stop_timestamp"],
+        "job running for" => job_status["duration"],
+        "job run with parameters" => job_status["parameters"],
+      }
+
+      
+      unless job_status["error"].nil? || job_status["error"]["message"].nil?
+        error_context = get_error_message_context(job_status["error"]["message"])
+        unless error_context == ""
+          job_status["error"]["help"] = error_context
+        end
+        job_status["error"].each_pair do |key, value|
+          job_display_entries["error - #{key}"] = value
+        end
       end
+      
+      if job_status["num_hadoop_jobs"] && job_status["num_hadoop_jobs_succeeded"]
+        job_display_entries["hadoop jobs complete"] = 
+          '%0.2f / %0.2f' % [job_status["num_hadoop_jobs_succeeded"], job_status["num_hadoop_jobs"]]
+      end
+      
+      if job_status["outputs"] && job_status["outputs"].length > 0
+        job_display_entries["outputs"] = Hash[job_status["outputs"].select{|o| o["alias"]}.collect do |output|
+          output_hash = {}
+          output_hash["location"] = output["location"] if output["location"]
+          output_hash["records"] = output["records"] if output["records"]
+          [output['alias'], output_hash]
+        end]
+      end
+      
+      styled_header("#{job_status["project_name"]}: #{job_status["pigscript_name"]} (job_id: #{job_status["job_id"]})")
+      styled_hash(job_display_entries)
     end
     
-    if job_status["num_hadoop_jobs"] && job_status["num_hadoop_jobs_succeeded"]
-      job_display_entries["hadoop jobs complete"] = 
-        '%0.2f / %0.2f' % [job_status["num_hadoop_jobs_succeeded"], job_status["num_hadoop_jobs"]]
-    end
-    
-    if job_status["outputs"] && job_status["outputs"].length > 0
-      job_display_entries["outputs"] = Hash[job_status["outputs"].select{|o| o["alias"]}.collect do |output|
-        output_hash = {}
-        output_hash["location"] = output["location"] if output["location"]
-        output_hash["records"] = output["records"] if output["records"]
-        [output['alias'], output_hash]
-      end]
+    # If polling the status
+    if options[:poll]
+      ticking(polling_interval) do |ticks|
+        job_status = api.get_job(job_id).body
+        # If the job is complete exit and display the table normally 
+        if Mortar::API::Jobs::STATUSES_COMPLETE.include?(job_status["status_code"] )
+          redisplay("")
+          display_job_status(job_status)
+          break
+        end
+
+        # If the job is running show the progress bar
+        if job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING
+          progressbar = "=" + ("=" * (job_status["progress"].to_i / 5)) + ">"
+
+          if job_status["num_hadoop_jobs"] && job_status["num_hadoop_jobs_succeeded"]
+            hadoop_jobs_ratio_complete = 
+              '%0.2f / %0.2f' % [job_status["num_hadoop_jobs_succeeded"], job_status["num_hadoop_jobs"]]
+          end
+
+          printf("\r[#{spinner(ticks)}] Status: [%-22s] %s%% Complete (%s MapReduce jobs finished)", progressbar, job_status["progress"], hadoop_jobs_ratio_complete)
+
+        # If the job is not complete, but not in the running state, just display its status
+        else
+          redisplay("[#{spinner(ticks)}] Status: #{job_status['status_description']}")
+        end
+      end
+    # If not polling, get the job status and display the results
+    else
+      job_status = api.get_job(job_id).body
+      display_job_status(job_status)
     end
-    
-    styled_header("#{job_status["project_name"]}: #{job_status["pigscript_name"]} (job_id: #{job_status["job_id"]})")
-    styled_hash(job_display_entries)
   end
 
   # jobs:stop JOB_ID

data/lib/mortar/command/projects.rb

@@ -105,6 +105,41 @@ class Mortar::Command::Projects < Mortar::Command::Base
     end
     
   end
+
+  # projects:set_remote PROJECT
+  #
+  # Adds the Mortar remote to the local git project. This is necessary for successfully executing many of the Mortar commands.
+  #
+  #Example:
+  #
+  # $ mortar projects:set_remote my_project
+  #
+  def set_remote
+    project_name = shift_argument
+
+    unless project_name
+      error("Usage: mortar projects:set_remote PROJECT\nMust specify PROJECT.")
+    end
+
+    unless git.has_dot_git?
+      error("Can only set the remote for an existing git project.  Please run:\n\ngit init\ngit add .\ngit commit -a -m \"first commit\"\n\nto initialize your project in git.")
+    end
+
+    if git.remotes(git_organization).include?("mortar")
+      display("The remote has already been set for project: #{project_name}")
+      return
+    end
+
+    projects = api.get_projects().body["projects"]
+    project = projects.find { |p| p['name'] == project_name}
+    unless project
+      error("No project named: #{project_name} exists. You can create this project using:\n\n mortar projects:create")
+    end
+
+    git.remote_add("mortar", project['git_url'])
+    display("Successfully added the mortar remote to the #{project_name} project")
+
+  end
   
   # projects:clone PROJECT
   #

data/lib/mortar/command/validate.rb

@@ -66,10 +66,8 @@ class Mortar::Command::Validate < Mortar::Command::Base
       validate_result = api.get_validate(validate_id).body
       is_finished =
         Mortar::API::Validate::STATUSES_COMPLETE.include?(validate_result["status_code"])
-        
-      redisplay("Status: %s %s" % [
-        validate_result['status_description']  + (is_finished ? "" : "..."),
-        is_finished ? " " : spinner(ticks)],
+       
+      redisplay("[#{spinner(ticks)}] Checking your script for problems with: Pig syntax, Python syntax, and S3 data access",
         is_finished) # only display newline on last message
       if is_finished
         display

data/lib/mortar/templates/pigscript/pigscript.pig

@@ -12,7 +12,7 @@
  * User-Defined Functions (UDFs)
  */
 
-REGISTER '../udfs/python/<%= script_name %>.py' using streaming_python as <%= script_name %>;
+REGISTER '../udfs/python/<%= script_name %>.py' USING streaming_python AS <%= script_name %>;
 <% end %>
 
 -- This is an example of loading up input data

data/lib/mortar/templates/project/gitignore

@@ -1,3 +1,4 @@
+Gemfile.lock
 *.pyc
 *.class
 *.log

data/lib/mortar/templates/project/pigscripts/pigscript.pig

@@ -10,7 +10,7 @@
 /**
  * User-Defined Functions (UDFs)
  */
-REGISTER '../udfs/python/<%= project_name %>.py' using streaming_python as <%= project_name %>;
+REGISTER '../udfs/python/<%= project_name %>.py' USING streaming_python AS <%= project_name %>;
 
 -- This is an example of loading up input data
 my_input_data = LOAD '$INPUT_PATH' 

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/spec/mortar/command/clusters_spec.rb

@@ -52,8 +52,7 @@ STDOUT
         mock(Mortar::Auth.api).get_clusters().returns(Excon::Response.new(:body => {"clusters" => []}))
          stderr, stdout = execute("clusters", nil, nil)
          stdout.should == <<-STDOUT
-cluster_id  Size (# of Nodes)  Status  Type  Start Timestamp  Elapsed Time
-----------  -----------------  ------  ----  ---------------  ------------
+There are no running or recent clusters
 STDOUT
       end
     end

data/spec/mortar/command/describe_spec.rb

@@ -86,7 +86,7 @@ Taking code snapshot... done
 Sending code snapshot to Mortar... done
 Starting describe... done
 
-\r\e[0KStatus: Pending... /\r\e[0KStatus: Gateway starting... -\r\e[0KStatus: Starting pig... \\\r\e[0KStatus: Success  
+\r\e[0K[/] Calculating schema for my_alias and ancestors...\r\e[0K[-] Calculating schema for my_alias and ancestors...\r\e[0K[\\] Calculating schema for my_alias and ancestors...\r\e[0K[|] Calculating schema for my_alias and ancestors...
 
 Results available at https://api.mortardata.com/describe/c571a8c7f76a4fd4a67c103d753e2dd5
 Opening web browser to show results... done
@@ -119,7 +119,7 @@ Taking code snapshot... done
 Sending code snapshot to Mortar... done
 Starting describe... done
 
-\r\e[0KStatus: Pending... /\r\e[0KStatus: Failed  
+\r\e[0K[/] Calculating schema for my_alias and ancestors...\r\e[0K[-] Calculating schema for my_alias and ancestors...
 
 STDOUT
           stderr.should == <<-STDERR

data/spec/mortar/command/jobs_spec.rb

@@ -56,7 +56,7 @@ Job status can be viewed on the web at:
 
 Or by running:
 
-  mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5
+  mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 --poll
 
 STDOUT
         end
@@ -87,7 +87,7 @@ Job status can be viewed on the web at:
 
 Or by running:
 
-  mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5
+  mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 --poll
 
 STDOUT
         end
@@ -116,7 +116,7 @@ Job status can be viewed on the web at:
 
 Or by running:
 
-  mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5
+  mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 --poll
 
 STDOUT
         end
@@ -339,11 +339,10 @@ STDOUT
           stdout.should == <<-STDOUT
 === myproject: my_script (job_id: c571a8c7f76a4fd4a67c103d753e2dd5)
 cluster_id:              e2790e7e8c7d48e39157238d58191346
-error:                   
-  column_number:   34
-  line_number:     43
-  message:         An error occurred and here's some more info
-  type:            RuntimeError
+error - column_number:   34
+error - line_number:     43
+error - message:         An error occurred and here's some more info
+error - type:            RuntimeError
 hadoop jobs complete:    0.00 / 4.00
 job began running at:    2012-02-28T03:41:52.613000+00:00
 job finished at:         2012-02-28T03:45:52.613000+00:00
@@ -354,6 +353,85 @@ job running for:         6 mins
 job submitted at:        2012-02-28T03:35:42.831000+00:00
 progress:                55%
 status:                  Execution error
+STDOUT
+        end
+      end
+      it "gets status for a running job using polling" do
+        with_git_initialized_project do |p|
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          pigscript_name = "my_script"
+          project_name = "myproject"
+          status_code = Mortar::API::Jobs::STATUS_RUNNING
+          progress = 0
+          cluster_id = "e2790e7e8c7d48e39157238d58191346"
+          start_timestamp = "2012-02-28T03:35:42.831000+00:00"
+          stop_timestamp = "2012-02-28T03:44:52.613000+00:00"
+          running_timestamp = "2012-02-28T03:41:52.613000+00:00"
+          parameters = {"my_param_1" => "value1", "MY_PARAM_2" => "3"}
+
+          mock(Mortar::Auth.api).get_job(job_id).returns(Excon::Response.new(:body => {"job_id" => job_id,
+            "pigscript_name" => pigscript_name,
+            "project_name" => project_name,
+            "status_code" => status_code,
+            "status_description" => "Execution error",
+            "progress" => progress,
+            "cluster_id" => cluster_id,
+            "start_timestamp" => start_timestamp,
+            "running_timestamp" => running_timestamp,
+            "duration" => "6 mins",
+            "num_hadoop_jobs" => 4,
+            "num_hadoop_jobs_succeeded" => 0,
+            "parameters" => parameters
+            }))
+
+          status_code = Mortar::API::Jobs::STATUS_SUCCESS
+          progress = 100
+          outputs = [{'name'=> 'hottest_songs_of_the_decade', 
+                      'records' => 10, 
+                      'alias' => 'output_data',
+                      'location' => 's3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data'},
+                     {'name'=> 'hottest_songs_of_the_decade', 
+                      'records' => 100,
+                      'alias' => 'output_data_2',
+                      'location' => 's3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data_2'}]
+
+          mock(Mortar::Auth.api).get_job(job_id).returns(Excon::Response.new(:body => {"job_id" => job_id,
+            "pigscript_name" => pigscript_name,
+            "project_name" => project_name,
+            "status_code" => status_code,
+            "status_description" => "Success",
+            "progress" => progress,
+            "cluster_id" => cluster_id,
+            "start_timestamp" => start_timestamp,
+            "running_timestamp" => running_timestamp,
+            "stop_timestamp" => stop_timestamp,
+            "duration" => "6 mins",
+            "num_hadoop_jobs" => 4,
+            "num_hadoop_jobs_succeeded" => 4,
+            "parameters" => parameters,
+            "outputs" => outputs
+            }))
+          stderr, stdout = execute("jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 -p", p, @git)
+          stdout.should == <<-STDOUT
+\r[/] Status: [=>                    ] 0% Complete (0.00 / 4.00 MapReduce jobs finished)\r\e[0K=== myproject: my_script (job_id: c571a8c7f76a4fd4a67c103d753e2dd5)
+cluster_id:              e2790e7e8c7d48e39157238d58191346
+hadoop jobs complete:    4.00 / 4.00
+job began running at:    2012-02-28T03:41:52.613000+00:00
+job finished at:         2012-02-28T03:44:52.613000+00:00
+job run with parameters: 
+  MY_PARAM_2:   3
+  my_param_1:   value1
+job running for:         6 mins
+job submitted at:        2012-02-28T03:35:42.831000+00:00
+outputs:                 
+  output_data:   
+    location:     s3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data
+    records:      10
+  output_data_2: 
+    location:     s3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data_2
+    records:      100
+progress:                100%
+status:                  Success
 STDOUT
         end
       end
@@ -380,7 +458,6 @@ STDOUT
 
 
       end
-            
     end
   end
 end

data/spec/mortar/command/projects_spec.rb

@@ -129,6 +129,62 @@ STDOUT
       end
       
     end
+
+    context("set_remote") do
+      
+      it "sets the remote of a project" do
+        with_git_initialized_project do |p|           
+          project_name = p.name
+          project_git_url = "git@github.com:mortarcode-dev/#{project_name}"
+          `git remote rm mortar`
+          mock(Mortar::Auth.api).get_projects().returns(Excon::Response.new(:body => {"projects" => [ { "name" => project_name, "status" => Mortar::API::Projects::STATUS_ACTIVE, "git_url" => project_git_url } ] })).ordered   
+
+          mock(@git).remote_add("mortar", project_git_url)
+
+          stderr, stdout = execute("projects:set_remote #{project_name}", p, @git)
+          stdout.should == <<-STDOUT
+Successfully added the mortar remote to the myproject project
+STDOUT
+        end
+      end
+
+      it "remote already added" do
+        with_git_initialized_project do |p|           
+          project_name = p.name
+
+          stderr, stdout = execute("projects:set_remote #{project_name}", p, @git)
+          stdout.should == <<-STDERR
+The remote has already been set for project: myproject
+STDERR
+        end
+      end
+
+      it "No project given" do
+        with_git_initialized_project do |p|           
+          stderr, stdout = execute("projects:set_remote", p, @git)
+          stderr.should == <<-STDERR
+ !    Usage: mortar projects:set_remote PROJECT
+ !    Must specify PROJECT.
+STDERR
+        end
+      end
+
+      it "No project with that name" do
+        with_git_initialized_project do |p|           
+          project_name = p.name
+          project_git_url = "git@github.com:mortarcode-dev/#{project_name}"
+          mock(Mortar::Auth.api).get_projects().returns(Excon::Response.new(:body => {"projects" => [ { "name" => "derp", "status" => Mortar::API::Projects::STATUS_ACTIVE, "git_url" => project_git_url } ] })).ordered   
+          `git remote rm mortar`
+
+          stderr, stdout = execute("projects:set_remote #{project_name}", p, @git)
+          stderr.should == <<-STDERR
+ !    No project named: myproject exists. You can create this project using:
+ !    
+ !     mortar projects:create
+STDERR
+        end
+      end
+    end
     
     
     context("clone") do

data/spec/mortar/command/validate_spec.rb

@@ -71,7 +71,7 @@ Taking code snapshot... done
 Sending code snapshot to Mortar... done
 Starting validate... done
 
-\r\e[0KStatus: Pending... /\r\e[0KStatus: GATEWAY_STARTING... -\r\e[0KStatus: Starting... \\\r\e[0KStatus: Success  
+\r\e[0K[/] Checking your script for problems with: Pig syntax, Python syntax, and S3 data access\r\e[0K[-] Checking your script for problems with: Pig syntax, Python syntax, and S3 data access\r\e[0K[\\] Checking your script for problems with: Pig syntax, Python syntax, and S3 data access\r\e[0K[|] Checking your script for problems with: Pig syntax, Python syntax, and S3 data access
 
 Your script is valid.
 STDOUT
@@ -103,7 +103,7 @@ Taking code snapshot... done
 Sending code snapshot to Mortar... done
 Starting validate... done
 
-\r\e[0KStatus: Pending... /\r\e[0KStatus: Failed  
+\r\e[0K[/] Checking your script for problems with: Pig syntax, Python syntax, and S3 data access\r\e[0K[-] Checking your script for problems with: Pig syntax, Python syntax, and S3 data access
 
 STDOUT
           stderr.should == <<-STDERR

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mortar
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 21
   prerelease: 
   segments: 
   - 0
   - 2
-  - 0
-  version: 0.2.0
+  - 1
+  version: 0.2.1
 platform: ruby
 authors: 
 - Mortar Data