mortar 0.1.0 → 0.2.0

data/lib/mortar/command/clusters.rb

@@ -16,7 +16,7 @@
 
 require "mortar/command/base"
 
-## manage clusters
+# view running and recent clusters
 #
 class Mortar::Command::Clusters < Mortar::Command::Base
     
@@ -28,7 +28,9 @@ class Mortar::Command::Clusters < Mortar::Command::Base
   #
   # $ mortar clusters
   # 
-  #TBD
+  #cluster_id                Size (# of Nodes)  Status    Type         Start Timestamp  Elapsed Time
+  #------------------------  -----------------  --------  -----------  ---------------  ------------
+  #5023ca3ce0c222b7caaa546a                  3  Starting  Multi - Job                   
   #
   def index
     validate_arguments!

data/lib/mortar/command/describe.rb

@@ -17,7 +17,7 @@
 require "mortar/command/base"
 require "mortar/snapshot"
 
-# manage pig scripts
+# show data schema for pigscript
 #
 class Mortar::Command::Describe < Mortar::Command::Base
   
@@ -32,9 +32,16 @@ class Mortar::Command::Describe < Mortar::Command::Base
   #
   # Examples:
   #
-  # $ mortar describe
+  # $ mortar describe generate_regression_model_coefficients songs_sample
   # 
-  # TBD
+  #Taking code snapshot... done
+  #Sending code snapshot to Mortar... done
+  #Starting describe... done
+  #
+  #Status: Success  
+  #
+  #Results available at https://hawk.mortardata.com/describes/2000cf2e421aa909690000af
+  #Opening web browser to show results... done
   #
   def index
     pigscript_name = shift_argument

data/lib/mortar/command/generate.rb

@@ -20,20 +20,30 @@ require "mortar/generators/pigscript_generator"
 require "mortar/generators/macro_generator"
 require "mortar/command/base"
 
-# generate new projects and scaffolding
+# generate mortar code (project, pigscript, python_udf, macro)
 #
 class Mortar::Command::Generate < Mortar::Command::Base
 
-  # generate:project
+  # generate:project [PROJECTNAME]
   #
-  # generate new project
+  # Generate the files and directory structure necessary for a Mortar project.
   # 
   #
   # Examples:
   #
   # $ mortar generate:project
   # 
-  # TBD
+  #create  README.md
+  #create  .gitignore
+  #create  Gemfile
+  #create  pigscripts
+  #create  pigscripts/my_new_project.pig
+  #create  macros
+  #create  macros/.gitkeep
+  #create  udfs
+  #create  udfs/python
+  #create  udfs/python/my_new_project.py
+  #   run  bundle install
   # 
   def _project
     project_name = shift_argument
@@ -49,16 +59,16 @@ class Mortar::Command::Generate < Mortar::Command::Base
 
 
 
-  # generate:python_udf
+  # generate:python_udf [UDFNAME]
   #
-  # generate new python user defined function
+  # Generate a new python user defined function
   # 
   #
   # Examples:
   #
-  # $ mortar generate:python_udf UDFNAME
+  # $ mortar generate:python_udf my_new_udf
   # 
-  # TBD
+  #create  udfs/python/my_new_udf.py
   # 
   def python_udf
     udf_name = shift_argument
@@ -70,17 +80,18 @@ class Mortar::Command::Generate < Mortar::Command::Base
 
   end
 
-  # generate:pigscript
+  # generate:pigscript [SCRIPTNAME]
   #
-  # generate new pig script
+  # Generate new pig script.
   #
   # --skip-udf # Create the pig script without a partnered python udf 
   #
   # Examples:
   #
-  # $ mortar generate:pigscript SCRIPTNAME
+  # $ mortar generate:pigscript my_new_pigscript
   # 
-  # TBD
+  #create  pigscripts/my_new_pigscript.pig
+  #create  udfs/python/my_new_pigscript.py
   # 
   def pigscript
     script_name = shift_argument
@@ -94,16 +105,16 @@ class Mortar::Command::Generate < Mortar::Command::Base
 
   end
 
-  # generate:macro
+  # generate:macro [MACRONAME]
   #
-  # generate new macro
+  # Generate a new pig macro.
   #
   #
   # Examples:
   #
-  # $ mortar generate:macro MACRONAME
+  # $ mortar generate:macro my_new_macro
   # 
-  # TBD
+  #create  macros/my_new_macro.pig
   # 
   def macro
     macro_name = shift_argument

data/lib/mortar/command/help.rb

@@ -23,7 +23,7 @@ require "mortar/command/base"
 #
 class Mortar::Command::Help < Mortar::Command::Base
 
-  PRIMARY_NAMESPACES = []
+  PRIMARY_NAMESPACES = %w( auth clusters generate illustrate jobs pigscripts projects )
 
   # help [COMMAND]
   #

data/lib/mortar/command/illustrate.rb

@@ -17,7 +17,7 @@
 require "mortar/command/base"
 require "mortar/snapshot"
 
-# manage pig scripts
+# sample and show data flowing through a pigscript
 #
 class Mortar::Command::Illustrate < Mortar::Command::Base
   
@@ -32,10 +32,16 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
   #
   # Examples:
   #
-  # $ mortar illustrate
+  # $ mortar illustrate generate_regression_model_coefficients songs_sample
   # 
-  # TBD
+  #Taking code snapshot... done
+  #Sending code snapshot to Mortar... done
+  #Starting illustrate... done
   #
+  #Status: Success  
+  #
+  #Results available at https://hawk.mortardata.com/illustrates/2000ce2f421aa909680000af
+  #Opening web browser to show results... done
   def index
     pigscript_name = shift_argument
     alias_name = shift_argument

data/lib/mortar/command/jobs.rb

@@ -17,7 +17,8 @@
 require "mortar/command/base"
 require "mortar/snapshot"
 require "time"
-# manage pig scripts
+
+# run and view status of pig jobs (run, status)
 #
 class Mortar::Command::Jobs < Mortar::Command::Base
 
@@ -32,10 +33,12 @@ class Mortar::Command::Jobs < Mortar::Command::Base
   #
   # Examples:
   #
-  # $ mortar jobs
+  # $ mortar jobs -l 2
   # 
-  # TBD
-  #
+  #job_id                    script                 status   start_date                           elapsed_time  cluster_size  cluster_id
+  #------------------------  ---------------------  -------  -----------------------------------  ------------  ------------  ------------------------
+  #2000cbbba40a860a6f000000  rollup: mock_expanded  Success  Friday, August 31, 2012, 10:40 AM    2 mins                   3  2000cc3cb0c635b7cbff5aaa
+  #20009deca40a866cd5000000  rollup: mock_expanded  Stopped  Thursday, August 30, 2012,  1:08 PM  < 1 min                  2  2000857ae4b0dd5573da35aa
   def index
     options[:limit] ||= '10'
     options[:skip] ||= '0'
@@ -62,10 +65,20 @@ class Mortar::Command::Jobs < Mortar::Command::Base
   #
   #Examples:
   #
-  # $ mortar jobs:run
+  # $ mortar jobs:run --clustersize 3 geenrate_regression_model_coefficients
   # 
-  # TBD
+  #Taking code snapshot... done
+  #Sending code snapshot to Mortar... done
+  #Requesting job execution... done
+  #job_id: 2000cbbba40a860a6f000000
+  #
+  #Job status can be viewed on the web at:
+  #
+  #https://hawk.mortardata.com/jobs/job_detail?job_id=2000cbbba40a860a6f000000
+  #
+  #Or by running:
   #
+  #mortar jobs:status 2000cbbba40a860a6f000000
   def run
     # arguemnts
     pigscript_name = shift_argument
@@ -124,9 +137,12 @@ class Mortar::Command::Jobs < Mortar::Command::Base
   #
   #Examples:
   #
-  # $ mortar jobs:status 84f3c86f20034ed4bf5e359120a47f5a
+  # $ mortar jobs:status 2000cbbba40a860a6f000000
   #
-  # TBD
+  #=== songhotness: generate_regression_model_coefficients (job_id: 2000cbbba40a860a6f000000)
+  #hadoop jobs complete:    0.00 / 1.00
+  #progress:                0%
+  #status:                  Starting Cluster
   def status
     job_id = shift_argument
     unless job_id
@@ -171,4 +187,29 @@ class Mortar::Command::Jobs < Mortar::Command::Base
     styled_header("#{job_status["project_name"]}: #{job_status["pigscript_name"]} (job_id: #{job_status["job_id"]})")
     styled_hash(job_display_entries)
   end
+
+  # jobs:stop JOB_ID
+  #
+  # Stop a running job.
+  #
+  #Examples:
+  #
+  # $ mortar jobs:stop 2000cbbba40a860a6f000000
+  #
+  #Stopping job 2000cbbba40a860a6f000000
+  def stop
+    job_id = shift_argument
+    unless job_id
+      error("Usage: mortar jobs:stop JOB_ID\nMust specify JOB_ID.")
+    end
+
+    response = api.stop_job(job_id).body  
+
+    #TODO: jkarn - Once all servers have the additional message field we can remove this check.
+    if response['message'].nil?
+      display("Stopping job #{job_id}.")
+    else
+      display(response['message'])
+    end
+  end
 end

data/lib/mortar/command/pigscripts.rb

@@ -16,20 +16,21 @@
 
 require "mortar/command/base"
 
-# manage pig scripts
+# list available pigscripts
 #
 class Mortar::Command::PigScripts < Mortar::Command::Base
     
   # pigscripts
   #
-  # display the available set of pigscripts
+  # Display the available set of pigscripts
   #
   #Examples:
   #
   # $ mortar pigscripts
   # 
-  # hourly_top_searchers
-  # user_engagement
+  #=== pigscripts
+  #hourly_top_searchers
+  #user_engagement
   #
   def index
     # validation

data/lib/mortar/command/projects.rb

@@ -16,7 +16,7 @@
 
 require "mortar/command/base"
 
-## manage projects
+# manage projects (create, clone)
 #
 class Mortar::Command::Projects < Mortar::Command::Base
   
@@ -71,31 +71,39 @@ class Mortar::Command::Projects < Mortar::Command::Base
     end
     
     project_id = nil
-    action("Creating project", {:success => "started"}) do
+    action("Sending request to create project: #{name}") do
       project_id = api.post_project(name).body["project_id"]
     end
     
-    last_project_result = nil
-    while last_project_result.nil? || (! Mortar::API::Projects::STATUSES_COMPLETE.include?(last_project_result["status"]))
-      sleep polling_interval
-      current_project_result = api.get_project(project_id).body
-      if last_project_result.nil? || (last_project_result["status"] != current_project_result["status"])
-        display(" ... #{current_project_result['status']}")
+    project_result = nil
+    project_status = nil
+    display
+    ticking(polling_interval) do |ticks|
+      project_result = api.get_project(project_id).body
+      project_status = project_result.fetch("status_code", project_result["status"])
+      project_description = project_result.fetch("status_description", project_status)
+      is_finished = Mortar::API::Projects::STATUSES_COMPLETE.include?(project_status)
+
+      redisplay("Status: %s %s" % [
+        project_description + (is_finished ? "" : "..."),
+        is_finished ? " " : spinner(ticks)],
+        is_finished) # only display newline on last message
+      if is_finished
+        display
+        break
       end
-      
-      last_project_result = current_project_result
     end
     
-    case last_project_result['status']
+    case project_status
     when Mortar::API::Projects::STATUS_FAILED
-      error("Project creation failed.\nError message: #{last_project_result['error_message']}")
+      error("Project creation failed.\nError message: #{project_result['error_message']}")
     when Mortar::API::Projects::STATUS_ACTIVE
-      git.remote_add("mortar", last_project_result['git_url'])
+      git.remote_add("mortar", project_result['git_url'])
+      display "Your project is ready for use.  Type 'mortar help' to see the commands you can perform on the project.\n\n"
     else
-      raise RuntimeError, "Unknown project status: #{last_project_result['status']} for project_id: #{project_id}"
+      raise RuntimeError, "Unknown project status: #{project_status} for project_id: #{project_id}"
     end
     
-    
   end
   
   # projects:clone PROJECT
@@ -124,5 +132,7 @@ class Mortar::Command::Projects < Mortar::Command::Base
     end
 
     git.clone(project['git_url'], project['name'])
+
+    display "\nYour project is ready for use.  Type 'mortar help' to see the commands you can perform on the project.\n\n"
   end
 end

data/lib/mortar/command/validate.rb

@@ -17,7 +17,7 @@
 require "mortar/command/base"
 require "mortar/snapshot"
 
-# manage pig scripts
+# check script syntax
 #
 class Mortar::Command::Validate < Mortar::Command::Base
   
@@ -35,9 +35,15 @@ class Mortar::Command::Validate < Mortar::Command::Base
   #
   # Examples:
   #
-  # $ mortar validate
+  # $ mortar validate generate_regression_model_coefficients
   # 
-  # TBD
+  #Taking code snapshot... done
+  #Sending code snapshot to Mortar... done
+  #Starting validate... done
+  #
+  #Status: Success  
+  #
+  #Your script is valid.
   #
   def index
     pigscript_name = shift_argument

data/lib/mortar/templates/project/README.md

@@ -1,8 +1,7 @@
-# Weclome to Mortar!
-
-Copy TK
+# Welcome to Mortar!
 
+Mortar is a platform-as-a-service for Hadoop.  With Mortar, you can run jobs on Hadoop using Apache Pig and Python without any special training.  You create your project using the Mortar command-line tool, deploy code using the Git revision control system, and Mortar does the rest.
 
 # Getting Started
 
-Copy TK
+For help getting started with Mortar, check out the [Mortar Help](http://preview.help.mortardata.com/) site.

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/spec/mortar/command/jobs_spec.rb

@@ -357,8 +357,30 @@ status:                  Execution error
 STDOUT
         end
       end
-      
-      
+
+      context("stop") do
+        it "Stops a running job with default message" do
+          job_id = "1234abcd"
+          mock(Mortar::Auth.api).stop_job(job_id) {Excon::Response.new(:body => {"success" => true})}
+
+          stderr, stdout = execute("jobs:stop #{job_id}")
+          stdout.should == <<-STDOUT
+Stopping job #{job_id}.
+STDOUT
+        end
+
+        it "Stops a running job with server message" do
+          job_id = "1234abcd"
+          message = "some awesome message"
+          mock(Mortar::Auth.api).stop_job(job_id) {Excon::Response.new(:body => {"success" => true, "message" => message})}
+
+          stderr, stdout = execute("jobs:stop #{job_id}")
+          stdout.should == "#{message}\n"
+        end
+
+
+      end
+            
     end
   end
 end

data/spec/mortar/command/projects_spec.rb

@@ -31,10 +31,10 @@ module Mortar::Command
     
     project1 = {'name' => "Project1",
                 'status' => Mortar::API::Projects::STATUS_ACTIVE,
-                'git_url' => "git@github.com:mortarcode/Project1"}
+                'git_url' => "git@github.com:mortarcode-dev/Project1"}
     project2 = {'name' => "Project2",
                 'status' => Mortar::API::Projects::STATUS_ACTIVE,
-                'git_url' => "git@github.com:mortarcode/Project2"}
+                'git_url' => "git@github.com:mortarcode-dev/Project2"}
         
     context("index") do
       
@@ -94,10 +94,10 @@ STDERR
          end
       end
       
-      it "create a new project successfully" do
+      it "create a new project successfully - with status" do
         project_id = "1234abcd1234abcd1234"
         project_name = "some_new_project"
-        project_git_url = "git@github.com:mortarcode/#{project_name}"
+        project_git_url = "git@github.com:mortarcode-dev/#{project_name}"
         mock(Mortar::Auth.api).post_project("some_new_project") {Excon::Response.new(:body => {"project_id" => project_id})}
         mock(Mortar::Auth.api).get_project(project_id).returns(Excon::Response.new(:body => {"status" => Mortar::API::Projects::STATUS_PENDING})).ordered
         mock(Mortar::Auth.api).get_project(project_id).returns(Excon::Response.new(:body => {"status" => Mortar::API::Projects::STATUS_CREATING})).ordered
@@ -107,10 +107,24 @@ STDERR
 
         stderr, stdout = execute("projects:create #{project_name}  --polling_interval 0.05", nil, @git)
         stdout.should == <<-STDOUT
-Creating project... started
- ... PENDING
- ... CREATING
- ... ACTIVE
+Sending request to create project: some_new_project... done\n\n\r\e[0KStatus: PENDING... /\r\e[0KStatus: CREATING... -\r\e[0KStatus: ACTIVE  \n\nYour project is ready for use.  Type 'mortar help' to see the commands you can perform on the project.\n
+STDOUT
+      end
+
+      it "create a new project successfully - with status_code and status_description" do
+        project_id = "1234abcd1234abcd1234"
+        project_name = "some_new_project"
+        project_git_url = "git@github.com:mortarcode-dev/#{project_name}"
+        mock(Mortar::Auth.api).post_project("some_new_project") {Excon::Response.new(:body => {"project_id" => project_id})}
+        mock(Mortar::Auth.api).get_project(project_id).returns(Excon::Response.new(:body => {"status_description" => "Pending", "status_code" => Mortar::API::Projects::STATUS_PENDING})).ordered
+        mock(Mortar::Auth.api).get_project(project_id).returns(Excon::Response.new(:body => {"status_description" => "Creating", "status_code" => Mortar::API::Projects::STATUS_CREATING})).ordered
+        mock(Mortar::Auth.api).get_project(project_id).returns(Excon::Response.new(:body => {"status_description" => "Active", "status_code" => Mortar::API::Projects::STATUS_ACTIVE,
+                                                                                             "git_url" => project_git_url})).ordered
+        mock(@git).remote_add("mortar", project_git_url)
+
+        stderr, stdout = execute("projects:create #{project_name}  --polling_interval 0.05", nil, @git)
+        stdout.should == <<-STDOUT
+Sending request to create project: some_new_project... done\n\n\r\e[0KStatus: Pending... /\r\e[0KStatus: Creating... -\r\e[0KStatus: Active  \n\nYour project is ready for use.  Type 'mortar help' to see the commands you can perform on the project.\n
 STDOUT
       end
       

data/spec/mortar/git_spec.rb

@@ -194,7 +194,7 @@ STASH
       
       it "finds a single remote" do
         with_git_initialized_project do |p|
-          remotes = @git.remotes("mortarcode")
+          remotes = @git.remotes("mortarcode-dev")
           remotes["mortar"].should == p.name
         end
       end

data/spec/spec_helper.rb

@@ -49,7 +49,7 @@ def execute(command_line, project=nil, git=nil)
       stub(base).project.returns(project)
     end
   end
-  
+
   # stub git
   if git
     # stub out any operations that affect remote resources
@@ -57,6 +57,7 @@ def execute(command_line, project=nil, git=nil)
     
     any_instance_of(Mortar::Command::Base) do |base|
       stub(base).git.returns(git)
+      stub(base).git_organization.returns("mortarcode-dev")
     end
   end
 
@@ -190,7 +191,7 @@ def with_git_initialized_project(&block)
     remote = "mortar"
     `git add README.txt`
     `git commit -a -m "First commit"`
-    `git remote add #{remote} git@github.com:mortarcode/4dbbd83cae8d5bf8a4000000_#{project.name}.git`
+    `git remote add #{remote} git@github.com:mortarcode-dev/4dbbd83cae8d5bf8a4000000_#{project.name}.git`
     project.remote = remote
     block.call(project)
   end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mortar
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 23
   prerelease: 
   segments: 
   - 0
-  - 1
+  - 2
   - 0
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Mortar Data
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-08-30 00:00:00 Z
+date: 2012-08-31 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mortar-api-ruby
@@ -25,12 +25,12 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 27
+        hash: 23
         segments: 
         - 0
-        - 1
+        - 2
         - 0
-        version: 0.1.0
+        version: 0.2.0
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency