mortar 0.1.0

data/lib/mortar/command/jobs.rb

@@ -0,0 +1,174 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mortar/command/base"
+require "mortar/snapshot"
+require "time"
+# manage pig scripts
+#
+class Mortar::Command::Jobs < Mortar::Command::Base
+
+  include Mortar::Snapshot
+
+  # jobs
+  #
+  # Show recent and running jobs.
+  #
+  # -l, --limit LIMITJOBS # Limit the number of jobs returned (defaults to 10)
+  # -s, --skip SKIPJOBS   # Skip a certain amount of jobs (defaults to 0)
+  #
+  # Examples:
+  #
+  # $ mortar jobs
+  # 
+  # TBD
+  #
+  def index
+    options[:limit] ||= '10'
+    options[:skip] ||= '0'
+    jobs = api.get_jobs(options[:skip], options[:limit]).body['jobs']
+    jobs.each do |job|
+      if job['start_timestamp']
+        job['start_timestamp'] = Time.parse(job['start_timestamp']).strftime('%A, %B %e, %Y, %l:%M %p')
+      end
+    end
+    headers = [ 'job_id', 'script' , 'status' , 'start_date' , 'elapsed_time' , 'cluster_size' , 'cluster_id']
+    columns = [ 'job_id', 'display_name', 'status_description', 'start_timestamp', 'duration', 'cluster_size', 'cluster_id']
+    display_table(jobs, columns, headers)
+  end
+    
+  # jobs:run PIGSCRIPT
+  #
+  # Run a job on a Mortar Hadoop cluster.
+  #
+  # -c, --clusterid CLUSTERID   # Run job on an existing cluster with ID of CLUSTERID (optional)
+  # -s, --clustersize NUMNODES  # Run job on a new cluster, with NUMNODES nodes (optional; must be >= 2 if provided)
+  # -1, --singlejobcluster      # Stop the cluster after job completes.  (Default: false—-cluster can be used for other jobs, and will shut down after 1 hour of inactivity)
+  # -p, --parameter NAME=VALUE  # Set a pig parameter value in your script.
+  # -f, --param-file PARAMFILE  # Load pig parameter values from a file.
+  #
+  #Examples:
+  #
+  # $ mortar jobs:run
+  # 
+  # TBD
+  #
+  def run
+    # arguemnts
+    pigscript_name = shift_argument
+    unless pigscript_name
+      error("Usage: mortar jobs:run PIGSCRIPT\nMust specify PIGSCRIPT.")
+    end
+    validate_arguments!
+
+    unless options[:clusterid] || options[:clustersize]
+      error("Please provide either the --clustersize option to run job on a new cluster, or --clusterid to run on an existing one.")
+    end
+      
+    if options[:clusterid]
+      [:clustersize, :singlejobcluster].each do |opt|
+        unless options[opt].nil?
+          error("Option #{opt.to_s} cannot be set when running a job on an existing cluster (with --clusterid option)")
+        end
+      end
+    end
+
+
+        
+    validate_git_based_project!
+    pigscript = validate_pigscript!(pigscript_name)
+    git_ref = create_and_push_snapshot_branch(git, project)
+    
+    # post job to API
+    response = action("Requesting job execution") do
+      if options[:clustersize]
+        cluster_size = options[:clustersize].to_i
+        keepalive = ! options[:singlejobcluster]
+        api.post_job_new_cluster(project.name, pigscript.name, git_ref, cluster_size, 
+          :parameters => pig_parameters,
+          :keepalive => keepalive).body
+      else
+        cluster_id = options[:clusterid]
+        api.post_job_existing_cluster(project.name, pigscript.name, git_ref, cluster_id,
+          :parameters => pig_parameters).body
+      end
+    end
+
+    display("job_id: #{response['job_id']}")
+    display
+    display("Job status can be viewed on the web at:\n\n #{response['web_job_url']}")
+    display
+    display("Or by running:\n\n  mortar jobs:status #{response['job_id']}")
+    display
+  end
+  
+  alias_command "run", "jobs:run"
+  
+
+  # jobs:status JOB_ID
+  #
+  # Check the status of a job.
+  #
+  #Examples:
+  #
+  # $ mortar jobs:status 84f3c86f20034ed4bf5e359120a47f5a
+  #
+  # TBD
+  def status
+    job_id = shift_argument
+    unless job_id
+      error("Usage: mortar jobs:status JOB_ID\nMust specify JOB_ID.")
+    end
+    
+    job_status = api.get_job(job_id).body
+
+    job_display_entries = {
+      "status" => job_status["status_description"],
+      "progress" => "#{job_status["progress"]}%",
+      "cluster_id" => job_status["cluster_id"],
+      "job submitted at" => job_status["start_timestamp"],
+      "job began running at" => job_status["running_timestamp"],
+      "job finished at" => job_status["stop_timestamp"],
+      "job running for" => job_status["duration"],
+      "job run with parameters" => job_status["parameters"],
+      "error" => job_status["error"]
+    }
+    
+    unless job_status["error"].nil? || job_status["error"]["message"].nil?
+      error_context = get_error_message_context(job_status["error"]["message"])
+      unless error_context == ""
+        job_status["error"]["help"] = error_context
+      end
+    end
+    
+    if job_status["num_hadoop_jobs"] && job_status["num_hadoop_jobs_succeeded"]
+      job_display_entries["hadoop jobs complete"] = 
+        '%0.2f / %0.2f' % [job_status["num_hadoop_jobs_succeeded"], job_status["num_hadoop_jobs"]]
+    end
+    
+    if job_status["outputs"] && job_status["outputs"].length > 0
+      job_display_entries["outputs"] = Hash[job_status["outputs"].select{|o| o["alias"]}.collect do |output|
+        output_hash = {}
+        output_hash["location"] = output["location"] if output["location"]
+        output_hash["records"] = output["records"] if output["records"]
+        [output['alias'], output_hash]
+      end]
+    end
+    
+    styled_header("#{job_status["project_name"]}: #{job_status["pigscript_name"]} (job_id: #{job_status["job_id"]})")
+    styled_hash(job_display_entries)
+  end
+end

data/lib/mortar/command/pigscripts.rb

@@ -0,0 +1,45 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mortar/command/base"
+
+# manage pig scripts
+#
+class Mortar::Command::PigScripts < Mortar::Command::Base
+    
+  # pigscripts
+  #
+  # display the available set of pigscripts
+  #
+  #Examples:
+  #
+  # $ mortar pigscripts
+  # 
+  # hourly_top_searchers
+  # user_engagement
+  #
+  def index
+    # validation
+    validate_arguments!
+    if project.pigscripts.any?
+      styled_header("pigscripts")
+      styled_array(project.pigscripts.keys)
+    else
+      display("You have no pigscripts.")
+    end
+  end
+
+end

data/lib/mortar/command/projects.rb

@@ -0,0 +1,128 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mortar/command/base"
+
+## manage projects
+#
+class Mortar::Command::Projects < Mortar::Command::Base
+  
+  # projects
+  #
+  # Display the available set of projects
+  #
+  #Examples:
+  #
+  # $ mortar projects
+  # 
+  #=== projects
+  #demo
+  #rollup
+  #
+  def index
+    validate_arguments!
+    projects = api.get_projects().body["projects"]
+    if projects.any?
+      styled_header("projects")
+      styled_array(projects.collect{ |x| x["name"] })
+    else
+      display("You have no projects.")
+    end
+  end
+  
+  # projects:create PROJECT
+  #
+  # create a mortar project for the current directory with the name PROJECT
+  #
+  #Example:
+  #
+  # $ mortar projects:create my_new_project
+  #
+  def create
+    name = shift_argument
+    unless name
+      error("Usage: mortar projects:create PROJECT\nMust specify PROJECT.")
+    end
+    validate_arguments!
+    
+    unless git.has_dot_git?
+      error("Can only create a mortar project for an existing git project.  Please run:\n\ngit init\ngit add .\ngit commit -a -m \"first commit\"\n\nto initialize your project in git.")
+    end
+    
+    unless git.remotes(git_organization).empty?
+      begin
+        error("Currently in project: #{project.name}.  You can not create a new project inside of an existing mortar project.")
+      rescue Mortar::Command::CommandFailed => cf
+        error("Currently in an existing Mortar project.  You can not create a new project inside of an existing mortar project.")
+      end
+    end
+    
+    project_id = nil
+    action("Creating project", {:success => "started"}) do
+      project_id = api.post_project(name).body["project_id"]
+    end
+    
+    last_project_result = nil
+    while last_project_result.nil? || (! Mortar::API::Projects::STATUSES_COMPLETE.include?(last_project_result["status"]))
+      sleep polling_interval
+      current_project_result = api.get_project(project_id).body
+      if last_project_result.nil? || (last_project_result["status"] != current_project_result["status"])
+        display(" ... #{current_project_result['status']}")
+      end
+      
+      last_project_result = current_project_result
+    end
+    
+    case last_project_result['status']
+    when Mortar::API::Projects::STATUS_FAILED
+      error("Project creation failed.\nError message: #{last_project_result['error_message']}")
+    when Mortar::API::Projects::STATUS_ACTIVE
+      git.remote_add("mortar", last_project_result['git_url'])
+    else
+      raise RuntimeError, "Unknown project status: #{last_project_result['status']} for project_id: #{project_id}"
+    end
+    
+    
+  end
+  
+  # projects:clone PROJECT
+  #
+  # clone the mortar project PROJECT into the current directory.
+  #
+  #Example:
+  #
+  # $ mortar projects:clone my_new_project
+  #
+  def clone
+    name = shift_argument
+    unless name
+      error("Usage: mortar projects:clone PROJECT\nMust specify PROJECT.")
+    end
+    validate_arguments!
+    projects = api.get_projects().body["projects"]
+    project = projects.find{|p| p['name'] == name}
+    unless project
+      error("No project named: #{name} exists.  Your valid projects are:\n#{projects.collect{ |x| x["name"]}.join("\n")}")
+    end
+
+    project_dir = File.join(Dir.pwd, project['name'])
+    unless !File.exists?(project_dir)
+      error("Can't clone project: #{project['name']} since directory with that name already exists.")
+    end
+
+    git.clone(project['git_url'], project['name'])
+  end
+end

data/lib/mortar/command/validate.rb

@@ -0,0 +1,94 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mortar/command/base"
+require "mortar/snapshot"
+
+# manage pig scripts
+#
+class Mortar::Command::Validate < Mortar::Command::Base
+  
+  include Mortar::Snapshot
+    
+  # validate [PIGSCRIPT]
+  #
+  # Validate a pig script.  Checks script for problems with:
+  #    * Pig syntax
+  #    * Python syntax
+  #    * S3 data access
+  #
+  # -p, --parameter NAME=VALUE  # Set a pig parameter value in your script.
+  # -f, --param-file PARAMFILE  # Load pig parameter values from a file.
+  #
+  # Examples:
+  #
+  # $ mortar validate
+  # 
+  # TBD
+  #
+  def index
+    pigscript_name = shift_argument
+    unless pigscript_name
+      error("Usage: mortar validate PIGSCRIPT\nMust specify PIGSCRIPT.")
+    end
+    validate_arguments!
+    validate_git_based_project!
+    pigscript = validate_pigscript!(pigscript_name)
+    git_ref = create_and_push_snapshot_branch(git, project)
+    
+    validate_id = nil
+    action("Starting validate") do
+      validate_id = api.post_validate(project.name, pigscript.name, git_ref, :parameters => pig_parameters).body["validate_id"]
+    end
+        
+    validate_result = nil
+    display
+    ticking(polling_interval) do |ticks|
+      validate_result = api.get_validate(validate_id).body
+      is_finished =
+        Mortar::API::Validate::STATUSES_COMPLETE.include?(validate_result["status_code"])
+        
+      redisplay("Status: %s %s" % [
+        validate_result['status_description']  + (is_finished ? "" : "..."),
+        is_finished ? " " : spinner(ticks)],
+        is_finished) # only display newline on last message
+      if is_finished
+        display
+        break
+      end
+    end
+    
+    case validate_result['status_code']
+    when Mortar::API::Validate::STATUS_FAILURE
+      error_message = "Validate failed with #{validate_result['error_type'] || 'error'}"
+      if line_number = validate_result["line_number"]
+        error_message += " at Line #{line_number}"
+        if column_number = validate_result["column_number"]
+          error_message += ", Column #{column_number}"
+        end
+      end
+      error_context = get_error_message_context(validate_result['error_message'])
+      error_message += ":\n\n#{validate_result['error_message']}\n\n#{error_context}"
+      error(error_message)
+    when Mortar::API::Validate::STATUS_KILLED
+      error("Validate killed by user.")
+    when Mortar::API::Validate::STATUS_SUCCESS
+      display("Your script is valid.")
+    else
+      raise RuntimeError, "Unknown validate status: #{validate_result['status']} for validate_id: #{validate_id}"
+    end
+  end
+end

data/lib/mortar/command/version.rb

@@ -0,0 +1,42 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2012,
+# used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
+#
+
+require "mortar/command/base"
+require "mortar/version"
+
+# display version
+#
+class Mortar::Command::Version < Mortar::Command::Base
+
+  # version
+  #
+  # show mortar client version
+  #
+  #Example:
+  #
+  # $ mortar version
+  # mortar/1.2.3 (x86_64-darwin11.4.2) ruby/1.9.3
+  #
+  def index
+    validate_arguments!
+
+    display(Mortar::USER_AGENT)
+  end
+
+end