mortar 0.1.0

data/README.md

@@ -0,0 +1,36 @@
+# Mortar CLI
+
+The Mortar CLI lets you run Hadoop jobs on the Mortar service.
+
+# Setup
+
+## Ruby
+
+First, install [rvm](https://rvm.io/rvm/install/).
+
+        curl -kL https://get.rvm.io | bash -s stable
+        
+Afterward, add the line recommended by rvm to your bash initialization file.
+
+Then, switch to the directory where you've cloned mortar.  If you don't have the right version of Ruby installed, you will be prompted to upgrade via rvm.
+
+## Dependencies
+
+Install required gems:
+
+        bundle install
+
+# Running
+
+You can run the command line through bundle:
+
+        bundle exec mortar <command> <args>
+        
+        # example
+        bundle exec mortar help
+
+# Testing
+
+To run the tests, do:
+
+        rake spec

data/bin/mortar

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# resolve bin path, ignoring symlinks
+require "pathname"
+bin_file = Pathname.new(__FILE__).realpath
+
+# add self to libpath
+$:.unshift File.expand_path("../../lib", bin_file)
+
+# start up the CLI
+require "mortar/cli"
+Mortar::CLI.start(*ARGV)

data/lib/mortar.rb

@@ -0,0 +1,23 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mortar/version"
+
+module Mortar
+
+  USER_AGENT = "mortar-gem/#{Mortar::VERSION} (#{RUBY_PLATFORM}) ruby/#{RUBY_VERSION}"
+
+end

data/lib/mortar/auth.rb

@@ -0,0 +1,312 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2012,
+# used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
+#
+
+require "mortar"
+require "mortar/helpers"
+require "mortar/errors"
+
+require "netrc"
+
+class Mortar::Auth
+  class << self
+    include Mortar::Helpers
+
+    attr_accessor :credentials
+    
+    def api
+      @api ||= begin
+        require("mortar-api-ruby")
+        api = Mortar::API.new(default_params.merge(:user => user, :api_key => password))
+
+        def api.request(params, &block)
+          response = super
+          if response.headers.has_key?('X-Mortar-Warning')
+            Mortar::Command.warnings.concat(response.headers['X-Mortar-Warning'].split("\n"))
+          end
+          response
+        end
+
+        api
+      end
+    end
+
+    def login
+      delete_credentials
+      get_credentials
+    end
+    
+    def logout
+      delete_credentials
+    end
+    
+    def check
+      @mortar_user = api.get_user.body
+      #Need to ensure user has a github_username
+      unless @mortar_user.fetch("user_github_username", nil)
+        begin
+          ask_for_and_save_github_username
+        rescue Mortar::CLI::Errors::InvalidGithubUsername => e
+          retry if retry_set_github_username?
+          raise e
+        end
+      end
+    end
+    
+    def default_host
+      "mortardata.com"
+    end
+    
+    def host
+      ENV['MORTAR_HOST'] || default_host
+    end
+  
+    def reauthorize
+      @credentials = ask_for_and_save_credentials
+    end
+
+    def user    # :nodoc:
+      get_credentials[0]
+    end
+
+    def password    # :nodoc:
+      get_credentials[1]
+    end
+
+    def api_key(user = get_credentials[0], password = get_credentials[1])
+      require("mortar-api-ruby")
+      api = Mortar::API.new(default_params)
+      api.post_login(user, password).body["api_key"]
+    end
+
+    def get_credentials    # :nodoc:
+      @credentials ||= (read_credentials || ask_for_and_save_credentials)
+    end
+
+    def delete_credentials
+      if netrc
+        netrc.delete("api.#{host}")
+        netrc.save
+      end
+      @api, @client, @credentials = nil, nil
+    end
+
+    def netrc_path
+      default = Netrc.default_path
+      encrypted = default + ".gpg"
+      if File.exists?(encrypted)
+        encrypted
+      else
+        default
+      end
+    end
+
+    def netrc   # :nodoc:
+      @netrc ||= begin
+        File.exists?(netrc_path) && Netrc.read(netrc_path)
+      rescue => error
+        if error.message =~ /^Permission bits for/
+          perm = File.stat(netrc_path).mode & 0777
+          abort("Permissions #{perm} for '#{netrc_path}' are too open. You should run `chmod 0600 #{netrc_path}` so that your credentials are NOT accessible by others.")
+        else
+          raise error
+        end
+      end
+    end
+
+    def read_credentials
+      if ENV['MORTAR_API_KEY']
+        ['', ENV['MORTAR_API_KEY']]
+      else
+        if netrc
+          netrc["api.#{host}"]
+        end
+      end
+    end
+
+    def write_credentials
+      FileUtils.mkdir_p(File.dirname(netrc_path))
+      FileUtils.touch(netrc_path)
+      unless running_on_windows?
+        FileUtils.chmod(0600, netrc_path)
+      end
+      netrc["api.#{host}"] = self.credentials
+      netrc.save
+    end
+
+    def echo_off
+      with_tty do
+        system "stty -echo"
+      end
+    end
+
+    def echo_on
+      with_tty do
+        system "stty echo"
+      end
+    end
+
+    def ask_for_credentials
+      puts
+      puts "Enter your Mortar credentials."
+
+      print "Email: "
+      user = ask
+
+      print "Password (typing will be hidden): "
+      password = running_on_windows? ? ask_for_password_on_windows : ask_for_password
+
+      [user, api_key(user, password)]
+    end
+
+    def ask_for_github_username
+      puts
+      puts "Please enter your github username (not email address)."
+
+      print "Github Username: "
+      github_username = ask
+      github_username
+    end
+
+    def ask_for_password_on_windows
+      require "Win32API"
+      char = nil
+      password = ''
+
+      while char = Win32API.new("crtdll", "_getch", [ ], "L").Call do
+        break if char == 10 || char == 13 # received carriage return or newline
+        if char == 127 || char == 8 # backspace and delete
+          password.slice!(-1, 1)
+        else
+          # windows might throw a -1 at us so make sure to handle RangeError
+          (password << char.chr) rescue RangeError
+        end
+      end
+      puts
+      return password
+    end
+
+    def ask_for_password
+      echo_off
+      password = ask
+      puts
+      echo_on
+      return password
+    end
+
+    def ask_for_and_save_credentials
+      require("mortar-api-ruby") # for the errors
+      begin
+        @credentials = ask_for_credentials
+        write_credentials
+        check
+      rescue Mortar::API::Errors::NotFound, Mortar::API::Errors::Unauthorized => e
+        delete_credentials
+        display "Authentication failed."
+        retry if retry_login?
+        exit 1
+      rescue Mortar::CLI::Errors::InvalidGithubUsername => e
+        #Too many failures at setting github username
+        display "Authentication failed."
+        delete_credentials
+        exit 1
+      rescue Exception => e
+        delete_credentials
+        raise e
+      end
+      # TODO: ensure that keys exist
+      #check_for_associated_ssh_key unless Mortar::Command.current_command == "keys:add"
+      @credentials
+    end
+
+    def ask_for_and_save_github_username
+      require ("mortar-api-ruby")
+      begin
+        @github_username = ask_for_github_username
+        save_github_username
+      end
+    end
+
+    def save_github_username
+      task_id = api.update_user(@mortar_user['user_id'], {'user_github_username' => @github_username}).body['task_id']
+
+      task_result = nil
+      ticking(polling_interval) do |ticks|
+        task_result = api.get_task(task_id).body
+        is_finished =
+          Mortar::API::Task::STATUSES_COMPLETE.include?(task_result["status_code"])
+        
+        redisplay("Setting github username: %s" % 
+          [is_finished ? " Done!" : spinner(ticks)],
+          is_finished) # only display newline on last message
+        if is_finished
+          display
+          break
+        end
+      end
+
+      case task_result['status_code']
+      when Mortar::API::Task::STATUS_FAILURE
+        error_message = "Setting github username failed with #{task_result['error_type'] || 'error'}"
+        error_message += ":\n\n#{task_result['error_message']}\n\n"
+        output_with_bang error_message
+        raise Mortar::CLI::Errors::InvalidGithubUsername.new
+      when Mortar::API::Task::STATUS_SUCCESS
+        display "Successfully set github username." 
+      else
+        #Raise error so .netrc file is wiped out.
+        raise RuntimeError, "Unknown task status: #{task_result['status_code']}"
+      end
+    end
+
+    
+    def retry_login?
+      @login_attempts ||= 0
+      @login_attempts += 1
+      @login_attempts < 3
+    end
+
+    def retry_set_github_username?
+      @set_github_username_attempts ||= 0
+      @set_github_username_attempts += 1
+      @set_github_username_attempts < 3
+    end
+
+    def polling_interval
+      (2.0).to_f
+    end
+    
+
+    protected
+
+    def default_params
+      full_host  = (host =~ /^http/) ? host : "https://api.#{host}"
+      verify_ssl = ENV['MORTAR_SSL_VERIFY'] != 'disable' && full_host =~ %r|^https://api.mortardata.com|
+      uri = URI(full_host)
+      {
+        :headers          => {
+          'User-Agent'    => Mortar::USER_AGENT
+        },
+        :host             => uri.host,
+        :port             => uri.port.to_s,
+        :scheme           => uri.scheme,
+        :ssl_verify_peer  => verify_ssl
+      }
+    end
+  end
+end

data/lib/mortar/cli.rb

@@ -0,0 +1,54 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2012,
+# used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
+#
+
+require "mortar"
+require "mortar/command"
+require "mortar/helpers"
+
+# workaround for rescue/reraise to define errors in command.rb failing in 1.8.6
+#if RUBY_VERSION =~ /^1.8.6/
+#  require('mortar-api')
+#  require('rest_client')
+#end
+
+class Mortar::CLI
+
+  extend Mortar::Helpers
+
+  def self.start(*args)
+    begin
+      if $stdin.isatty
+        $stdin.sync = true
+      end
+      if $stdout.isatty
+        $stdout.sync = true
+      end
+      command = args.shift.strip rescue "help"
+      Mortar::Command.load
+      Mortar::Command.run(command, args)
+    rescue Interrupt
+      `stty icanon echo`
+      error("Command cancelled.")
+    rescue => error
+      styled_error(error)
+      exit(1)
+    end
+  end
+
+end

data/lib/mortar/command.rb

@@ -0,0 +1,267 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2012,
+# used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
+#
+
+require 'rexml/document'
+require 'mortar/helpers'
+require 'mortar/project'
+require 'mortar/version'
+require 'mortar/api'
+require "optparse"
+
+
+module Mortar
+  module Command
+    class CommandFailed  < RuntimeError; end
+
+    extend Mortar::Helpers
+
+    def self.load
+      Dir[File.join(File.dirname(__FILE__), "command", "*.rb")].each do |file|
+        require file
+      end
+    end
+
+    def self.commands
+      @@commands ||= {}
+    end
+
+    def self.command_aliases
+      @@command_aliases ||= {}
+    end
+
+    def self.files
+      @@files ||= Hash.new {|hash,key| hash[key] = File.readlines(key).map {|line| line.strip}}
+    end
+
+    def self.namespaces
+      @@namespaces ||= {}
+    end
+
+    def self.register_command(command)
+      commands[command[:command]] = command
+    end
+
+    def self.register_namespace(namespace)
+      namespaces[namespace[:name]] = namespace
+    end
+
+    def self.current_command
+      @current_command
+    end
+
+    def self.current_command=(new_current_command)
+      @current_command = new_current_command
+    end
+
+    def self.current_args
+      @current_args
+    end
+
+    def self.current_options
+      @current_options ||= {}
+    end
+
+    def self.global_options
+      @global_options ||= []
+    end
+
+    def self.invalid_arguments
+      @invalid_arguments
+    end
+
+    def self.shift_argument
+      # dup argument to get a non-frozen string
+      @invalid_arguments.shift.dup rescue nil
+    end
+
+    def self.validate_arguments!
+      unless invalid_arguments.empty?
+        arguments = invalid_arguments.map {|arg| "\"#{arg}\""}
+        if arguments.length == 1
+          message = "Invalid argument: #{arguments.first}"
+        elsif arguments.length > 1
+          message = "Invalid arguments: "
+          message << arguments[0...-1].join(", ")
+          message << " and "
+          message << arguments[-1]
+        end
+        $stderr.puts(format_with_bang(message))
+        run(current_command, ["--help"])
+        exit(1)
+      end
+    end
+
+    def self.warnings
+      @warnings ||= []
+    end
+
+    def self.display_warnings
+      unless warnings.empty?
+        $stderr.puts(warnings.map {|warning| " !    #{warning}"}.join("\n"))
+      end
+    end
+
+    def self.global_option(name, *args, &blk)
+      global_options << { :name => name, :args => args, :proc => blk }
+    end
+
+    global_option :help,    "--help", "-h"
+    global_option :remote,  "--remote REMOTE"
+    global_option :polling_interval, "--polling_interval SECONDS", "-p"
+
+    def self.prepare_run(cmd, args=[])
+      command = parse(cmd)
+
+      if args.include?('-h') || args.include?('--help')
+        args.unshift(cmd) unless cmd =~ /^-.*/
+        cmd = 'help'
+        command = parse('help')
+      end
+
+      unless command
+        if %w( -v --version ).include?(cmd)
+          cmd = 'version'
+          command = parse(cmd)
+        else
+          error([
+            "`#{cmd}` is not a mortar command.",
+            suggestion(cmd, commands.keys + command_aliases.keys),
+            "See `mortar help` for a list of available commands."
+          ].compact.join("\n"))
+        end
+      end
+
+      @current_command = cmd
+
+      opts = {}
+      invalid_options = []
+
+      parser = OptionParser.new do |parser|
+        # overwrite OptionParsers Officious['version'] to avoid conflicts
+        # see: https://github.com/ruby/ruby/blob/trunk/lib/optparse.rb#L814
+        parser.on("--version") do |value|
+          invalid_options << "--version"
+        end
+        global_options.each do |global_option|
+          parser.on(*global_option[:args]) do |value|
+            global_option[:proc].call(value) if global_option[:proc]
+            opts[global_option[:name]] = value
+          end
+        end
+        command[:options].each do |name, option|
+          parser.on("-#{option[:short]}", "--#{option[:long]}", option[:desc]) do |value|
+            opt_name_sym = name.gsub("-", "_").to_sym
+            if opts[opt_name_sym]
+              # convert multiple instances of an option to an array
+              unless opts[opt_name_sym].is_a?(Array)
+                opts[opt_name_sym] = [opts[opt_name_sym]]
+              end
+              opts[opt_name_sym] << value
+            else
+              opts[opt_name_sym] = value
+            end
+          end
+        end
+      end
+
+      begin
+        parser.order!(args) do |nonopt|
+          invalid_options << nonopt
+        end
+      rescue OptionParser::InvalidOption => ex
+        invalid_options << ex.args.first
+        retry
+      end
+
+      args.concat(invalid_options)
+
+      @current_args = args
+      @current_options = opts
+      @invalid_arguments = invalid_options
+
+      [ command[:klass].new(args.dup, opts.dup), command[:method] ]
+    end
+
+    def self.run(cmd, arguments=[])
+      begin
+        object, method = prepare_run(cmd, arguments.dup)
+        object.send(method)
+      rescue Interrupt, StandardError, SystemExit => error
+        # load likely error classes, as they may not be loaded yet due to defered loads
+        require 'mortar-api-ruby'
+        raise(error)
+      end
+    rescue Mortar::API::Errors::Unauthorized
+      puts "Authentication failure"
+      unless ENV['MORTAR_API_KEY']
+        run "login"
+        retry
+      end
+    rescue Mortar::API::Errors::NotFound => e
+      error extract_error(e.response.body) {
+        e.response.body =~ /^([\w\s]+ not found).?$/ ? $1 : e.message # "Resource not found"
+      }
+    rescue Mortar::Project::ProjectError => e
+      error e.message
+    rescue Mortar::API::Errors::Timeout
+      error "API request timed out. Please try again, or contact support@mortardata.com if this issue persists."
+    rescue Mortar::API::Errors::ErrorWithResponse => e
+      error extract_error(e.response.body)
+    rescue CommandFailed => e
+      error e.message
+    rescue OptionParser::ParseError
+      commands[cmd] ? run("help", [cmd]) : run("help")
+    ensure
+      display_warnings
+    end
+
+    def self.parse(cmd)
+      commands[cmd] || commands[command_aliases[cmd]]
+    end
+
+    def self.extract_error(body, options={})
+      default_error = block_given? ? yield : "Internal server error."
+      parse_error_xml(body) || parse_error_json(body) || parse_error_plain(body) || default_error
+    end
+
+    def self.parse_error_xml(body)
+      xml_errors = REXML::Document.new(body).elements.to_a("//errors/error")
+      msg = xml_errors.map { |a| a.text }.join(" / ")
+      return msg unless msg.empty?
+    rescue Exception
+    end
+
+    def self.parse_error_json(body)
+      json = json_decode(body.to_s) rescue false
+      case json
+      when Array
+        json.first.last # message like [['base', 'message']]
+      when Hash
+        json['error']   # message like {'error' => 'message'}
+      else
+        nil
+      end
+    end
+
+    def self.parse_error_plain(body)
+      return unless body.respond_to?(:headers) && body.headers[:content_type].to_s.include?("text/plain")
+      body.to_s
+    end
+  end
+end