hodor 1.0.2

data/lib/hodor/api/oozie/session.rb

@@ -0,0 +1,170 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'singleton'
+
+module Hodor::Oozie
+  class Session
+    include Singleton
+
+    attr_accessor :mode, :verbose, :filter, :len, :offset
+    attr_reader :last_query
+
+    def env
+      Hodor::Environment.instance
+    end
+
+    def logger
+      env.logger
+    end
+
+    def initialize
+      @len = env.prefs[:default_list_length] || 30
+      @offset = 0
+    end
+
+    def rest_call(api)
+      num_retries = 0
+      begin
+        url = "#{env[:oozie_url]}#{api}".gsub(/oozie\/\//,'oozie/')
+        @last_query = url
+        #puts "REST CALL: #{url}"
+        uri = URI.parse(url)
+
+        http = Net::HTTP.new(uri.host, uri.port)
+
+        http.read_timeout = 10
+        http.open_timeout = 10
+
+        data = http.start() {|http|
+          http.get(uri.request_uri).body
+        }
+      rescue Net::OpenTimeout => ex
+        logger.error "Network connection timed out! Make sure you are connected to the Internet or VPN. Retrying..."
+        if num_retries <= 4
+          num_retries += 1
+          retry
+        else
+          nil
+        end
+      end
+    end
+
+    def search_jobs(*args)
+      json = rest_call("/v2/jobs?#{args.map { |v| v.nil? || v.size == 0 ? nil : v }.compact.join('&')}")
+      @root_query = @last_query
+      json
+    end
+
+    def get_job_state(job_id, *args)
+      rest_call("/v1/job/#{job_id}?#{args.map { |v| v.nil? || v.size == 0 ? nil : v }.compact.join('&')}")
+    end
+
+    def refresh_index(children, current_id, parent_id)
+      if children
+        children.each_with_index { |c, i|
+          c.set_index(i)
+        }
+        child_ids = children.map { |c| c.skip_to || c.id }
+      else
+        child_ids = nil
+      end
+      index_overwrite = { children: child_ids,
+                current_id: current_id,
+                parent_id: parent_id,
+                root_query: @root_query }
+      File.open(cache_file, 'wb') {|f| f.write(::Marshal.dump(index_overwrite)) }
+      children
+    end
+
+    def index
+      if @index.nil?
+        @index = load_index
+      end
+      @index
+    end
+
+    def child_id(child_index)
+      children = index[:children]
+      if children
+        index_size = index[:children].length
+        if child_index < index_size
+          cid = index[:children][child_index]
+          cid
+        else
+          raise "No child with index '#{child_index}' was found"
+        end
+      end
+    end
+
+    def current_id
+      index[:current_id]
+    end
+
+    def parent_id
+      index[:parent_id]
+    end
+
+    def root_query
+      @root_query || index[:root_query]
+    end
+
+    def cache_file
+      if @cache_file.nil?
+        if env[:display_job_query_mode]
+          default_id = 'default'
+        else
+          default_id = `ps -p #{Process.pid} -o ppid=`.strip
+        end
+        index_id = ENV['HODOR_INDEX_ID'] || default_id
+        @cache_file = "/tmp/hodor-#{index_id}.index"
+      end
+      @cache_file
+    end
+
+    def load_index
+      index_read = {}
+      if File.exists? cache_file
+        File.open(cache_file, 'rb') {|f| index_read = ::Marshal.load(f) } 
+        @root_query ||= index_read[:root_query] if index_read.has_key?(:root_query)
+      else
+        index_read = { children: nil,
+                current_id: nil,
+                parent_id: nil,
+                root_query: nil }
+      end
+      index_read || { children: nil,
+                      current_id: nil,
+                      parent_id: nil,
+                      root_query: nil }
+    rescue => ex
+      raise "Failed to load Hodor cache file. #{ex.message}"
+    end
+
+    def pwj
+      { current_id: session.current_id,
+        parent_id: session.parent_id,
+        root_query: session.root_query }
+    end
+
+    def job_relative(movement, request = nil)
+      case movement
+      when :root;
+        nil
+      when :up; 
+        parent_id
+      when :down; 
+        child_id(request.to_i)
+      when :none; 
+        current_id
+      when :jump;
+        request
+      end
+    end
+
+    def make_current(job)
+      refresh_index(job.children, job.id, job.parent_id) if job
+      job
+    end
+  end
+end

data/lib/hodor/api/oozie/workflow.rb

@@ -0,0 +1,58 @@
+require_relative 'job'
+
+module Hodor::Oozie
+
+  class Workflow < Job
+
+    attr_reader :json, :app_path, :acl, :status, :created_at, :conf, :last_mod_time, :run,
+      :end_time, :external_id, :name, :app_name, :id, :start_time, :materialization_id, :parent_id,
+      :materialization, :to_string, :group, :console_url, :user
+
+    class << self
+      def default_columns
+        [:index, :id, :status, :created_at, :last_mod_time, :app_name]
+      end
+    end
+
+    def initialize(json)
+      super()
+      @json = json
+
+      @app_path = json["appPath"]
+      @acl = json["acl"]
+      @status = json["status"]
+      @created_at = parse_time json["createdTime"]
+      @conf = json["conf"]
+      @last_mod_time = parse_time json["lastModTime"]
+      @run = json["run"]
+      @end_time = parse_time json["endTime"]
+      @external_id = json["externalId"]
+      @name = @app_name = json["appName"]
+      @id = json["id"]
+      @start_time = parse_time json["startTime"]
+      @materialization_id = json["parentId"]
+      ati = @materializeation_id.nil? ? nil : @materialization_id.index('@')
+      if ati && ati > 0
+        @parent_id = @materialization_id[0..ati-1]
+      else
+        @parent_id = @materialization_id
+        @materialization = nil
+      end
+
+      @to_string = json["toString"]
+      @group = json["group"]
+      @console_url = json["consoleUrl"]
+      @user = json["user"]
+    end
+
+    def expand
+      # expand immediate children
+      @actions = json["actions"].map do |item|
+        require_relative 'action'
+        Hodor::Oozie::Action.new(item)
+      end.compact
+    end
+
+  end
+
+end

data/lib/hodor/cli.rb

@@ -0,0 +1,146 @@
+require "thor/runner"
+require_relative 'environment'
+
+module Hodor
+
+  module Cli
+    class Usage < StandardError
+    end
+
+    class CommandNotFound < StandardError
+    end
+
+    class AbnormalExitStatus  < StandardError
+      attr_reader :exit_status
+      def initialize(exit_status, error_lines)
+        @exit_status = exit_status
+        super error_lines
+      end
+    end
+
+    class Runner < ::Thor::Runner
+
+      def help(meth = nil)
+        if meth && !self.respond_to?(meth)
+          super
+        else
+          overview = %Q[Hodor is an object-oriented scripting toolkit and Ruby-based API that automates and simplifies the way you
+          specify, deploy, test, inspect and administer your hadoop cluster and Oozie workflows. Hodor commands follow
+          the convention of:
+
+             $ hodor [namespace]:[command] [arguments] [options]
+
+          To get more information about the namespaces and commands available in Hodor, run:
+
+             $ hodor -T
+
+          WARNING! Hodor must be run via 'bundle exec'. For example:
+
+             $ bundle exec hodor -T
+
+          Note: examples shown in help pages don't show the 'bundle exec' prefix because they assume you have the following alias in place:
+
+             $ alias hodor='bundle exec hodor'
+          ].unindent(10)
+          say overview
+        end
+      end
+
+      desc "list [SEARCH]", "List the available thor commands (--substring means .*SEARCH)"
+      method_options :substring => :boolean, :group => :string, :all => :boolean, :debug => :boolean
+      def list(search = "")
+        overview = %Q[
+        Hodor's Namespaces & Commands
+        ======================================================================================================
+        Hodor divides its command set into the namespaces shown below (e.g. 'oozie', 'hdfs', 'master' etc.) Each
+        namespace contains a set of commands that support the overall purpose of its parent namespace. For example, the
+        hdfs namespace includes commands to list, put and get files to/from a remote HDFS volume. The following table shows
+        all the namespaces Hodor supports, along with a short description of the commands that fall within each namespace.
+
+        ].unindent(8)
+
+        say overview
+        super
+
+        more_help = %Q[Getting More Help:
+        ------------------
+        Each Hodor namespace offers full help, including an overview of the namespace itself, references to "topic
+        pages" that explain core concepts implemented by the namespace and detailed help for each command that falls
+        within the namespace. To access help for a Hodor namespace, run hodor passing <namespace> as the sole
+        argument. For example, to see help for Hodor's Oozie namespace, run:
+
+          $ hodor oozie
+          $ hodor help oozie   # alternate, works the same
+
+        Furthermore, to see detailed help for the oozie:display_job command, run:
+
+          $ hodor help oozie:display_job
+          $ hodor oozie:help display_job    # alternate, works the same
+
+        Lastly, to see the topic page that explains the "corresponding paths" concept, that is central to the
+        Hdfs namespace, run:
+
+          $ hodor hdfs:topic corresponding_paths
+
+        And to obtain a list of all topics available within the oozie namespace, for example, run:
+
+          $ hodor oozie:topics
+        ].unindent(8)
+        say more_help
+      end
+
+
+      def method_missing(meth, *args)
+        if args[0].eql?('nocorrect')
+          fail %Q[You are using a shell alias with an improper trailing space. For example:
+                     alias dj='bundle exec hodor oozie:display_job' (works)
+                     alias dj='bundle exec hodor oozie:display_job ' (fails)]
+        end
+        super meth, *args
+      rescue
+        raise
+      end
+
+      def self.handle_no_command_error(command, bv)
+        raise CommandNotFound.new("No Such Command: #{command.inspect}")
+      end
+
+      no_tasks do
+        def thorfiles(*args)
+          Dir[File.join(File.dirname(__FILE__), '..', 'tasks/**/*.thor')]
+        end
+      end
+  
+    end
+  end
+end
+
+
+class Thor
+  module Shell
+    class Basic # rubocop:disable ClassLength
+      def print_wrapped(message, options = {})
+        indent = options[:indent] || 0
+        width = terminal_width - indent - 5
+        paras = message.split("\n\n")
+
+        paras.map! do |unwrapped|
+          unwrapped.strip.gsub(/\n([^\s\-\005])/, ' \1').gsub(/.{1,#{width}}(?:\s|\Z)/) {
+            ($& + 5.chr).gsub(/\n\005/, "\n").gsub(/\005/, "\n")
+          }
+        end
+
+        paras.each do |para|
+          para.split("\n").each do |line|
+            stdout.puts line.insert(0, " " * indent)
+          end
+          stdout.puts unless para == paras.last
+        end
+      end
+    end
+  end
+end
+
+require_relative "command"
+require_relative "ui/table"
+require_relative "api/oozie"

data/lib/hodor/command.rb

@@ -0,0 +1,164 @@
+require 'thor'
+
+module Hodor
+  class Command < ::Thor
+
+    no_tasks do
+
+      def env
+        Environment.instance
+      end
+
+      def target
+        env.settings[:target]
+      end
+
+      def logger
+        env.logger
+      end
+
+      # Part of workaround to prevent parent command arguments from being appended
+      # to child commands
+      #  NOTE: the args argument below should actually be *args.
+      def invoke(name=nil, *args)
+
+        name.sub!(/^Hodor:/, '') if name && $hodor_runner
+        super(name, args + ["-EOLSTOP"])
+      end
+
+      def invoke_command(command, trailing)
+        env.options = options
+        @invoking_command = command.name
+        workaround_thor_trailing_bug(trailing)
+        erb_expand_command_line(trailing)
+        @trailing = trailing
+
+        if self.respond_to?(:intercept_dispatch)
+          @was_intercepted = false
+          intercept_dispatch(command.name.to_sym, trailing)
+          super unless @was_intercepted
+        else
+          super
+        end
+      rescue Hodor::Cli::Usage => ex
+        logger.error "CLI Usage: #{ex.message}"
+      rescue SystemExit, Interrupt
+      rescue => ex
+        if env.prefs[:debug_mode]
+          logger.error "EXCEPTION! #{ex.class.name} :: #{ex.message}\nBACKTRACE:\n\t#{ex.backtrace.join("\n\t")}"
+        else
+          logger.error "#{ex.message}\nException Class: '#{ex.class.name}'"
+        end
+      end
+
+      # This function works around a bug in thor. Basically, when one thor command
+      # calls another (ie. via "invoke"), the parent command's last argument is
+      # appended to the arguments array of the invoked command.  This function
+      # just chops off the extra arguments that shouldn't be in the trailing string.
+      def workaround_thor_trailing_bug(trailing)
+        sentinel = false
+        trailing.select! { |element| 
+          sentinel = true if element.eql?("-EOLSTOP")
+          !sentinel
+        }
+      end
+
+      # Expand any ERB variables on the command line against the loaded environment. If
+      # the environment has no value for the specified key, leave the command line unchanged.
+      # 
+      # Examples:
+      #   $ bthor sandbox:oozie --oozie "<%= env[:oozie_url] %>"
+      #   $ bthor sandbox:oozie --oozie :oozie_url
+      #
+      # Note: Either of above works, since :oozie_url is gsub'd to <%= env[:oozie_url] %>
+      #
+      def erb_expand_command_line(trailing)
+        trailing.map! { |subarg| 
+          env.erb_sub(
+            subarg.gsub(/(?<!\[):[a-zA-Z][_0-9a-zA-Z~]+/) { |match|
+              if env.settings.has_key?(match[1..-1].to_sym)
+                "<%= env[#{match}] %>" 
+              else
+                match
+              end
+            }
+          )
+        }
+      end
+
+      def hadoop_command(cmd, trailing)
+        @was_intercepted = true
+        cmdline = cmd ? "#{cmd} " : ""
+        cmdline << trailing.join(' ')
+        env.ssh cmdline, echo: true, echo_cmd: true
+      end
+
+      def dest_path
+        options[:to] || "."
+      end
+
+      def scp_file(file)
+        # If the file has .erb extension, perform ERB expansion of the file first
+        if file.end_with?('.erb')
+          dest_file = file.sub(/\.erb$/,'')
+          erb_expanded = env.erb_load(file)
+          src_file = "/tmp/#{File.basename(dest_file)}"
+          File.open(src_file, 'w') { |f| f.write(erb_expanded) }
+        else
+          dest_file = "#{options[:parent] || ''}#{file}"
+          src_file = file
+        end
+
+        file_path = "#{dest_path}/#{File.basename(src_file)}"
+        env.run_local %Q[scp #{src_file} #{env.ssh_user}@#{env[:ssh_host]}:#{file_path}],
+        echo: true, echo_cmd: true
+        return file_path
+      end
+
+      def self.load_topic(title)
+        topics = File.join(File.dirname(__FILE__), '..', '..', 'topics', name.split('::').last.downcase)
+        contents = File.open( File.join(topics, "#{title}.txt"), 'rt') { |f| f.read }
+        contents.gsub(/^\\x5/, "\x5")
+      end
+
+      def load_topics
+        topics = File.join(File.dirname(__FILE__), '..', '..', 'topics', self.class.name.split('::').last)
+        Dir.glob(File.join(topics, '*.txt'))
+      end
+    end
+
+    desc "topic [title]", "Display named help topic [title]"
+    def topic(title)
+      say self.class.load_topic(title)
+    end
+
+    desc "topics", "Display a list of topic discussions available for the namespace"
+    def topics
+      say "The following topics (in no particular order) are available within the namespace:"
+      load_topics.each_with_index { |topic, i|
+        say "  Topic: #{File.basename(topic).sub(/.txt$/, '')}"
+      }
+    end
+
+    class << self
+      def inherited(base) #:nodoc:
+        base.send :extend,  ClassMethods
+      end
+    end
+
+    module ClassMethods
+      def namespace(name=nil)
+        case name
+        when nil
+          constant = self.to_s.gsub(/^Thor::Sandbox::/, "")
+          strip = $hodor_runner ? /^Hodor::Cli::/ : /(?<=Hodor::)Cli::/
+          constant = constant.gsub(strip, "")
+          constant =  ::Thor::Util.snake_case(constant).squeeze(":")          
+          @namespace ||= constant
+        else
+          super
+        end
+      end
+    end
+  end
+end