hodor 1.0.2

data/lib/hodor/api/oozie/bundle.rb

@@ -0,0 +1,27 @@
+module Hodor::Oozie
+
+  class Bundle < Job
+
+    attr_reader :status, :json
+
+    class << self
+      def default_columns
+        [:index, :id, :status]
+      end
+    end
+
+    def initialize(json)
+      super()
+      @json = json
+      @status = json["status"]
+    end
+
+    def expand
+      # Expand immediate children
+      @coordinators = json["coords"].map do |item|
+        Coordinator.new(item)
+      end.compact
+    end
+
+  end
+end

data/lib/hodor/api/oozie/coordinator.rb

@@ -0,0 +1,53 @@
+module Hodor::Oozie
+
+  class Coordinator < Job
+
+    attr_reader :id, :json, :name, :path, :timezone, :frequency, :conf, :end_time, :execution_policy, :start_time, :time_unit,
+        :concurrency, :id, :last_action, :acl, :mat_throttling, :timeout, :next_materialized_time, :parent_id,
+        :external_id, :group, :user, :console_url, :actions, :acl, :status, :materializations
+
+    class << self
+      def default_columns
+        [:index, :id, :name, :status, :start_time, :time_unit, :external_id]
+      end
+    end
+
+    def initialize(json)
+      super()
+      @json = json
+      @name = json["coordJobName"]
+      @path = json["coordJobPath"]
+      @timezone = json["timeZone"]
+      @frequency = json["frequency"]
+      @conf = json["conf"]
+      @end_time = parse_time(json["endTime"])
+      @execution_policy = json["executionPolicy"]
+      @start_time = parse_time(json["startTime"])
+      @time_unit = json["timeUnit"]
+      @concurrency = json["concurrency"]
+      @id = json["coordJobId"]
+      @last_action = parse_time(json["lastAction"])
+      @acl = json["acl"]
+      @mat_throttling = json["mat_throttling"]
+      @timeout = json["timeOut"]
+      @next_materialized_time = parse_time(json["nextMaterializedTime"])
+      @parent_id = @bundle_id = json["bundleId"]
+      @external_id = json["coordExternalId"]
+      @group = json["group"]
+      @user = json["user"]
+      @console_url = json["consoleUrl"]
+      @actions = json["actions"]
+      @acl = json["acl"]
+      @status = json["status"]
+    end
+
+    def expand
+      # Expand immediate children
+      @materializations = json["actions"].map do |item|
+        Materialization.new(item)
+      end.compact.reverse
+      @materializations
+    end
+
+  end
+end

data/lib/hodor/api/oozie/hadoop_job.rb

@@ -0,0 +1,29 @@
+module Hodor::Oozie
+
+  class HadoopJob < Job
+
+    attr_reader :id, :parent_id, :rest_call
+
+    class << self
+      def default_columns
+        [:index, :id]
+      end
+    end
+
+    def initialize(parent_id, job_id)
+      super()
+      @id = job_id
+      @parent_id = parent_id
+      session.verbose = true # force verbosity for hadoop jobs
+    end
+
+    def log
+      hadoop_id = @id.sub('job_','')
+      trash = hadoop_id.index(/[^0-9_]/)
+      hadoop_id = hadoop_id[0..trash-1] if trash
+      session.env.ssh "mapred job -logs job_#{hadoop_id} attempt_#{hadoop_id}_m_000000_0"
+    end
+
+  end
+
+end

data/lib/hodor/api/oozie/job.rb

@@ -0,0 +1,192 @@
+require 'fileutils'
+require 'awesome_print'
+require 'stringio'
+require 'ox'
+
+require_relative 'session'
+
+module Hodor::Oozie
+
+  class Job 
+    attr_reader :index, :id, :parent_id, :skip_to, :conf, :rest_call
+    attr_accessor :columns
+
+    def session
+      Hodor::Oozie::Session.instance
+    end
+
+    def oozie
+      Hodor::Oozie
+    end
+
+    def initialize
+      @index = -1
+      @rest_call = session.last_query
+    end
+
+    def set_index(i)
+      @index = i
+    end
+
+    def indexed_job_id
+      nil
+    end
+
+    def parse_time timestamp
+      Time.strptime(timestamp, "%a, %d %b %Y %H:%M:%S %Z") if timestamp
+    end
+
+    def expand
+      nil
+    end
+
+    def children
+      if @children.nil?
+        @children = expand
+      end
+      @children
+    end
+
+    def display_as_time(val)
+      display_date = val.strftime("%Y-%m-%d %H:%M")
+      cur_date = Time.now.strftime("%Y-")
+      if display_date[0..4].eql?(cur_date)
+        display_date[5..-1]
+      else
+        display_date
+      end
+    end
+
+    def sanitize val, max_length = 120
+      sval = val.to_s.gsub(/\s+/, ' ')
+      if sval.length > max_length
+        sval.to_s[0..max_length-3] + '...'
+      else
+        sval
+      end
+    end
+
+    def title
+      "#{self.class.name.split('::').last} Properties"
+    end
+
+    def display_properties
+      if self.class.respond_to?(:suppress_properties)
+        suppress = self.class.suppress_properties
+      else
+        suppress = false
+      end
+      props = suppress ? nil :
+        instance_variables.map { |var| var[1..-1] }.select { |var| !var.eql?("index") }
+      if props
+        rows = props.inject([]) { |result, prop|
+          display_override = "display_#{prop.to_s}".to_sym
+          if respond_to?(display_override)
+            val = method(display_override.to_s).call
+          else
+            val = instance_variable_get("@#{prop}")
+            if val.is_a?(Time)
+              val = display_as_time(val)
+            else
+              val = val
+            end
+          end
+          result << [prop, sanitize(val)]
+        }
+        { rows: rows }
+      else
+        nil
+      end
+    end
+
+    def display_as_array(columns, ellipsis = false)
+      row = columns.inject([]) { |cols, head|
+        if ellipsis
+          val = "..."
+        else
+          display_override = "display_#{head.to_s}".to_sym
+          if respond_to?(display_override)
+            val = method(display_override.to_s).call
+          else
+            val = instance_variable_get("@#{head}")
+            val = display_as_time(val) if val.is_a?(Time)
+          end
+        end
+        cols << sanitize(val, 80)
+      }
+    end
+
+    def child_columns
+      first_child = children.first
+      if first_child
+        first_child.class.default_columns
+      else
+        nil
+      end
+    end
+
+    def children_title
+      "#{self.class.name.split('::').last} Children"
+    end
+
+    def display_children
+      if children.nil? || children.length == 0
+        nil
+      else
+        headings = child_columns.map { |head| 
+          head.to_s.split('_').map { |w| w.capitalize }.join(' ')
+        }
+        children.each_with_index { |c, i|
+          c.set_index(i)
+        }
+
+        truncated = children.length > session.len
+        childrows = truncated ? children[0..session.len-1] : children
+
+        rows = childrows.inject([]) { |result, v|
+          result << v.display_as_array(child_columns)
+          result
+        }
+
+        rows[rows.length-1][0] = "#{rows[rows.length-1][0]}+" if truncated
+        { headings: headings, rows: rows }
+      end
+    end
+
+    class Configuration < ::Ox::Sax
+      attr_reader :map
+
+      def initialize 
+        @map = {}
+      end
+
+      def start_element(key)
+        @incoming = key
+      end
+
+      def text(incoming)
+        case @incoming
+        when :name;
+          @key = incoming
+        when :value;
+          @map[@key] = incoming
+        end
+      end
+    end
+
+    def conf_map
+      io = StringIO.new(conf || "")
+      handler = Configuration.new()
+      Ox.sax_parse(handler, io)
+      handler.map
+    end
+
+    def log
+      session.get_job_state(id, "show=log")
+    end
+
+    def definition
+      session.get_job_state(id, "show=definition")
+    end
+  end
+end

data/lib/hodor/api/oozie/materialization.rb

@@ -0,0 +1,56 @@
+module Hodor::Oozie
+
+  class Materialization < Job
+
+    attr_reader :json, :error_message, :last_modified_time, :created_at, :status, :push_missing_dependencies,
+      :external_status, :type, :nominal_time, :external_id, :created_conf, :missing_dependencies,
+      :run_conf, :action_number, :error_code, :tracker_uri, :to_string, :parent_id, :coord_job_id,
+      :console_url
+
+    class << self
+      def default_properties
+        [ :error_message, :last_modified_time, :created_at, :status, :push_missing_dependencies,
+          :external_status, :type, :nominal_time, :external_id, :created_conf, :missing_dependencies,
+          :run_conf, :action_number, :error_code, :tracker_uri, :to_string, :parent_id, :coord_job_id,
+          :console_url]
+      end
+
+      def default_columns
+        [:index, :id, :status, :external_id, :type, :created_at, :nominal_time, :last_modified]
+      end
+    end
+
+    def initialize(json)
+      super()
+      @json = json
+      @error_message = json["errorMessage"]
+      @last_modified = @last_modified_time = parse_time(json["lastModifiedTime"])
+      @created_at = parse_time(json["createdTime"])
+      @status = json["status"]
+      @push_missing_dependencies = json["pushMissingDependencies"]
+      @external_status = json["externalStatus"]
+      @type = json["type"]
+      @nominal_time = parse_time(json["nominalTime"])
+      @external_id = json["externalId"]
+      @id = json["id"]
+      @created_conf = json["createdConf"]
+      @missing_dependencies = json["missingDependencies"]
+      @run_conf = json["runConf"]
+      @action_number = json["actionNumber"]
+      @error_code = json["errorCode"]
+      @tracker_uri = json["trackerUri"]
+      @to_string = json["toString"]
+      @parent_id = @coord_job_id = json["coordJobId"]
+      @console_url = json["consoleUrl"]
+    end
+
+    def expand
+      [ oozie.job_by_id(@external_id) ]
+    end
+
+    def display_id
+      @id[@id.rindex('C@')..-1]
+    end
+  end
+
+end

data/lib/hodor/api/oozie/query.rb

@@ -0,0 +1,115 @@
+module Hodor::Oozie
+
+  class Query < Job
+    attr_reader :request, :json, :jobs, :filter
+
+    def session
+      Hodor::Oozie::Session.instance
+    end
+
+    class << self
+      def suppress_properties
+        true
+      end
+      def default_columns
+        [:index, :name, :status, :id, :start_time, :time_unit, :external_id]
+      end
+    end
+
+    def initialize(filter = {} )
+      @filter = filter
+      @jobtype = filter[:jobtype] || "coord"
+      @json = run_query(filter)
+      @request = session.last_query
+    end
+
+    def run_query(filter)
+      params = []
+      if filter.has_key?(:user)
+        params << "user%3D#{filter[:user]}"
+      end
+
+      if filter.has_key?(:status)
+        stats = filter[:status]
+        params += stats.map { |val|
+          case val
+          when :running_first;
+            @running_first = true
+            "status%3DRUNNING"
+          when :running; "status%3DRUNNING"
+          when :killed; "status%3DKILLED"
+          when :suspended; "status%3DSUSPENDED"
+          when :timedout; "status%3DTIMEDOUT"
+          when :failed; "status%3DFAILED"
+          when :succeeded; "status%3DSUCCEEDED"
+          end
+        }
+      end
+
+      if params.size > 0
+        filter_exp = "filter=#{params.join(';')}"
+      else
+        filter_exp = ""
+      end
+
+      pagination = []
+      pagination << "offset=#{session.offset}"
+      pagination << "len=#{session.len.to_i+1}"
+      pagination = pagination.join('&')
+
+      response = session.search_jobs("jobtype=#{@jobtype}", filter_exp, pagination)
+      @json = JSON.parse(response)
+    end
+
+    def title
+      if @jobtype.start_with?('coord')
+        "List of Coordinators"
+      elsif @jobtype.start_with?('w')
+        "List of Workflows"
+      elsif @jobtype.start_with?('b')
+        "List of Bundles"
+      else
+        "List of Matching Jobs"
+      end
+    end
+
+    def expand
+      # expand immediate children
+      if @json.has_key?("workflows")
+        all_jobs = @json["workflows"].map do |item|
+          Hodor::Oozie::Workflow.new(item)
+        end.compact
+        if @running_first
+          more_json = run_query( { status: [:killed, :succeeded, :failed, :suspended] } )
+          if more_json.has_key?("workflows")
+            all_jobs += more_json["workflows"].map do |item|
+              Hodor::Oozie::Workflow.new(item)
+            end.compact
+          end
+        end
+      elsif @json.has_key?("coordinatorjobs")
+        all_jobs = @json["coordinatorjobs"].map do |item|
+          Hodor::Oozie::Coordinator.new(item)
+        end.compact
+        if @running_first
+          more_json = run_query( { status: [:succeeded, :killed, :failed, :suspended] } )
+          if more_json.has_key?("coordinatorjobs")
+            all_jobs += more_json["coordinatorjobs"].map do |item|
+              Hodor::Oozie::Coordinator.new(item)
+            end.compact
+          end
+        end
+      end
+      if @filter[:match]
+        pattern = @filter[:match]
+        @jobs = all_jobs.select { | job| 
+          job.name.include?(pattern)
+        }
+      else
+        @jobs = all_jobs
+      end
+      @jobs
+    end
+
+  end
+end