vayacondios-server 0.0.4

data/lib/vayacondios/server/model/document.rb

@@ -0,0 +1,25 @@
+class Vayacondios::Document < Hash
+  attr_accessor :organization, :topic
+
+  def initialize(options = {})
+    options = options.symbolize_keys
+    @organization = options[:organization]
+    @topic = options[:topic]
+  end
+
+  def self.create
+    raise NotImplementedError.new("#{self.name} must be overriden by a subclass.")
+  end
+
+  def self.update
+    raise NotImplementedError.new("#{self.name} must be overriden by a subclass.")
+  end
+
+  def self.find
+    raise NotImplementedError.new("#{self.name} must be overriden by a subclass.")
+  end
+
+  def destroy
+    raise NotImplementedError.new("#{self.name} must be overriden by a subclass.")
+  end
+end

data/lib/vayacondios/server/model/event_document.rb

@@ -0,0 +1,94 @@
+require 'vayacondios/server/model/document'
+
+# The event model
+#
+# Event documents are key-value pairs, represented in JSON. A document
+# consists of a primary key called the topic (_id in mongodb). It belongs to a
+# collection named "#{organization_name}.#{topic}.events"
+#
+# Note: mongodb is passed in beacuse Goliath makes Thread lookups will not
+# work while Goliath is in a streaming context.
+
+class Vayacondios::EventDocument < Vayacondios::Document
+  attr_reader :organization, :topic, :body
+
+  def initialize(mongodb, options = {})
+    super options
+    @mongo = mongodb
+    options = sanitize_options(options)
+
+    @body   = nil
+    @id     = format_id(options[:id])
+    @mongo  = mongodb
+
+    collection_name = [organization, topic, 'events'].join('.')
+    @collection = @mongo.collection(collection_name)
+  end
+
+  def self.create(mongodb, document, options={})
+    self.new(mongodb, options).update(document)
+  end
+
+  def self.find(mongodb, options={})
+    self.new(mongodb, options).find
+  end
+
+  def find
+    result = @collection.find_one({_id: @id})
+    if result.present?
+      result.delete("_id")
+      result['_timestamp'] = result.delete("t")
+      result.merge! result.delete("d") if result["d"].present?
+      @body = result
+      self
+    else
+      nil
+    end
+  end
+
+  def update(document)
+    document = to_mongo(document)
+
+    @body = document[:d]
+    if @id
+      @collection.update({:_id => @id}, document, {upsert: true})
+    else
+      @collection.insert(document)
+    end
+
+    self
+  end
+
+  def destroy(document)
+    super
+  end
+
+  protected
+
+  def sanitize_options(options)
+    options = options.symbolize_keys
+
+    topic = options[:topic].gsub(/\W+/, '_')
+    id = format_id options[:id]
+
+    options.merge!(topic: topic, id: id)
+  end
+
+  def format_id(id)
+    if (id.is_a?(Hash) && id["$oid"].present?)
+      id = BSON::ObjectId(id["$oid"])
+    else
+      id = id.to_s.gsub(/\W/,'')
+      id = BSON::ObjectId(id) if id.match(/^[a-f0-9]{24}$/)
+    end
+    id
+  end
+
+  def to_mongo(document)
+    {}.tap do |result|
+      result[:d] = document.dup
+      result[:_id] = @id if @id
+      result[:t] = document.delete(:_timestamp) || Time.now
+    end
+  end
+end

data/lib/vayacondios/version.rb

@@ -0,0 +1,3 @@
+class Vayacondios
+  VERSION = '0.0.4'
+end

data/lib/vayacondios-client.rb

@@ -0,0 +1,20 @@
+require 'configliere'
+require 'multi_json'
+require 'net/http'
+
+require 'gorillib/builder'
+require 'gorillib/configurable'
+require 'gorillib/enumerable/sum'
+require 'gorillib/exception/raisers'
+require 'gorillib/hash/deep_compact'
+require 'gorillib/hash/deep_merge'
+require 'gorillib/hash/keys'
+require 'gorillib/logger/log'
+require 'gorillib/metaprogramming/class_attribute'
+require 'gorillib/object/blank'
+require 'gorillib/string/constantize'
+require 'gorillib/string/inflections'
+
+require 'vayacondios/client/http_client'
+require 'vayacondios/client/notifier'
+require 'vayacondios/client/configliere'

data/lib/vayacondios-server.rb

@@ -0,0 +1,18 @@
+require 'goliath'
+require 'em-mongo'
+require 'em-synchrony/em-http'
+require 'em-synchrony/em-mongo'
+
+require 'gorillib/object/blank'
+require 'gorillib/enumerable/sum'
+require 'gorillib/hash/deep_merge'
+require 'gorillib/hash/keys'
+require 'gorillib/string/constantize'
+require 'gorillib/string/inflections'
+require 'multi_json'
+
+require 'vayacondios/server/model/config_document'
+require 'vayacondios/server/model/event_document'
+
+require 'vayacondios/server/handlers/config_handler'
+require 'vayacondios/server/handlers/event_handler'

data/scripts/hadoop_monitor/configurable.rb

@@ -0,0 +1,74 @@
+require 'configliere'
+require 'logger'
+
+module Vayacondios
+
+  module Configurable
+
+    #
+    # Declare a name CONST_NAME = :const_name
+    #
+    def self.declare_name symbol
+      const_set symbol.to_s.upcase.to_sym, symbol
+    end
+
+    declare_name :cluster_busy
+    declare_name :cluster_quiet
+    declare_name :event
+    declare_name :time
+
+    attr_reader :logger
+
+    def settings
+      init_settings
+      return @settings
+    end
+
+    def init_settings
+      return if defined? @settings
+      
+      @settings = Configliere::Param.new
+      @settings.use :env_var, :config_file, :commandline
+      
+      @settings.define(:sleep_seconds,
+                       default: 5,
+                       description: "Time to sleep in main loops")
+      @settings.define(:log_level,
+                       default: "info",
+                       description: "Log level. See standard Logger class")
+      @settings.define(:mongo_jobs_db,
+                       default: 'job_info',
+                       description: "Mongo database to dump hadoop job information into")
+      @settings.define(:mongo_job_logs_collection,
+                       default: 'job_logs',
+                       description: "Mongo collection to dump job logs into.")
+      @settings.define(:mongo_job_events_collection,
+                       default: 'job_events',
+                       description: "Mongo collection containing jobs events.")
+      @settings.define(:mongo_machine_stats_collection,
+                       default: 'machine_stats',
+                       description: "Mongo collection containing machine stats.")
+      @settings.define(:mongo_ip,
+                       default: nil,
+                       description: "IP address of Hadoop monitor node")
+      @settings.define(:job_logs_size,
+                       default: 10 * (1 << 20),
+                       description: ("Size (in bytes) of Mongo jobs log collection"))
+      @settings.define(:job_events_size,
+                       default: 10 * (1 << 20),
+                       description: ("Size (in bytes) of Mongo job events collection"))
+      @settings.define(:machine_stats_size,
+                       default: 100 * (1 << 20),
+                       description: ("Size (in bytes) of machine stats collection"))
+      
+      @settings.resolve!
+
+      @logger = Logger.new(STDERR)
+      @logger.level = Logger.const_get(@settings.log_level.upcase.to_sym)
+
+      @logger.info "Settings: #{@settings}"
+
+      @settings
+    end
+  end
+end

data/scripts/hadoop_monitor/hadoop_client.rb

@@ -0,0 +1,249 @@
+require_relative 'configurable'
+require_relative 'hadoopable'
+require 'json'
+require 'optparse'
+require 'ostruct'
+require 'logger'
+require 'pp'
+require 'gorillib/string/inflections'
+require 'swineherd-fs'
+
+module Vayacondios
+
+  class HadoopClient
+
+    include Configurable
+    include Hadoopable
+
+    RUNNING = JobStatus::RUNNING
+
+    def initialize
+      init_settings
+      logger.info "Connecting to job tracker."
+      @job_client = JobClient.new JobConf.new(get_hadoop_conf)
+    end
+    
+    #
+    # (Equality doesn't work for jobs, so - will not work as intended
+    # on arrays of jobs.)
+    #
+    def subtract jobs_array1, jobs_array2
+      jobs_array1.reject{|j| jobs_array2.map(&:job_id).map(&:to_s).index j.job_id.to_s}
+    end
+
+    #
+    # Returns the jobs with the specified state. States are specified
+    # by constants in this class.
+    # 
+    def jobs_with_state state
+      jobs_by_state[state] || []
+    end
+
+    #
+    # Returns the properties of the specified job as a hash.
+    #
+    def job_properties job
+      host_port = job.get_tracking_url[/^(http:\/\/)?[^\/]*/]
+      job_id = job.get_id.to_s
+      conf_uri = "#{host_port}/logs/#{job_id}_conf.xml"
+      properties = parse_properties(open conf_uri)
+      recordize_properties(properties, job_id.to_s)
+    end
+
+    #
+    # Returns the stats for the current job as a hash.
+    #
+    def job_stats job, finish_time
+      parse_job job.get_id.to_s, finish_time
+    end
+
+  private
+    
+    #
+    # Returns a hash JobStatus::<SOME_STATE> => <array of jobs>
+    #
+    def jobs_by_state
+      job_statuses_by_state = @job_client.get_all_jobs.group_by(&:get_run_state)
+      Hash[job_statuses_by_state.map{|state, job_statuses| [state, jobs_from_statuses(job_statuses)]}]
+    end
+
+    #
+    # Some hadoop stuff returns JobStatus objects. This converts them
+    # to RunningJob objects.
+    #
+    def jobs_from_statuses job_statuses
+      job_statuses.map{|job_status| @job_client.get_job job_status.get_job_id}
+    end
+
+    #
+    # Takes an org.apache.hadoop.mapred.RunningJob and returns a hash
+    # object that represents it.
+    #
+    def parse_job job_id, finish_time
+      job = @job_client.get_job job_id
+      job_status = @job_client.get_all_jobs.select{|j| j.get_job_id.to_s == job_id.to_s}.first
+      finished_status = [:FAILED, :KILLED, :COMPLETE]
+      failed_status   = [:FAILED]
+
+      job_data = {
+        
+        _id:              job_id.to_s,
+
+                          # not sure what is what. I'm guessing
+                          # JobStatus.getStartTime corresponds to the
+                          # launch time in the logs, but I'm going to
+                          # go ahead and use it twice here.
+
+        launch_time:      Time.at(job_status.get_start_time / 1000),
+        submit_time:      Time.at(job_status.get_start_time / 1000),
+
+        finish_time:      finish_time,
+
+        job_status:       case job_status.get_run_state
+                          when JobStatus::FAILED    then :FAILED 
+                          when JobStatus::KILLED    then :KILLED
+                          when JobStatus::PREP      then :PREP
+                          when JobStatus::RUNNING   then :RUNNING
+                          when JobStatus::SUCCEEDED then :SUCCEEDED
+                          end,
+
+        finished_maps:    num_tasks(job_id, :map,    finished_status),
+        finished_reduces: num_tasks(job_id, :reduce, finished_status),
+        failed_maps:      num_tasks(job_id, :map,    failed_status),
+        failed_reduces:   num_tasks(job_id, :reduce, failed_status),
+
+        counters:         parse_counters(job.get_counters),
+        type:             :job,
+
+      }
+
+      job_progress = {
+        
+        parent_id:        job.job_id,
+        type:             :job_progress,
+                          # report time in milliseconds for consistency
+        time:             Time.now,
+        cleanup_progress: job.cleanup_progress,
+        map_progress:     job.map_progress,
+        reduce_progress:  job.reduce_progress,
+        setup_progress:   job.setup_progress,
+
+      }
+
+      map_task_data    = @job_client.get_map_task_reports    job_id
+      reduce_task_data = @job_client.get_reduce_task_reports job_id
+
+      m_reports, m_progress_reports, r_reports, r_progress_reports =
+        [
+         map_task_data   .map{|task| parse_task          task, "MAP",    job_id },
+         map_task_data   .map{|task| parse_task_progress task, "MAP"            },
+         reduce_task_data.map{|task| parse_task          task, "REDUCE", job_id },
+         reduce_task_data.map{|task| parse_task_progress task, "REDUCE"         },
+        ]
+      
+      [job_data, job_progress] + m_reports + r_reports + m_progress_reports + r_progress_reports
+    end
+
+    def recordize_properties properties, job_id
+      {
+        parent_id:  job_id,
+        type:       :conf,
+        properties: properties,
+        _id:        [job_id, "_properties"].join
+      }
+    end
+
+    #
+    # Return a hash containing a name => value hash representing the
+    # config for a hadoop job.
+    #
+    def parse_properties conf
+      properties = {}
+      conf.read.scan /[^\n]*\n/ do |line,|
+        m = /<name>([^<]+)<\/name><value>([^<]+)<\/value>/.match line
+        if m and m[1] !~ /fs\.s3n?\.awsSecretAccessKey/ then
+          properties[parse_key m[1]] = parse_atom m[2]
+        end
+      end
+      properties
+    end
+
+    #
+    # Takes an org.apache.hadoop.mapred.TaskReport and returns a Hash
+    # object that represents it.
+    #
+    def parse_task task_report, task_type, parent_job_id
+      {
+        _id:              task_report.get_task_id.to_s,
+        parent_id:        parent_job_id,
+        task_type:        task_type,
+        task_status:      task_report.get_current_status.to_s,
+        start_time:       Time.at(task_report.get_start_time / 1000),
+        finish_time:      Time.at(task_report.get_finish_time / 1000),
+        counters:         parse_counters(task_report.get_counters),
+        type:             :task,
+        diagnostics:      task_report.get_diagnostics.map(&:to_s),
+        running_attempts: task_report.get_running_task_attempts.map(&:to_s),
+      }
+    end
+
+    def parse_task_progress task_report, task_type
+      {
+        parent_id:   task_report.get_task_id.to_s,
+        time:        Time.now,
+        type:        :task_progress,
+        progress:    task_report.get_progress,
+      }
+    end
+
+    #
+    # Takes a class of type org.apache.hadoop.mapred.Counters and
+    # returns a Hash object that represents this counter.
+    #
+    def parse_counters counters
+      Hash[counters.map do |group|
+             [parse_key(group.get_name), Hash[group.map do |counter|
+                                                [parse_key(counter.get_name), counter.get_counter]
+                                              end]]
+           end]
+    end
+
+    #
+    # Parse a key in a log entry. Log entries consist of a type, which I
+    # consider a key, and a list of key=value pairs.
+    #
+    def parse_key key
+      return (parse_atom key).underscore.gsub ".", "_"
+    end
+
+    #
+    # Parse a value in a Hadoop log.
+    #
+    def parse_atom a
+      if /[0-9][ \r\t\n]*\/[ \r\t\n]*[0-9]+/.match a
+        # "0/6" -> [0,6]
+        return a.split("/").collect{|s| s.to_i}
+      elsif /^[0-9,]*$/.match a
+        # "224" -> 224
+        return a.gsub(',', '').to_i
+      else
+        # \. -> .
+        return a.gsub(/([^\\])\\(.)/, '\1\2')
+      end
+    end
+
+    #
+    # Returns the number of tasks of the specified TIPStatus from the
+    # specified job_client of the specified type (map or reduce)
+    #
+    def num_tasks job_id, map_or_reduce, statuses
+      method_name = "get_#{map_or_reduce}_task_reports".to_sym
+      @job_client.send(method_name, job_id).select do |report|
+        tip_statuses = statuses.map do |status|
+          TIPStatus.const_get status
+        end
+        tip_statuses.index report.get_current_status
+      end.size
+    end
+  end
+end