vayacondios-server 0.2.11 → 0.3.0

data/lib/vayacondios/server/rack/path_validation.rb

@@ -1,22 +0,0 @@
-class Vayacondios
-  module Rack   
-    class PathValidation
-      include Goliath::Rack::AsyncMiddleware
-      
-      def initialize(app, opts = {})
-        @app = app ; @opts = opts
-      end
-      
-      def call(env)
-        return [400, {}, MultiJson.dump({ error: "Bad Request. Format path is <host>/v1/<org>/event/<topic>" })] unless valid_paths? env[:vayacondios_path]
-        @app.call(env)
-      end
-      
-      def valid_paths?(path)
-        # use @opts for validation later
-        path.nil? ? false : true
-      end
-      
-    end
-  end
-end

data/lib/vayacondios/version.rb

@@ -1,3 +0,0 @@
-class Vayacondios
-  VERSION = '0.2.11'
-end

data/lib/vayacondios-client.rb

@@ -1,22 +0,0 @@
-require 'configliere'
-require 'multi_json'
-require 'net/http'
-
-require 'gorillib/builder'
-require 'gorillib/configurable'
-require 'gorillib/enumerable/sum'
-require 'gorillib/exception/raisers'
-require 'gorillib/hash/deep_compact'
-require 'gorillib/hash/deep_merge'
-require 'gorillib/hash/keys'
-require 'gorillib/logger/log'
-require 'gorillib/metaprogramming/class_attribute'
-require 'gorillib/object/blank'
-require 'gorillib/string/constantize'
-require 'gorillib/string/inflections'
-
-require 'vayacondios/client/http_client'
-require 'vayacondios/client/cube_client'
-require 'vayacondios/client/zabbix_client'
-require 'vayacondios/client/notifier'
-require 'vayacondios/client/configliere'

data/scripts/hadoop_monitor/configurable.rb

@@ -1,66 +0,0 @@
-require 'configliere'
-require 'logger'
-
-class Vayacondios
-
-  module Configurable
-
-    #
-    # Declare a name CONST_NAME = :const_name
-    #
-    def self.declare_name symbol
-      const_set symbol.to_s.upcase.to_sym, symbol
-    end
-
-    declare_name :cluster_busy
-    declare_name :cluster_quiet
-    declare_name :event
-    declare_name :time
-
-    attr_reader :logger
-
-    def settings
-      init_settings
-      return @settings
-    end
-
-    def init_settings
-      return if defined? @settings
-
-      @settings = Configliere::Param.new
-      @settings.use :env_var, :config_file, :commandline
-
-      @settings.define(:config_file,
-                       description: "Config file location")
-      @settings.define(:sleep_seconds,
-                       default: 5,
-                       description: "Time to sleep in main loops")
-      @settings.define(:log_level,
-                       default: "info",
-                       description: "Log level. See standard Logger class")
-      @settings.define(:mongo_db,
-                       default: 'job_info',
-                       description: "Mongo database to dump hadoop job information into")
-      @settings.define(:mongo_ip,
-                       default: nil,
-                       description: "IP address of Hadoop monitor node")
-      @settings.define(:mongo_collection_size,
-                       default: 10 * (1 << 20),
-                       description: ("Size (in bytes) of Mongo job events collection"))
-
-      @settings.resolve!
-
-      if @settings.config_file
-        @settings.read(@settings.config_file)
-        @settings.resolve!
-      end
-
-      @logger = Logger.new(STDERR)
-      @logger.level = Logger.const_get(@settings.log_level.upcase.to_sym)
-
-      @logger.info "Settings: #{@settings}"
-
-      @settings
-    end
-  end
-end

data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb

@@ -1,45 +0,0 @@
-require 'open-uri'
-require 'nibbler'
-require 'socket'
-
-class HadoopAttemptScraper < Nibbler
-  attr_accessor :task_id
-
-  def self.scrape_task(task_id)
-    task_id = task_id.to_s
-
-    url     = "http://#{Socket.gethostname}:50030/taskdetails.jsp?tipid=#{task_id}"
-    scrape  = parse(open(url))
-    scrape.task_id = task_id
-
-    scrape
-  end
-
-    elements 'table.jobtasks tbody > tr' => :attempts do
-    element 'td:nth-child(1)' => 'attempt_id'
-    element 'td:nth-child(2) a/@href' => 'machine'
-    element 'td:nth-child(3)' => 'status'
-    element 'td:nth-child(4)' => 'progress'
-    element 'td:nth-child(5)' => 'start_time'
-    element 'td:nth-child(6)' => 'finish_time'
-    element 'td:nth-child(7)' => 'errors'
-  end
-
-  def to_attempts
-    attempts.map do |attempt|
-      start_time  = Time.parse(attempt.start_time) rescue nil
-      finish_time = attempt.finish_time.length > 0 ? Time.parse(attempt.finish_time) : nil
-      {
-        _id:         attempt.attempt_id.to_s,
-        task_id:     task_id,
-        host:        attempt.machine.to_s.gsub(/^http:\/\//, '').gsub(/:[0-9]+$/, ''),
-        status:      attempt.status,
-        progress:    attempt.progress.to_f / 100.0,
-        start_time:  start_time,
-        finish_time: finish_time,
-        duration:    start_time ? (finish_time || Time.now) - start_time : nil,
-        errors:      attempt.errors
-      }
-    end
-  end
-end

data/scripts/hadoop_monitor/hadoop_client.rb

@@ -1,273 +0,0 @@
-require_relative 'configurable'
-require_relative 'hadoopable'
-require_relative 'hadoop_attempt_scraper'
-
-require 'json'
-require 'optparse'
-require 'ostruct'
-require 'logger'
-require 'pp'
-require 'gorillib/string/inflections'
-require 'swineherd-fs'
-
-class Vayacondios
-
-  class HadoopClient
-
-    include Configurable
-    include Hadoopable
-
-    RUNNING = JobStatus::RUNNING
-
-    def initialize
-      init_settings
-      logger.info "Connecting to job tracker."
-      @job_client = JobClient.new JobConf.new(get_hadoop_conf)
-    end
-
-    #
-    # (Equality doesn't work for jobs, so - will not work as intended
-    # on arrays of jobs.)
-    #
-    def subtract jobs_array1, jobs_array2
-      jobs_array1.reject{|j| jobs_array2.map(&:job_id).map(&:to_s).index j.job_id.to_s}
-    end
-
-    #
-    # Returns the jobs with the specified state. States are specified
-    # by constants in this class.
-    #
-    def jobs_with_state state
-      jobs_by_state[state] || []
-    end
-
-    #
-    # Returns the properties of the specified job as a hash.
-    #
-    def job_properties job
-      host_port = job.get_tracking_url[/^(http:\/\/)?[^\/]*/]
-      job_id = job.get_id.to_s
-      conf_uri = "#{host_port}/logs/#{job_id}_conf.xml"
-
-      parse_properties(open conf_uri)
-    end
-
-    #
-    # Returns the stats for the current job as a hash.
-    #
-    def job_stats job, finish_time
-      parse_job job.get_id, finish_time
-    end
-
-  private
-
-    #
-    # Returns a hash JobStatus::<SOME_STATE> => <array of jobs>
-    #
-    def jobs_by_state
-      job_statuses_by_state = @job_client.get_all_jobs.group_by(&:get_run_state)
-      Hash[job_statuses_by_state.map{|state, job_statuses| [state, jobs_from_statuses(job_statuses)]}]
-    end
-
-    #
-    # Some hadoop stuff returns JobStatus objects. This converts them
-    # to RunningJob objects.
-    #
-    def jobs_from_statuses job_statuses
-      job_statuses.map{|job_status| @job_client.get_job job_status.get_job_id}
-    end
-
-    #
-    # Takes an org.apache.hadoop.mapred.RunningJob and returns a hash
-    # object that represents it.
-    #
-    def parse_job job_id, finish_time
-      job             = @job_client.get_job job_id
-      job_status      = @job_client.get_all_jobs.select{|j| j.get_job_id.to_s == job_id.to_s}.first
-      finished_status = [:FAILED, :KILLED, :COMPLETE]
-      failed_status   = [:FAILED]
-
-
-      # not sure what is what. I'm guessing
-      # JobStatus.getStartTime corresponds to the
-      # launch time in the logs
-
-      start_time      = Time.at(job_status.get_start_time / 1000)
-      reduce_progress = job.reduce_progress
-      map_progress    = job.map_progress
-      run_duration    = (finish_time || Time.now) - start_time
-
-      map_eta    = map_progress    && map_progress    > 0.0 ? (start_time + (run_duration / map_progress))    : nil
-      reduce_eta = reduce_progress && reduce_progress > 0.0 ? (start_time + (run_duration / reduce_progress)) : nil
-
-      job_data = {
-
-        _id:              job_id.to_s,
-        name:             job.get_job_name.to_s,
-
-        start_time:       start_time,
-        finish_time:      finish_time,
-
-        duration:         run_duration,
-
-        map_eta:          map_eta,
-        reduce_eta:       reduce_eta,
-        eta:              reduce_eta,
-
-        status:       case job_status.get_run_state
-                          when JobStatus::FAILED    then :FAILED
-                          when JobStatus::KILLED    then :KILLED
-                          when JobStatus::PREP      then :PREP
-                          when JobStatus::RUNNING   then :RUNNING
-                          when JobStatus::SUCCEEDED then :SUCCEEDED
-                          end,
-
-        finished_maps:    num_tasks(job_id, :map,    finished_status),
-        finished_reduces: num_tasks(job_id, :reduce, finished_status),
-        failed_maps:      num_tasks(job_id, :map,    failed_status),
-        failed_reduces:   num_tasks(job_id, :reduce, failed_status),
-
-        counters:         parse_counters(job.get_counters)
-      }
-
-      job_event = {
-        t:                Time.now,
-        d: {
-          job_id:           job.job_id,
-          cleanup_progress: job.cleanup_progress,
-          map_progress:     job.map_progress,
-          reduce_progress:  job.reduce_progress,
-          setup_progress:   job.setup_progress,
-        }
-      }
-
-      setup_task_data   = @job_client.get_setup_task_reports    job_id
-      map_task_data     = @job_client.get_map_task_reports      job_id
-      reduce_task_data  = @job_client.get_reduce_task_reports   job_id
-      cleanup_task_data = @job_client.get_cleanup_task_reports  job_id
-
-      setup_reports       = setup_task_data.map{|task| parse_task task, "SETUP", job_id }
-      setup_event_reports = setup_task_data.map{|task| parse_task_progress task, "SETUP" }
-
-      map_reports       = map_task_data.map{|task| parse_task task, "MAP", job_id }
-      map_event_reports = map_task_data.map{|task| parse_task_progress task, "MAP" }
-
-      reduce_reports       = reduce_task_data.map{|task| parse_task task, "REDUCE", job_id }
-      reduce_event_reports = reduce_task_data.map{|task| parse_task_progress task, "REDUCE" }
-
-      cleanup_reports       = cleanup_task_data.map{|task| parse_task task, "CLEANUP", job_id }
-      cleanup_event_reports = cleanup_task_data.map{|task| parse_task_progress task, "CLEANUP" }
-
-      tasks = setup_reports + map_reports + reduce_reports + cleanup_reports
-      task_events = setup_event_reports + map_event_reports + reduce_event_reports + cleanup_event_reports
-
-      attempt_reports = tasks.map{|task| HadoopAttemptScraper.scrape_task(task[:_id]).to_attempts }.flatten
-
-      {
-        job: job_data,
-        job_event: job_event,
-        tasks: tasks,
-        task_events: task_events,
-        attempts: attempt_reports
-      }
-    end
-
-    #
-    # Return a hash containing a name => value hash representing the
-    # config for a hadoop job.
-    #
-    def parse_properties conf
-      properties = {}
-      conf.read.scan /[^\n]*\n/ do |line,|
-        m = /<name>([^<]+)<\/name><value>([^<]+)<\/value>/.match line
-        if m and m[1] !~ /fs\.s3n?\.awsSecretAccessKey/ then
-          properties[parse_key m[1]] = parse_atom m[2]
-        end
-      end
-      properties
-    end
-
-    #
-    # Takes an org.apache.hadoop.mapred.TaskReport and returns a Hash
-    # object that represents it.
-    #
-    def parse_task task_report, task_type, parent_job_id
-      start_time  = task_report.get_start_time > 0  ? Time.at(task_report.get_start_time  / 1000) : nil
-      finish_time = task_report.get_finish_time > 0 ? Time.at(task_report.get_finish_time / 1000) : nil
-
-      {
-        _id:                   task_report.get_task_id.to_s,
-        job_id:                parent_job_id.to_s,
-        type:                  task_type,
-        status:                task_report.get_current_status.to_s,
-        start_time:            start_time,
-        finish_time:           finish_time,
-        duration:              start_time ? (finish_time || Time.now) - start_time : nil,
-        counters:              parse_counters(task_report.get_counters),
-        diagnostics:           task_report.get_diagnostics.map(&:to_s),
-        successful_attempt_id: task_report.get_successful_task_attempt.to_s
-      }
-    end
-
-    def parse_task_progress task_report, task_type
-      {
-        t: Time.now,
-        d: {
-          task_id:             task_report.get_task_id.to_s,
-          progress:            task_report.get_progress,
-          running_attempt_ids: task_report.get_running_task_attempts.map(&:to_s)
-        }
-      }
-    end
-
-    #
-    # Takes a class of type org.apache.hadoop.mapred.Counters and
-    # returns a Hash object that represents this counter.
-    #
-    def parse_counters counters
-      Hash[counters.map do |group|
-             [parse_key(group.get_name), Hash[group.map do |counter|
-                                                [parse_key(counter.get_name), counter.get_counter]
-                                              end]]
-           end]
-    end
-
-    #
-    # Parse a key in a log entry. Log entries consist of a type, which I
-    # consider a key, and a list of key=value pairs.
-    #
-    def parse_key key
-      return (parse_atom key).underscore.gsub ".", "_"
-    end
-
-    #
-    # Parse a value in a Hadoop log.
-    #
-    def parse_atom a
-      if /[0-9][ \r\t\n]*\/[ \r\t\n]*[0-9]+/.match a
-        # "0/6" -> [0,6]
-        return a.split("/").collect{|s| s.to_i}
-      elsif /^[0-9,]*$/.match a
-        # "224" -> 224
-        return a.gsub(',', '').to_i
-      else
-        # \. -> .
-        return a.gsub(/([^\\])\\(.)/, '\1\2')
-      end
-    end
-
-    #
-    # Returns the number of tasks of the specified TIPStatus from the
-    # specified job_client of the specified type (map or reduce)
-    #
-    def num_tasks job_id, map_or_reduce, statuses
-      method_name = "get_#{map_or_reduce}_task_reports".to_sym
-      @job_client.send(method_name, job_id).select do |report|
-        tip_statuses = statuses.map do |status|
-          TIPStatus.const_get status
-        end
-        tip_statuses.index report.get_current_status
-      end.size
-    end
-  end
-end

data/scripts/hadoop_monitor/hadoop_monitor.rb

@@ -1,101 +0,0 @@
-#!/usr/bin/env jruby19
-
-require_relative 'hadoop_client'
-require_relative 'configurable'
-require 'java'
-require 'mongo'
-require 'scanf'
-require 'gorillib/hash/slice'
-require 'thread'
-require 'open-uri'
-require 'json'
-
-class Vayacondios
-
-  class HadoopMonitor
-    def initialize
-      init_settings
-
-      @hadoop = HadoopClient.new
-
-      @monitored_jobs = []
-
-      logger.debug "Creating mongo collections."
-      @conn = Mongo::Connection.new settings.mongo_ip
-      @db = @conn[settings.mongo_db]
-
-      capped_collection_opts = {
-        :capped => true,
-        :size => settings.mongo_collection_size
-      }
-
-      @collections = {
-        jobs:     @db.create_collection('jobs'),
-        tasks:    @db.create_collection('job_tasks'),
-        attempts: @db.create_collection('job_task_attempts'),
-
-        job_events:     @db.create_collection('job_events',      capped_collection_opts),
-        task_events:    @db.create_collection('job_task_events', capped_collection_opts),
-      }
-    end
-
-    def run
-      loop do
-
-        logger.debug "In main event loop."
-
-        running_jobs  = @hadoop.jobs_with_state HadoopClient::RUNNING
-        started_jobs  = @hadoop.subtract(running_jobs, @monitored_jobs)
-        finished_jobs = @hadoop.subtract(@monitored_jobs, running_jobs)
-
-        finished_jobs.each do |job|
-          logger.debug "#{job.get_id.to_s} is complete."
-          update_job_stats job, Time.now
-        end
-
-        running_jobs.each{|job| update_job_stats job, nil, @hadoop.subtract([job], started_jobs).empty? }
-
-        @monitored_jobs = running_jobs
-
-        sleep settings.sleep_seconds
-
-      end
-    end
-
-  private
-
-    include Configurable
-
-    def update_job_stats job, finish_time = nil, include_properties = false
-      stats = @hadoop.job_stats(job, finish_time)
-
-      if include_properties
-        stats[:job][:properties] = @hadoop.job_properties job
-      end
-
-      logger.debug "upserting job #{JSON.generate stats[:job]}"
-      @collections[:jobs].update({_id: stats[:job][:_id]}, stats[:job], upsert: true)
-
-      logger.debug "upserting job_event #{JSON.generate stats[:job_event]}"
-      @collections[:job_events].insert(stats[:job_event])
-
-      logger.debug "upserting tasks #{JSON.generate stats[:tasks]}"
-      stats[:tasks].each do |task|
-        @collections[:tasks].update({_id: task[:_id]}, task, upsert: true)
-      end
-
-      logger.debug "upserting task_events #{JSON.generate stats[:task_events]}"
-      stats[:task_events].each do |task_event|
-        @collections[:task_events].insert(task_event)
-      end
-
-      logger.debug "upserting attempts #{JSON.generate stats[:attempts]}"
-      stats[:attempts].each do |attempt|
-        @collections[:attempts].update({_id: attempt[:_id]}, attempt, upsert: true)
-      end
-    end
-
-  end
-end
-
-Vayacondios::HadoopMonitor.new.run

data/scripts/hadoop_monitor/hadoopable.rb

@@ -1,65 +0,0 @@
-require 'stringio'
-
-class Vayacondios
-
-  module Hadoopable
-
-    include Configurable
-
-    #--------------------------------------------------------------------------------
-    # Initialize jruby and tell it about hadoop.
-    #--------------------------------------------------------------------------------
-
-    begin
-      require 'java'
-    rescue LoadError => e
-      raise "\nJava not found. Are you sure you're running with JRuby?\n#{e.message}"
-    end
-
-    hadoop_home = ENV['HADOOP_HOME'] || '/usr/lib/hadoop'
-
-    raise "\nHadoop installation not found. Try setting $HADOOP_HOME\n" unless (hadoop_home and (File.exist? hadoop_home))
-
-    $CLASSPATH << File.join(File.join(hadoop_home, 'conf') || ENV['HADOOP_CONF_DIR'],
-                            '') # add trailing slash
-
-    Dir["#{hadoop_home}/{hadoop*.jar,lib/*.jar}"].each{|jar| require jar}
-
-    include_class org.apache.hadoop.mapred.JobConf
-    include_class org.apache.hadoop.mapred.JobClient
-    include_class org.apache.hadoop.mapred.JobStatus
-    include_class org.apache.hadoop.mapred.TIPStatus
-    include_class org.apache.hadoop.conf.Configuration
-    #--------------------------------------------------------------------------------
-
-    def get_hadoop_conf
-      logger.debug "Getting hadoop configuration"
-
-      stderr, $stderr = $stderr, StringIO.new
-
-      conf = Configuration.new
-
-      # per-site defaults
-      %w[capacity-scheduler.xml core-site.xml hadoop-policy.xml hadoop-site.xml hdfs-site.xml mapred-site.xml].each do |conf_file|
-        conf.addResource conf_file
-      end
-
-      conf.reload_configuration
-
-      # per-user overrides
-      if Swineherd.config[:aws]
-        conf.set("fs.s3.awsAccessKeyId",Swineherd.config[:aws][:access_key])
-        conf.set("fs.s3.awsSecretAccessKey",Swineherd.config[:aws][:secret_key])
-
-        conf.set("fs.s3n.awsAccessKeyId",Swineherd.config[:aws][:access_key])
-        conf.set("fs.s3n.awsSecretAccessKey",Swineherd.config[:aws][:secret_key])
-      end
-
-      return conf
-    ensure
-      stderr_lines = $stderr.string.split("\n")
-      $stderr = stderr
-      stderr_lines.each{|line| logger.debug line}
-    end
-  end
-end