vayacondios-server 0.1.1 → 0.1.2

data/Gemfile

@@ -4,8 +4,9 @@ gemspec name: 'vayacondios-server'
 gemspec name: 'vayacondios-client'
 
 group :hadoop_monitor do
-  gem 'mongo'
-  gem 'bson_ext'
+  gem 'mongo',    '~> 1.8'
   gem 'gorillib', require: 'gorillib/hash/slice'
   gem 'json'
+  gem 'nokogiri', '~> 1.5'
+  gem 'nibbler',  '~> 1.3'
 end

data/lib/vayacondios/client/notifier.rb

@@ -21,6 +21,8 @@ class Vayacondios
   end
 
   class NullNotifier < Notifier
+    def initialize(*args) ; end
+    
     def notify topic, cargo={}
     end
   end
@@ -56,7 +58,7 @@ class Vayacondios
 
   class NotifierFactory
     def self.receive(attrs = {})
-      type = attrs.delete(:type)
+      type = attrs[:type]
       case type
       when 'http'        then HttpNotifier.new(attrs)
       when 'log'         then LogNotifier.new(attrs)

data/lib/vayacondios/version.rb

@@ -1,3 +1,3 @@
 class Vayacondios
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
 end

data/scripts/hadoop_monitor/configurable.rb

@@ -38,30 +38,15 @@ module Vayacondios
       @settings.define(:log_level,
                        default: "info",
                        description: "Log level. See standard Logger class")
-      @settings.define(:mongo_jobs_db,
+      @settings.define(:mongo_db,
                        default: 'job_info',
                        description: "Mongo database to dump hadoop job information into")
-      @settings.define(:mongo_job_logs_collection,
-                       default: 'job_logs',
-                       description: "Mongo collection to dump job logs into.")
-      @settings.define(:mongo_job_events_collection,
-                       default: 'job_events',
-                       description: "Mongo collection containing jobs events.")
-      @settings.define(:mongo_machine_stats_collection,
-                       default: 'machine_stats',
-                       description: "Mongo collection containing machine stats.")
       @settings.define(:mongo_ip,
                        default: nil,
                        description: "IP address of Hadoop monitor node")
-      @settings.define(:job_logs_size,
-                       default: 10 * (1 << 20),
-                       description: ("Size (in bytes) of Mongo jobs log collection"))
-      @settings.define(:job_events_size,
+      @settings.define(:mongo_collection_size,
                        default: 10 * (1 << 20),
                        description: ("Size (in bytes) of Mongo job events collection"))
-      @settings.define(:machine_stats_size,
-                       default: 100 * (1 << 20),
-                       description: ("Size (in bytes) of machine stats collection"))
 
       @settings.resolve!
 

data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb

@@ -0,0 +1,42 @@
+require 'open-uri'
+require 'nibbler'
+require 'socket'
+
+class HadoopAttemptScraper < Nibbler
+  attr_accessor :task_id
+
+  def self.scrape_task(task_id)
+    task_id = task_id.to_s
+
+    url     = "http://#{Socket.gethostname}:50030/taskdetails.jsp?tipid=#{task_id}"
+    scrape  = parse(open(url))
+    scrape.task_id = task_id
+
+    scrape
+  end
+
+    elements 'table.jobtasks tbody > tr' => :attempts do
+    element 'td:nth-child(1)' => 'attempt_id'
+    element 'td:nth-child(2) a/@href' => 'machine'
+    element 'td:nth-child(3)' => 'status'
+    element 'td:nth-child(4)' => 'progress'
+    element 'td:nth-child(5)' => 'start_time'
+    element 'td:nth-child(6)' => 'finish_time'
+    element 'td:nth-child(7)' => 'errors'
+  end
+
+  def to_attempts
+    attempts.map do |attempt|
+      {
+        _id:         attempt.attempt_id.to_s,
+        task_id:     task_id,
+        host:        attempt.machine.to_s.gsub(/^http:\/\//, '').gsub(/:[0-9]+$/, ''),
+        status:      attempt.status,
+        progress:    attempt.progress.to_f / 100.0,
+        start_time:  Time.parse(attempt.start_time),
+        finish_time: attempt.finish_time.length > 0 ? Time.parse(attempt.finish_time) : nil,
+        errors:      attempt.errors
+      }
+    end
+  end
+end

data/scripts/hadoop_monitor/hadoop_client.rb

@@ -1,5 +1,7 @@
 require_relative 'configurable'
 require_relative 'hadoopable'
+require_relative 'hadoop_attempt_scraper'
+
 require 'json'
 require 'optparse'
 require 'ostruct'
@@ -22,7 +24,7 @@ module Vayacondios
       logger.info "Connecting to job tracker."
       @job_client = JobClient.new JobConf.new(get_hadoop_conf)
     end
-    
+
     #
     # (Equality doesn't work for jobs, so - will not work as intended
     # on arrays of jobs.)
@@ -34,7 +36,7 @@ module Vayacondios
     #
     # Returns the jobs with the specified state. States are specified
     # by constants in this class.
-    # 
+    #
     def jobs_with_state state
       jobs_by_state[state] || []
     end
@@ -46,19 +48,19 @@ module Vayacondios
       host_port = job.get_tracking_url[/^(http:\/\/)?[^\/]*/]
       job_id = job.get_id.to_s
       conf_uri = "#{host_port}/logs/#{job_id}_conf.xml"
-      properties = parse_properties(open conf_uri)
-      recordize_properties(properties, job_id.to_s)
+
+      parse_properties(open conf_uri)
     end
 
     #
     # Returns the stats for the current job as a hash.
     #
     def job_stats job, finish_time
-      parse_job job.get_id.to_s, finish_time
+      parse_job job.get_id, finish_time
     end
 
   private
-    
+
     #
     # Returns a hash JobStatus::<SOME_STATE> => <array of jobs>
     #
@@ -80,27 +82,41 @@ module Vayacondios
     # object that represents it.
     #
     def parse_job job_id, finish_time
-      job = @job_client.get_job job_id
-      job_status = @job_client.get_all_jobs.select{|j| j.get_job_id.to_s == job_id.to_s}.first
+      job             = @job_client.get_job job_id
+      job_status      = @job_client.get_all_jobs.select{|j| j.get_job_id.to_s == job_id.to_s}.first
       finished_status = [:FAILED, :KILLED, :COMPLETE]
       failed_status   = [:FAILED]
 
+      start_time      = Time.at(job_status.get_start_time / 1000)
+      reduce_progress = job.reduce_progress
+      map_progress    = job.map_progress
+      run_duration    = (finish_time || Time.now) - start_time
+
+      map_eta    = map_progress    && map_progress    > 0.0 ? (start_time + (run_duration / map_progress))    : nil
+      reduce_eta = reduce_progress && reduce_progress > 0.0 ? (start_time + (run_duration / reduce_progress)) : nil
+
       job_data = {
-        
+
         _id:              job_id.to_s,
+        name:             job.get_job_name.to_s,
 
                           # not sure what is what. I'm guessing
                           # JobStatus.getStartTime corresponds to the
                           # launch time in the logs, but I'm going to
                           # go ahead and use it twice here.
 
-        launch_time:      Time.at(job_status.get_start_time / 1000),
-        submit_time:      Time.at(job_status.get_start_time / 1000),
-
+        launch_time:      start_time,
+        submit_time:      start_time,
         finish_time:      finish_time,
 
+        run_duration:     run_duration,
+
+        map_eta:          map_eta,
+        reduce_eta:       reduce_eta,
+        eta:              reduce_eta,
+
         job_status:       case job_status.get_run_state
-                          when JobStatus::FAILED    then :FAILED 
+                          when JobStatus::FAILED    then :FAILED
                           when JobStatus::KILLED    then :KILLED
                           when JobStatus::PREP      then :PREP
                           when JobStatus::RUNNING   then :RUNNING
@@ -117,39 +133,45 @@ module Vayacondios
 
       }
 
-      job_progress = {
-        
-        parent_id:        job.job_id,
-        type:             :job_progress,
-                          # report time in milliseconds for consistency
-        time:             Time.now,
-        cleanup_progress: job.cleanup_progress,
-        map_progress:     job.map_progress,
-        reduce_progress:  job.reduce_progress,
-        setup_progress:   job.setup_progress,
-
+      job_event = {
+        t:                Time.now,
+        d: {
+          job_id:           job.job_id,
+          cleanup_progress: job.cleanup_progress,
+          map_progress:     job.map_progress,
+          reduce_progress:  job.reduce_progress,
+          setup_progress:   job.setup_progress,
+        }
       }
 
-      map_task_data    = @job_client.get_map_task_reports    job_id
-      reduce_task_data = @job_client.get_reduce_task_reports job_id
-
-      m_reports, m_progress_reports, r_reports, r_progress_reports =
-        [
-         map_task_data   .map{|task| parse_task          task, "MAP",    job_id },
-         map_task_data   .map{|task| parse_task_progress task, "MAP"            },
-         reduce_task_data.map{|task| parse_task          task, "REDUCE", job_id },
-         reduce_task_data.map{|task| parse_task_progress task, "REDUCE"         },
-        ]
-      
-      [job_data, job_progress] + m_reports + r_reports + m_progress_reports + r_progress_reports
-    end
+      setup_task_data   = @job_client.get_setup_task_reports    job_id
+      map_task_data     = @job_client.get_map_task_reports      job_id
+      reduce_task_data  = @job_client.get_reduce_task_reports   job_id
+      cleanup_task_data = @job_client.get_cleanup_task_reports  job_id
+
+      setup_reports       = setup_task_data.map{|task| parse_task task, "SETUP", job_id }
+      setup_event_reports = setup_task_data.map{|task| parse_task_progress task, "SETUP" }
+
+      map_reports       = map_task_data.map{|task| parse_task task, "MAP", job_id }
+      map_event_reports = map_task_data.map{|task| parse_task_progress task, "MAP" }
+
+      reduce_reports       = reduce_task_data.map{|task| parse_task task, "REDUCE", job_id }
+      reduce_event_reports = reduce_task_data.map{|task| parse_task_progress task, "REDUCE" }
+
+      cleanup_reports       = cleanup_task_data.map{|task| parse_task task, "CLEANUP", job_id }
+      cleanup_event_reports = cleanup_task_data.map{|task| parse_task_progress task, "CLEANUP" }
+
+      tasks = setup_reports + map_reports + reduce_reports + cleanup_reports
+      task_events = setup_event_reports + map_event_reports + reduce_event_reports + cleanup_event_reports
+
+      attempt_reports = tasks.map{|task| HadoopAttemptScraper.scrape_task(task[:_id]).to_attempts }.flatten
 
-    def recordize_properties properties, job_id
       {
-        parent_id:  job_id,
-        type:       :conf,
-        properties: properties,
-        _id:        [job_id, "_properties"].join
+        job: job_data,
+        job_event: job_event,
+        tasks: tasks,
+        task_events: task_events,
+        attempts: attempt_reports
       }
     end
 
@@ -174,25 +196,26 @@ module Vayacondios
     #
     def parse_task task_report, task_type, parent_job_id
       {
-        _id:              task_report.get_task_id.to_s,
-        parent_id:        parent_job_id,
-        task_type:        task_type,
-        task_status:      task_report.get_current_status.to_s,
-        start_time:       Time.at(task_report.get_start_time / 1000),
-        finish_time:      Time.at(task_report.get_finish_time / 1000),
-        counters:         parse_counters(task_report.get_counters),
-        type:             :task,
-        diagnostics:      task_report.get_diagnostics.map(&:to_s),
-        running_attempts: task_report.get_running_task_attempts.map(&:to_s),
+        _id:                   task_report.get_task_id.to_s,
+        job_id:                parent_job_id,
+        task_type:             task_type,
+        task_status:           task_report.get_current_status.to_s,
+        start_time:            Time.at(task_report.get_start_time / 1000),
+        finish_time:           task_report.get_finish_time > 0 ? Time.at(task_report.get_finish_time / 1000) : nil,
+        counters:              parse_counters(task_report.get_counters),
+        diagnostics:           task_report.get_diagnostics.map(&:to_s),
+        successful_attempt_id: task_report.get_successful_task_attempt.to_s
       }
     end
 
     def parse_task_progress task_report, task_type
       {
-        parent_id:   task_report.get_task_id.to_s,
-        time:        Time.now,
-        type:        :task_progress,
-        progress:    task_report.get_progress,
+        t: Time.now,
+        d: {
+          job_id:              task_report.get_task_id.to_s,
+          progress:            task_report.get_progress,
+          running_attempt_ids: task_report.get_running_task_attempts.map(&:to_s)
+        }
       }
     end
 

data/scripts/hadoop_monitor/hadoop_monitor.rb

@@ -15,46 +15,47 @@ module Vayacondios
   class HadoopMonitor
     def initialize
       init_settings
-      
+
       @hadoop = HadoopClient.new
-      
+
       @monitored_jobs = []
 
       logger.debug "Creating mongo collections."
       @conn = Mongo::Connection.new settings.mongo_ip
-      @db = @conn[settings.mongo_jobs_db]
-      @job_logs = @db.create_collection(settings.mongo_job_logs_collection)
+      @db = @conn[settings.mongo_db]
 
-      # After we create the job_events database, one of the machine
-      # monitors will create the machine stats databse.
-      @job_events = @db.create_collection(settings.mongo_job_events_collection,
-                                          :capped => true,
-                                          :size => settings.job_events_size)
+      capped_collection_opts = {
+        :capped => true,
+        :size => settings.mongo_collection_size
+      }
 
-      @cluster_state = CLUSTER_QUIET
+      @collections = {
+        jobs:     @db.create_collection('jobs'),
+        tasks:    @db.create_collection('job_tasks'),
+        attempts: @db.create_collection('job_task_attempts'),
+
+        job_events:     @db.create_collection('job_events',       capped_collection_opts),
+        task_events:    @db.create_collection('job_tasks_events', capped_collection_opts),
+      }
     end
 
     def run
       loop do
-        
+
         logger.debug "In main event loop."
 
-        cur_running_jobs  = @hadoop.jobs_with_state HadoopClient::RUNNING
-        cur_cluster_state = (cur_running_jobs.size > 0) ? CLUSTER_BUSY : CLUSTER_QUIET
+        running_jobs  = @hadoop.jobs_with_state HadoopClient::RUNNING
+        started_jobs  = @hadoop.subtract(running_jobs, @monitored_jobs)
+        finished_jobs = @hadoop.subtract(@monitored_jobs, running_jobs)
 
-        @hadoop.subtract(@monitored_jobs, cur_running_jobs).each do |job|
+        finished_jobs.each do |job|
           logger.debug "#{job.get_id.to_s} is complete."
           update_job_stats job, Time.now
         end
-        @hadoop.subtract(cur_running_jobs, @monitored_jobs).each do |job|
-          logger.debug "#{job.get_id.to_s} started."
-          update_job_properties job
-        end
 
-        (@monitored_jobs + cur_running_jobs).each{|job| update_job_stats job}
+        running_jobs.each{|job| update_job_stats job, nil, @hadoop.subtract([job], started_jobs).empty? }
 
-        @monitored_jobs = cur_running_jobs
-        update_cluster_state cur_cluster_state
+        @monitored_jobs = running_jobs
 
         sleep settings.sleep_seconds
 
@@ -65,27 +66,36 @@ module Vayacondios
 
     include Configurable
 
-    def update_cluster_state new_state
-      return if new_state == @cluster_state
-      @cluster_state = new_state
-      logger.info "Cluster state changed to #{@cluster_state}"
-      @job_events.insert(EVENT => @cluster_state, TIME => Time.now.to_i)
-    end
+    def update_job_stats job, finish_time = nil, include_properties = false
+      stats = @hadoop.job_stats(job, finish_time)
 
-    def update_job_properties job
-      properties = @hadoop.job_properties job
-      logger.debug "upserting #{JSON.generate properties}"
-      @job_logs.save(properties, upsert: true, safe: true)
-    end
+      if include_properties
+        stats[:job][:properties] = @hadoop.job_properties job
+      end
+
+      logger.debug "upserting job #{JSON.generate stats[:job]}"
+      @collections[:jobs].update({_id: stats[:job][:_id]}, stats[:job], upsert: true)
+
+      logger.debug "upserting job_event #{JSON.generate stats[:job_event]}"
+      @collections[:job_events].insert(stats[:job_event])
+
+      logger.debug "upserting tasks #{JSON.generate stats[:tasks]}"
+      stats[:tasks].each do |task|
+        @collections[:tasks].update({_id: task[:_id]}, task, upsert: true)
+      end
+
+      logger.debug "upserting task_events #{JSON.generate stats[:task_events]}"
+      stats[:task_events].each do |task_event|
+        @collections[:task_events].insert(task_event)
+      end
 
-    def update_job_stats job, finish_time = nil
-      @hadoop.job_stats(job, finish_time || Time.now).each do |job_stat|
-        logger.debug "upserting #{JSON.generate job_stat}"
-        @job_logs.save(job_stat, upsert: true, safe: true)
+      logger.debug "upserting attempts #{JSON.generate stats[:attempts]}"
+      stats[:attempts].each do |attempt|
+        @collections[:attempts].update({_id: attempt[:_id]}, attempt, upsert: true)
       end
     end
 
   end
-end  
+end
 
 Vayacondios::HadoopMonitor.new.run

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vayacondios-server
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -12,11 +12,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-05 00:00:00.000000000Z
+date: 2012-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: configliere
-  requirement: &2170198080 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -24,10 +24,15 @@ dependencies:
         version: 0.4.13
   type: :runtime
   prerelease: false
-  version_requirements: *2170198080
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.13
 - !ruby/object:Gem::Dependency
   name: gorillib
-  requirement: &2170196740 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -35,10 +40,15 @@ dependencies:
         version: 0.4.2
   type: :runtime
   prerelease: false
-  version_requirements: *2170196740
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.2
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &2170190120 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -46,10 +56,15 @@ dependencies:
         version: 1.0.0.beta.4
   type: :runtime
   prerelease: false
-  version_requirements: *2170190120
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0.beta.4
 - !ruby/object:Gem::Dependency
   name: goliath
-  requirement: &2170188640 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -57,10 +72,15 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2170188640
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: em-http-request
-  requirement: &2170187920 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -68,10 +88,15 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2170187920
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: em-mongo
-  requirement: &2170187240 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -79,10 +104,15 @@ dependencies:
         version: 0.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *2170187240
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.3
 - !ruby/object:Gem::Dependency
   name: foreman
-  requirement: &2170186560 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -90,10 +120,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2170186560
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2170185580 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -101,10 +136,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2170185580
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &2170184720 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -112,7 +152,12 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2170184720
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Simple enough to use in a shell script, performant enough to use everywhere.
   Dios mío! Record that metric, ese!
 email: 
@@ -159,6 +204,7 @@ files:
 - lib/vayacondios/server/rack/path_validation.rb
 - lib/vayacondios/version.rb
 - scripts/hadoop_monitor/configurable.rb
+- scripts/hadoop_monitor/hadoop_attempt_scraper.rb
 - scripts/hadoop_monitor/hadoop_client.rb
 - scripts/hadoop_monitor/hadoop_monitor.rb
 - scripts/hadoop_monitor/hadoopable.rb
@@ -193,7 +239,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3537834639285918956
+      hash: 1142154938423914035
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -202,10 +248,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3537834639285918956
+      hash: 1142154938423914035
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Data goes in. The right thing happens