hadoop-metrics 0.1.2 → 0.1.3

data/lib/hadoop_metrics.rb

@@ -1,2 +1,3 @@
-require 'hadoop_metrics/job_tracker'
-require 'hadoop_metrics/task_tracker'
+require "hadoop_metrics/job_tracker"
+require "hadoop_metrics/task_tracker"
+require "hadoop_metrics/name_node"

data/lib/hadoop_metrics/api.rb

@@ -6,14 +6,20 @@ module HadoopMetrics
     def initialize(host, port, opts = {})
       @endpoint = "#{host}:#{port}"
       @metrics_endpoint = URI("http://#{@endpoint}/metrics?format=json")
-      @snake_case = opts[:snake_case] || true
+      @snake_case = opts.has_key?(:snake_case) ? opts[:snake_case] : true
       @name = opts[:name] || host
+      @metrics_cache = nil
     end
 
     attr_reader :name
 
-    def metrics
-      HadoopMetrics.get_response(@metrics_endpoint)
+    def metrics(force = true)
+      if !@metrics_cache.nil? and !force
+        return @metrics_cache
+      end
+
+      @metrics_cache = HadoopMetrics.get_response(@metrics_endpoint)
+      @metrics_cache
     end
 
     def gc
@@ -44,8 +50,12 @@ module HadoopMetrics
 
     private
 
-    def group_by(category, target, column)
-      categories = metrics[category]
+    def get_force(opts)
+      opts.has_key?(:force) ? opts[:force] : true
+    end
+
+    def group_by(category, target, column, force)
+      categories = metrics(force)[category]
       return nil if categories.nil?
 
       targets = categories[target]

data/lib/hadoop_metrics/job_tracker.rb

@@ -1,4 +1,4 @@
-require 'hadoop_metrics/api'
+require "hadoop_metrics/api"
 
 module HadoopMetrics
   class JobTracker
@@ -10,19 +10,27 @@ module HadoopMetrics
       via_jmx('hadoop:service=JobTracker,name=JobTrackerInfo', JSON_FILED_VALUES).first
     end
 
-    def fairscheduler_pools(column = 'name')
-      group_by('fairscheduler', 'pools', column)
+    def mapred(opts = {})
+      disable_snake_case {
+        group_by('mapred', 'jobtracker', 'hostName', get_force(opts)).each_pair { |k, v|
+          return v.first
+        }
+      }
+    end
+
+    def fairscheduler_pools(opts = {})
+      group_by('fairscheduler', 'pools', get_column(opts), get_force(opts))
     end
 
-    def fairscheduler_jobs(column = 'name')
-      group_by('fairscheduler','jobs', column)
+    def fairscheduler_jobs(opts = {})
+      group_by('fairscheduler','jobs', get_column(opts), get_force(opts))
     end
 
-    def fairscheduler_running_tasks(target = 'pools')
-      fs = metrics['fairscheduler']
+    def fairscheduler_running_tasks(opts = {})
+      fs = metrics(get_force(opts))['fairscheduler']
       return nil if fs.nil?
 
-      targets = fs[target]
+      targets = fs[get_target(opts)]
       return nil if targets.nil?
 
       each_tasks = {}
@@ -33,5 +41,15 @@ module HadoopMetrics
       }
       each_tasks
     end
+
+    private
+
+    def get_column(opts)
+      opts[:column] || 'name'
+    end
+
+    def get_target(opts)
+      opts[:target] || 'pools'
+    end
   end
 end

data/lib/hadoop_metrics/name_node.rb

@@ -0,0 +1,17 @@
+require "hadoop_metrics/api"
+
+module HadoopMetrics
+  class NameNode
+    include API
+
+    JSON_FILED_VALUES = %W(LiveNodes DeadNodes DecomNodes NameDirStatuses)
+
+    def info
+      via_jmx('Hadoop:service=NameNode,name=NameNodeInfo', JSON_FILED_VALUES).first
+    end
+
+    def dfs
+      via_jmx('Hadoop:service=NameNode,name=FSNamesystem').first
+    end
+  end
+end

data/lib/hadoop_metrics/task_tracker.rb

@@ -1,4 +1,4 @@
-require 'hadoop_metrics/api'
+require "hadoop_metrics/api"
 
 module HadoopMetrics
   class TaskTracker
@@ -10,14 +10,22 @@ module HadoopMetrics
       via_jmx('hadoop:service=TaskTracker,name=TaskTrackerInfo', JSON_FILED_VALUES).first
     end
 
-    def shuffle_output(column = 'sessionId')
+    def shuffle_output(opts = {})
+      column = get_column(opts)
       column = HadoopMetrics.to_snake_case(column) if @snake_case
-      group_by('mapred', 'shuffleOutput', column)
+      group_by('mapred', 'shuffleOutput', column, get_force(opts))
     end
 
-    def mapred(column = 'sessionId')
+    def mapred(opts = {})
+      column = get_column(opts)
       column = HadoopMetrics.to_snake_case(column) if @snake_case
-      group_by('mapred', 'tasktracker', column)
+      group_by('mapred', 'tasktracker', column, get_force(opts))
+    end
+
+    private
+
+    def get_column(opts)
+      opts[:column] || 'sessionId'
     end
   end
 end

data/lib/hadoop_metrics/version.rb

@@ -1,3 +1,3 @@
 module HadoopMetrics
-  VERSION = '0.1.2'
+  VERSION = '0.1.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hadoop-metrics
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-30 00:00:00.000000000 Z
+date: 2013-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -42,6 +42,7 @@ files:
 - lib/hadoop_metrics.rb
 - lib/hadoop_metrics/api.rb
 - lib/hadoop_metrics/job_tracker.rb
+- lib/hadoop_metrics/name_node.rb
 - lib/hadoop_metrics/task_tracker.rb
 - lib/hadoop_metrics/version.rb
 homepage: https://github.com/repeatedly/hadoop-metrics
@@ -58,7 +59,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -596763061643403807
+      hash: -2921215695546108603
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -67,7 +68,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -596763061643403807
+      hash: -2921215695546108603
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23