presto-metrics 0.3.8 → 0.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dbbf348083479afcf19bae2f803048d9670ce93e
-  data.tar.gz: 394b1e332c7d5c951194dfa042ca1d48ac5d6f4d
+  metadata.gz: a3c3d3f74169faa3e4cae0abe8eeb502faea7585
+  data.tar.gz: f2284da30958c56c7c23924379b159660a4c9e23
 SHA512:
-  metadata.gz: 5fb347c82a24cdd4ba8852784903ab788ea353475e5e1ae808f673895bcd09e6b5f92c2e6e1fc0d52270d1918e7fc671b8e61b85c63d08ad698549ccbff4edb1
-  data.tar.gz: 53bcaf2db51f5076a071b8eca41685158af78a8d4a215de271c867241466f53eb612b5370d0d12b65fc2e9491c84db8e4b49ebf2c6e8da1c1814a71f34c4cabb
+  metadata.gz: 8fafeb893d3f45120d0ad23a27359be2df3ebdc74d1dbe982c2f5a97c04c0ac43868032ee499e0ac29e4ba66f0b819d8e140544c4e1348ea4c66f889d7871643
+  data.tar.gz: eba6f37b29ccc3f3e2eca0967c28ec70d4b25a3c2cf3aab8f78dbc5d9eb89fc2e76934ca5c5dcdec0bf444dd6091f84e9f4dc4a3ad6e62dc0f529f8edc903284

data/README.md

@@ -32,7 +32,6 @@ client = Presto::Metrics::Client.new  # Access to http://localhost:8080 in defau
 client = Presto::Metrics::Client.new(:host => "localhost", :port=>8080) 
 
 
-
 client.os_metrics
 # => {:open_file_descriptor_count=>360, :max_file_descriptor_count=>10240, :committed_virtual_memory_size=>18683629568, :total_swap_space_size=>2147483648, :free_swap_space_size=>1132986368, :process_cpu_time=>240244441000, :free_physical_memory_size=>2088931328, :total_physical_memory_size=>17179869184, :system_cpu_load=>0.044989775051124746, :process_cpu_load=>0.002293214043176635, :name=>"Mac OS X", :version=>"10.9.4", :available_processors=>8, :arch=>"x86_64", :system_load_average=>2.0537109375, :object_name=>"java.lang:type=OperatingSystem"}
 
@@ -61,13 +60,25 @@ client.memory_usage_metrics      # java.lang:Type=Memory
 client.os_metrics                # java.lang:type=OperatingSystm
 client.gc_cms_metrics            # java.lang:type=GarbageCollector,name=ConcurrentMarkSweep
 client.gc_parnew_metrics         # java.lang:type=GarbageCollector,name=ParNew
-client.gc_g1_metrics             # java.lang:type=GarbageCollector,name=G1
+client.gc_g1_metrics             # java.lang:type=GarbageCollector,name=G1 Young Generation and G1 Old Genration
 client.query_manager_metrics     # com.facebook.presto.execution:name=QueryManager
 client.query_execution_metrics   # com.facebook.presto.execution:name=QueryExecution
 client.node_scheduler_metrics    # com.facebook.presto.execution:name=NodeScheduler
 client.task_executor_metrics     # com.facebook.presto.execution:name=TaskExecutor
 client.task_manager_metrics      # com.facebook.presto.execution:name=TaskManager
 
+
+# Memory resource manager metrics (since Presto 0.103)
+```ruby
+client.memory_pool_metrics
+=> {"general"=>{"free_bytes"=>59700045415, "max_bytes"=>59700045415}, "reserved"=>{"free_bytes"=>2147483648, "max_bytes"=>2147483648}}
+
+client.cluster_memory_pool_metrics
+=> {"general"=>{"blocked_nodes"=>0, "free_distributed_bytes"=>116833832981, "nodes"=>4, "queries"=>8, "total_distributed_bytes"=>131855495989},
+    "reserved"=>{"blocked_nodes"=>0, "free_distributed_bytes"=>15032385536, "nodes"=>4, "queries"=>0, "total_distributed_bytes"=>15032385536}}
+```
+
+
 # Retrieve presto worker metrics
 pp client.node_metrics
 [{"host"=>"xxx.xxx.xxx.xxx",
@@ -93,7 +104,7 @@ pp client.node_metrics
 
 
 # Retrieve the JSON representation of JMX properties
-client.get_json("java.lang:Type=Memory")
+client.get_mbean_json("java.lang:Type=Memory")
 
 # Pretty print 
 require 'pp'
@@ -125,3 +136,19 @@ pp c.memory_usage_metrics
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create a new Pull Request
+
+
+## For developers
+
+To develop presto-metrics
+
+```
+$ bundle exec irb -I lib -r presto/metrics.rb
+
+irb> client = Presto::Metrics::Client.new(:host => "ec2-54-205-192-36.compute-1.amazonaws.com", :port=>8080)
+
+# reload modified code
+irb> load "presto/metrics/client.rb"
+
+```
+

data/lib/presto/metrics/client.rb

@@ -7,6 +7,7 @@ module Presto
         require 'json'
         require 'set'
         require 'time'
+        require 'uri'
 
         @host = opts[:host] || 'localhost'
         @port = opts[:port] || '8080'
@@ -26,7 +27,12 @@ module Presto
           'query_execution' => 'com.facebook.presto.execution:name=QueryExecution',
           'node_scheduler' => 'com.facebook.presto.execution:name=NodeScheduler',
           'task_executor' => 'com.facebook.presto.execution:name=TaskExecutor',
-          'task_manager' => 'com.facebook.presto.execution:name=TaskManager'
+          'task_manager' => 'com.facebook.presto.execution:name=TaskManager',
+          'memory_pool_general' => 'com.facebook.presto.memory:type=MemoryPool,name=general',
+          'memory_pool_reserved' => 'com.facebook.presto.memory:type=MemoryPool,name=reserved',
+          'cluster_memory_manager' => 'com.facebook.presto.memory:name=ClusterMemoryManager',
+          'cluster_memory_pool_general' => 'com.facebook.presto.memory:type=ClusterMemoryPool,name=general',
+          'cluster_memory_pool_reserved' => 'com.facebook.presto.memory:type=ClusterMemoryPool,name=reserved',
       }
 
       def path(path)
@@ -71,7 +77,7 @@ module Presto
       end
 
       def get(path, default='{}')
-        resp = HTTParty.get("#{@endpoint}#{path}")
+        resp = HTTParty.get(URI.encode("#{@endpoint}#{path}"))
         if resp.code == 200
           resp.body
         else
@@ -112,6 +118,7 @@ module Presto
         str.gsub(/::/, '/').
             gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').
             gsub(/([a-z\d])([A-Z])/, '\1_\2').
+            gsub(/ +/,'_').
             tr('-', '_').
             downcase
       end
@@ -132,20 +139,55 @@ module Presto
         kv
       end
 
+      def sanitize_key(k)
+        @caml_case ? k : underscore(k)
+      end
+
+      def extract_value(v)
+        if v.is_a?(Hash)
+          if v.has_key?('key') && v.has_key?('value')
+            {sanitize_key(v['key']) => extract_value(v['value']) }
+          else
+            v.each_with_object({}) {|(k1, v1), h2|
+              h2[sanitize_key(k1)] = extract_value(v1)
+            }
+          end
+        elsif v.is_a?(Array) && v.all?{|e| e.is_a?(Hash) }
+          v.each_with_object({}){|e, h| h.merge!(extract_value(e)) }
+        else
+          v
+        end
+      end
+
+      def get_gc_metrics(mbean, target_attr=[])
+        h = {}
+        get_attributes(mbean)
+            .reject { |attr| attr['name'].nil? || attr['value'].nil? }
+            .each { |attr|
+          key = sanitize_key(attr['name'])
+          v = attr['value']
+          h[key] = extract_value(v)
+        }
+        h
+      end
+
+
       def memory_usage_metrics(target_attr=[])
         get_metrics('java.lang:type=Memory', target_attr)
       end
 
       def gc_cms_metrics(target_attr=[])
-        get_metrics('java.lang:type=GarbageCollector,name=ConcurrentMarkSweep', target_attr)
+        get_gc_metrics('java.lang:type=GarbageCollector,name=ConcurrentMarkSweep', target_attr)
       end
 
       def gc_parnew_metrics(target_attr=[])
-        get_metrics('java.lang:type=GarbageCollector,name=ParNew', target_attr)
+        get_gc_metrics('java.lang:type=GarbageCollector,name=ParNew', target_attr)
       end
 
       def gc_g1_metrics(target_attr=[])
-        get_metrics('java.lang:type=GarbageCollector,name=G1', target_attr)
+        ['G1 Old Generation', 'G1 Young Generation'].each_with_object({}){|gen, h|
+          h[sanitize_key(gen)] = get_gc_metrics("java.lang:type=GarbageCollector,name=#{gen}", target_attr)
+        }
       end
 
       def os_metrics(target_attr=[])
@@ -172,6 +214,23 @@ module Presto
         get_metrics('com.facebook.presto.execution:name=TaskManager', target_attr)
       end
 
+      def memory_pool_metrics(target_attr=[])
+        ['general', 'reserved'].each_with_object({}) {|type, hash|
+          hash[type] = get_metrics("com.facebook.presto.memory:type=MemoryPool,name=#{type}", target_attr)
+        }
+      end
+
+      def cluster_memory_pool_metrics(target_attr=[])
+        ['general', 'reserved'].each_with_object({}) {|type, hash|
+          hash[type] = get_metrics("com.facebook.presto.memory:type=ClusterMemoryPool,name=#{type}", target_attr)
+        }
+      end
+
+      def cluster_memory_manager_metrics(target_attr=[])
+        get_metrics("com.facebook.presto.memory:name=ClusterMemoryManager", target_attr)
+      end
+
+
       def node_metrics(target_attr=[])
         p = URI::Parser.new
         node_state = JSON.parse(get_node_json)

data/lib/presto/metrics/query.rb

@@ -59,11 +59,10 @@ module Presto
 	  			[q['queryId'], q['elapsedTime'], q['state'], q['runningDrivers'], q['completedDrivers'], q['totalDrivers'], s['user'], s['catalog'], s['schema'], s['source'], query]
 	  		}.sort_by{|row| row[0]}.reverse
 
-	  		format_table(
-	  			tbl, 
-	  			:label => %w|query time state r f t user catalog schema source sql|,
-	  			:align => %w|r     r    r     r r r l    l       r      l      l  |,
-					:sep => ' '
+				format_table(tbl,
+						%w|query time state r f t user catalog schema source sql|,
+						%w|r     r    r     r r r l    l       r      l      l  |,
+						' '
 	  		)
 	  	end
 
@@ -73,9 +72,9 @@ module Presto
 
 	  	def task_list(queryId)
 	  		qj = find(queryId) || {}
-        root_stage = qj['outputStage'] || []
+        root_stage = qj['outputStage'] || {}
         tasks = root_stage['tasks'] || []
-        tasks << find_tasks(root_stage['subStages'])
+        tasks << find_tasks(root_stage['subStages'] || [])
         tasks.flatten
       end
 
@@ -87,9 +86,9 @@ module Presto
           [t['taskId'], host, t['state'], s['rawInputPositions'], s['rawInputDataSize'], s['queuedDrivers'], s['runningDrivers'], s['completedDrivers']]
         }
         format_table(stats,
-                     :label => %w|task_id host    state processed_rows size|,
-                     :align => %w|l       l       l     r          r|,
-										 :sep => ' '
+                     %w|task_id host    state processed_rows size|,
+                     %w|l       l       l     r          r|,
+										 ' '
         )
       end
 

data/lib/presto/metrics/version.rb

@@ -1,5 +1,5 @@
 module Presto
   module Metrics
-    VERSION = '0.3.8'
+    VERSION = '0.3.9'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: presto-metrics
 version: !ruby/object:Gem::Version
-  version: 0.3.8
+  version: 0.3.9
 platform: ruby
 authors:
 - Taro L. Saito
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-25 00:00:00.000000000 Z
+date: 2015-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler