presto-metrics 0.3.9 → 0.3.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a3c3d3f74169faa3e4cae0abe8eeb502faea7585
-  data.tar.gz: f2284da30958c56c7c23924379b159660a4c9e23
+  metadata.gz: 76738b2193dac8ff0c1b85ab37c8d3773e7c1540
+  data.tar.gz: 024bef2488910dca9483240bc6fca29fe9d24c8d
 SHA512:
-  metadata.gz: 8fafeb893d3f45120d0ad23a27359be2df3ebdc74d1dbe982c2f5a97c04c0ac43868032ee499e0ac29e4ba66f0b819d8e140544c4e1348ea4c66f889d7871643
-  data.tar.gz: eba6f37b29ccc3f3e2eca0967c28ec70d4b25a3c2cf3aab8f78dbc5d9eb89fc2e76934ca5c5dcdec0bf444dd6091f84e9f4dc4a3ad6e62dc0f529f8edc903284
+  metadata.gz: 24cd82ee453475715789b9469458514d380b6f6027c9db7e912a60b18e0cb24b63df74da2700e85db2abfe08f73229bc10627ff35a2dd00913ec1d76e15fbaaf
+  data.tar.gz: 1deb4275a340a4162bf0a9111b6723acb1cfe949c179631b34f79bde31d8fa3df63863f88bd70d2524f73f6d879685636935738676f48454de0a5a3ed627dd31

data/README.md

@@ -66,17 +66,15 @@ client.query_execution_metrics   # com.facebook.presto.execution:name=QueryExecu
 client.node_scheduler_metrics    # com.facebook.presto.execution:name=NodeScheduler
 client.task_executor_metrics     # com.facebook.presto.execution:name=TaskExecutor
 client.task_manager_metrics      # com.facebook.presto.execution:name=TaskManager
-
+client.cluster_memory_manager_metrics # com.facebook.presto.memory:name=ClusterMemoryManager
 
 # Memory resource manager metrics (since Presto 0.103)
-```ruby
 client.memory_pool_metrics
 => {"general"=>{"free_bytes"=>59700045415, "max_bytes"=>59700045415}, "reserved"=>{"free_bytes"=>2147483648, "max_bytes"=>2147483648}}
 
 client.cluster_memory_pool_metrics
 => {"general"=>{"blocked_nodes"=>0, "free_distributed_bytes"=>116833832981, "nodes"=>4, "queries"=>8, "total_distributed_bytes"=>131855495989},
     "reserved"=>{"blocked_nodes"=>0, "free_distributed_bytes"=>15032385536, "nodes"=>4, "queries"=>0, "total_distributed_bytes"=>15032385536}}
-```
 
 
 # Retrieve presto worker metrics
@@ -102,6 +100,107 @@ pp client.node_metrics
   "age"=>"38.56m",
   "recent_failures_by_type"=>{}}]
 
+# Garbage Collection metrics
+pp client.gc_g1_metrics
+{"g1_old_generation"=>
+  {"last_gc_info"=>
+    {"gc_thread_count"=>28,
+     "duration"=>478,
+     "end_time"=>9530,
+     "id"=>3,
+     "memory_usage_after_gc"=>
+      {"g1_survivor_space"=>{"committed"=>0, "init"=>0, "max"=>-1, "used"=>0},
+       "metaspace"=>
+        {"committed"=>60444672, "init"=>0, "max"=>-1, "used"=>56490104},
+       "g1_old_gen"=>
+        {"committed"=>134217728,
+         "init"=>1895825408,
+         "max"=>103079215104,
+         "used"=>75768752},
+       "g1_eden_space"=>
+        {"committed"=>134217728, "init"=>117440512, "max"=>-1, "used"=>0},
+       "code_cache"=>
+        {"committed"=>28114944,
+         "init"=>2555904,
+         "max"=>314572800,
+         "used"=>27676416}},
+     "memory_usage_before_gc"=>
+      {"g1_survivor_space"=>
+        {"committed"=>33554432, "init"=>0, "max"=>-1, "used"=>33554432},
+       "metaspace"=>
+        {"committed"=>60444672, "init"=>0, "max"=>-1, "used"=>56829560},
+       "g1_old_gen"=>
+        {"committed"=>234881024,
+         "init"=>1895825408,
+         "max"=>103079215104,
+         "used"=>10082520},
+       "g1_eden_space"=>
+        {"committed"=>402653184,
+         "init"=>117440512,
+         "max"=>-1,
+         "used"=>318767104},
+       "code_cache"=>
+        {"committed"=>28114944,
+         "init"=>2555904,
+         "max"=>314572800,
+         "used"=>27676416}},
+     "start_time"=>9052},
+   "collection_count"=>3,
+   "collection_time"=>1624,
+   "memory_pool_names"=>["G1 Eden Space", "G1 Survivor Space", "G1 Old Gen"],
+   "valid"=>true,
+   "name"=>"G1 Old Generation",
+   "object_name"=>"java.lang:type=GarbageCollector,name=G1 Old Generation"},
+ "g1_young_generation"=>
+  {"last_gc_info"=>
+    {"gc_thread_count"=>28,
+     "duration"=>26,
+     "end_time"=>7034504,
+     "id"=>145,
+     "memory_usage_after_gc"=>
+      {"g1_survivor_space"=>
+        {"committed"=>100663296, "init"=>0, "max"=>-1, "used"=>100663296},
+       "metaspace"=>
+        {"committed"=>73138176, "init"=>0, "max"=>-1, "used"=>67016896},
+       "g1_old_gen"=>
+        {"committed"=>5989466112,
+         "init"=>1895825408,
+         "max"=>103079215104,
+         "used"=>3212283664},
+       "g1_eden_space"=>
+        {"committed"=>9512681472, "init"=>117440512, "max"=>-1, "used"=>0},
+       "code_cache"=>
+        {"committed"=>63045632,
+         "init"=>2555904,
+         "max"=>314572800,
+         "used"=>62560832}},
+     "memory_usage_before_gc"=>
+      {"g1_survivor_space"=>
+        {"committed"=>83886080, "init"=>0, "max"=>-1, "used"=>83886080},
+       "metaspace"=>
+        {"committed"=>73138176, "init"=>0, "max"=>-1, "used"=>67016896},
+       "g1_old_gen"=>
+        {"committed"=>5771362304,
+         "init"=>1895825408,
+         "max"=>103079215104,
+         "used"=>3206420472},
+       "g1_eden_space"=>
+        {"committed"=>9747562496,
+         "init"=>117440512,
+         "max"=>-1,
+         "used"=>9277800448},
+       "code_cache"=>
+        {"committed"=>63045632,
+         "init"=>2555904,
+         "max"=>314572800,
+         "used"=>62560832}},
+     "start_time"=>7034478},
+   "collection_count"=>145,
+   "collection_time"=>8946,
+   "memory_pool_names"=>["G1 Eden Space", "G1 Survivor Space"],
+   "valid"=>true,
+   "name"=>"G1 Young Generation",
+   "object_name"=>"java.lang:type=GarbageCollector,name=G1 Young Generation"}}
 
 # Retrieve the JSON representation of JMX properties
 client.get_mbean_json("java.lang:Type=Memory")

data/lib/presto/metrics/query.rb

@@ -9,69 +9,69 @@ include Presto::Client::Models
 module Presto
   module Metrics
 
-  	class Query 
-  		def initialize(client)
-  			@client = client
-  		end
-
-  		def format_table(tbl, label, align, sep)
-  			# Compute col length
-  			col = {}
-  			label.each_with_index{|l, i| col[i] = l.length}
-  			tbl.each{|row|
-  				row.each_with_index {|cell, i| 
-  					l = cell.to_s.size if cell 
-  					l ||= 0
-  					col[i] ||= l
-  					col[i] = [[col[i], l].max, 150].min
-  				}
-  			}
-  			# print label
-  			line = []
-  			label.each_with_index{|l, i|
-  				line << l.to_s[0..col[i]].ljust(col[i])
-  			}
-  			puts line.join(sep)
-  			tbl.each{|row|
-  				line = []
-  				row.each_with_index{|cell, i|
-  					str = cell.to_s[0..col[i]]
-  					a = align[i] || 'l'
-  					case a 
-  					when 'r'
-  						line << str.rjust(col[i])
-  					when 'l'
-  						line << str.ljust(col[i])
-  					else
-  						line << str.ljust(col[i])
-  					end
-  				}
-  				puts line.join(sep)
-  			}
-  			0
-  		end
-
-	  	def list
-	  		ql = query_list
-	  		tbl = ql.map {|q|
-	  			s = q['session'] || {}
-	  			query = q['query'].gsub(/[\t\r\n]/, ' ').gsub(/ {1,}/, ' ').strip
-	  			[q['queryId'], q['elapsedTime'], q['state'], q['runningDrivers'], q['completedDrivers'], q['totalDrivers'], s['user'], s['catalog'], s['schema'], s['source'], query]
-	  		}.sort_by{|row| row[0]}.reverse
-
-				format_table(tbl,
-						%w|query time state r f t user catalog schema source sql|,
-						%w|r     r    r     r r r l    l       r      l      l  |,
-						' '
-	  		)
-	  	end
-
-	  	def find(queryId)
-	  		JSON.parse(@client.get_query_json(queryId, "{}"))
-	  	end
-
-	  	def task_list(queryId)
-	  		qj = find(queryId) || {}
+    class Query
+      def initialize(client)
+        @client = client
+      end
+
+      def format_table(tbl, label, align, sep)
+        # Compute col length
+        col = {}
+        label.each_with_index { |l, i| col[i] = l.length }
+        tbl.each { |row|
+          row.each_with_index { |cell, i|
+            l = cell.to_s.size if cell
+            l ||= 0
+            col[i] ||= l
+            col[i] = [[col[i], l].max, 150].min
+          }
+        }
+        # print label
+        line = []
+        label.each_with_index { |l, i|
+          line << l.to_s[0..col[i]].ljust(col[i])
+        }
+        puts line.join(sep)
+        tbl.each { |row|
+          line = []
+          row.each_with_index { |cell, i|
+            str = cell.to_s[0..col[i]]
+            a = align[i] || 'l'
+            case a
+              when 'r'
+                line << str.rjust(col[i])
+              when 'l'
+                line << str.ljust(col[i])
+              else
+                line << str.ljust(col[i])
+            end
+          }
+          puts line.join(sep)
+        }
+        0
+      end
+
+      def list
+        ql = query_list
+        tbl = ql.map { |q|
+          s = q['session'] || {}
+          query = q['query'].gsub(/[\t\r\n]/, ' ').gsub(/ {1,}/, ' ').strip
+          [q['queryId'], q['elapsedTime'], q['state'], q['runningDrivers'], q['completedDrivers'], q['totalDrivers'], s['user'], s['catalog'], s['schema'], s['source'], query]
+        }.sort_by { |row| row[0] }.reverse
+
+        format_table(tbl,
+            %w|query time state r f t user catalog schema source sql|,
+            %w|r     r    r     r r r l    l       r      l      l  |,
+            ' '
+        )
+      end
+
+      def find(queryId)
+        JSON.parse(@client.get_query_json(queryId, "{}"))
+      end
+
+      def task_list(queryId)
+        qj = find(queryId) || {}
         root_stage = qj['outputStage'] || {}
         tasks = root_stage['tasks'] || []
         tasks << find_tasks(root_stage['subStages'] || [])
@@ -80,87 +80,83 @@ module Presto
 
       def tasks(queryId)
         tl = task_list(queryId)
-        stats = tl.map {|t|
+        stats = tl.map { |t|
           s = t['stats']
           host = (t['self'] || '').sub(/http:\/\/([a-z0-9\-.]+[\/:][0-9]+)\/.*/, '\1')
           [t['taskId'], host, t['state'], s['rawInputPositions'], s['rawInputDataSize'], s['queuedDrivers'], s['runningDrivers'], s['completedDrivers']]
         }
         format_table(stats,
-                     %w|task_id host    state processed_rows size|,
-                     %w|l       l       l     r          r|,
-										 ' '
+            %w|task_id host    state processed_rows size|,
+            %w|l       l       l     r          r|,
+            ' '
         )
       end
 
-	  	def query_list(path="")
-	  		JSON.parse(@client.get_query_json(path))
-	  	end
+      def query_list(path="")
+        JSON.parse(@client.get_query_json(path))
+      end
 
       def running_list
-        ql = query_list.select{|q| q['state'] == 'RUNNING'}.sort_by{|row| row[0]}.reverse
-        ql.each{|q|
+        ql = query_list.select { |q| q['state'] == 'RUNNING' }.sort_by { |row| row[0] }.reverse
+        ql.each { |q|
           tasks(q['queryId'])
         }
         ql.size
-			end
-
-
-			def count_total_processed_rows(stage)
-				return 0 unless stage
-				rows = (stage.tasks || []).map{|t| t.stats.raw_input_positions }.inject(0, :+)
-				rows + (stage.sub_stages || []).map{|ss| count_total_processed_rows(ss) }.inject(0, :+)
-			end
-
-			def processed_rows(query_id)
-				qi = QueryInfo.decode(find(query_id))
-				count_total_processed_rows(qi.output_stage)
-			end
-
-			def query_progress
-				running_queries = query_list.map{|q| QueryInfo.decode(q) }.select{|q| q.state == :running }
-		    query_info = running_queries.map{|q|
-					queryInfo = find(q.query_id)
-					QueryInfo.decode(queryInfo)
-				}
-
-				result = {}
-				query_info.each{|q|
-				   os = q.output_stage
-					 result[q.query_id] = count_total_processed_rows(os)
-				}
-				result
-			end
-
-	  	def metrics 
-	  		ql = query_list
-	  		ql.map{|qi|
-	  			h = {}
-	  			h['query_id'] = qi['queryId'] || ''
-	  			h['state'] = qi['state'] || ''
-	  			session = qi['session'] || {}
-	  			h['source'] = session['source'] || ''
-	  			h['user'] = session['user'] || h['source'].gsub(/[^a-zA-Z0-9]/,'')
-	  			h['running_drivers'] = qi['runningDrivers'] || 0
-	  			h['queued_drivers'] = qi['queuedDrivers'] || 0
-	  			h['completed_drivers'] = qi['completedDrivers'] || 0
-	  			h['total_drivers'] = qi['totalDrivers'] || 0
-	  			h['elapsed_time'] = qi['elapsedTime'] || '0.0m'
-	  			h['create_time'] = qi['createTime']
-	  			h['running_time'] = qi['endTime'] || Time.now.utc.iso8601(3)
-	  			#if(h['state'] == "FAILED")
-	  			#	h['errorCode'] = find(h['query_id'])['errorCode'] || {}
-	  			#end
-	  			h
-	  		}
       end
 
+      def count_total_processed_rows(stage)
+        return 0 unless stage
+        rows = stage.stage_stats.raw_input_positions + stage.stage_stats.output_positions
+        rows + (stage.sub_stages || []).map { |ss| count_total_processed_rows(ss) }.inject(0, :+)
+      end
+
+      def processed_rows(query_id)
+        qi = QueryInfo.decode(find(query_id))
+        count_total_processed_rows(qi.output_stage)
+      end
 
+      def query_progress
+        running_queries = query_list.map { |q| QueryInfo.decode(q) }.select { |q| q.state == :running }
+        query_info = running_queries.map { |q|
+          queryInfo = find(q.query_id)
+          QueryInfo.decode(queryInfo)
+        }
 
+        result = {}
+        query_info.each { |q|
+          os = q.output_stage
+          result[q.query_id] = count_total_processed_rows(os)
+        }
+        result
+      end
+
+      def metrics
+        ql = query_list
+        ql.map { |qi|
+          h = {}
+          h['query_id'] = qi['queryId'] || ''
+          h['state'] = qi['state'] || ''
+          session = qi['session'] || {}
+          h['source'] = session['source'] || ''
+          h['user'] = session['user'] || h['source'].gsub(/[^a-zA-Z0-9]/, '')
+          h['running_drivers'] = qi['runningDrivers'] || 0
+          h['queued_drivers'] = qi['queuedDrivers'] || 0
+          h['completed_drivers'] = qi['completedDrivers'] || 0
+          h['total_drivers'] = qi['totalDrivers'] || 0
+          h['elapsed_time'] = qi['elapsedTime'] || '0.0m'
+          h['create_time'] = qi['createTime']
+          h['running_time'] = qi['endTime'] || Time.now.utc.iso8601(3)
+          #if(h['state'] == "FAILED")
+          #	h['errorCode'] = find(h['query_id'])['errorCode'] || {}
+          #end
+          h
+        }
+      end
 
       def find_tasks(sub_stages)
         task_list = []
         return task_list unless sub_stages
-        sub_stages.each{|ss|
+        sub_stages.each { |ss|
           tl = ss['tasks']
           task_list << tl if tl
           task_list << find_tasks(ss['subStages'])

data/lib/presto/metrics/version.rb

@@ -1,5 +1,5 @@
 module Presto
   module Metrics
-    VERSION = '0.3.9'
+    VERSION = '0.3.10'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: presto-metrics
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 0.3.10
 platform: ruby
 authors:
 - Taro L. Saito
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-12 00:00:00.000000000 Z
+date: 2015-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler