presto-metrics 0.2.6 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c8592a3b20c830df773dd674d2788d97a1efcb2f
-  data.tar.gz: a52c98446f0a94b05802382aa89c41369521feb8
+  metadata.gz: f68643372b784275eac1dd75ec8c9e4bd899f177
+  data.tar.gz: 6e5aca1a188c6a526175c8e6b2a4af254b46132a
 SHA512:
-  metadata.gz: 77428d020bf697818cee9b087315dd75c02a7e65dd5ad1aed61b647c78477488eb674a4c626d9f0971c152f7c398d53b3f3f2278761935877766c831bd7f322e
-  data.tar.gz: b8229d7b34a91245292651408a40d62b050618cd7c0f2eb8260abcc3d564c4b5d4ec09c06f4b2c6c1cd86b495482338fd2138c9c5d359c4144d287abc65243a8
+  metadata.gz: 6dbdd29162bb6f26f4cfaa83210d95c4270c2534efc1175a21f825c8349b5dd3eedd04c481ebc88d011f6142c18a42e6f490ae75249cddc790b334bfc3891c5b
+  data.tar.gz: b4cc203242ac159e0e7bd1a95693197bf38e3cc57007bcb024ee3efb97a5e72f2944adf37765f8e15ad60c7a9cf8bcdf5d14a2cd2f23acc03eeb16203e69f472

data/.gitignore

@@ -12,3 +12,6 @@
 *.o
 *.a
 mkmf.log
+*.iml
+.idea
+.ruby-version

data/Rakefile

@@ -5,3 +5,7 @@ RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
 
+desc "Open an irb session preloaded with this library"
+task :console do
+  sh "irb -I lib -r presto/metrics.rb"
+end

data/lib/presto/metrics/client.rb

@@ -0,0 +1,174 @@
+
+module Presto
+  module Metrics
+
+  	class Client 
+	  	def initialize(opts={})
+	  		require 'httparty'
+	  		require 'json'
+	  		require 'set'
+	  		require 'time'
+
+ 			@host = opts[:host] || 'localhost'
+  			@port = opts[:port] || '8080'
+		 	@endpoint = opts[:endpoint] || "http://#{@host}:#{@port}"
+		 	@mbean_path = opts[:mbean_path] || '/v1/jmx/mbean'
+		 	@query_path = opts[:query_path] || '/v1/query'
+		 	@caml_case = opts[:caml_case] || false
+		end
+
+	  	@@MBEAN_ALIAS = {
+	  		'memory' => 'java.lang:type=Memory',
+	  		'gc_cms' => 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep',
+	  		'gc_parnew' => 'java.lang:type=GarbageCollector,name=ParNew',
+	  		'os' => 'java.lang:type=OperatingSystem',
+	  		'query_manager' => 'com.facebook.presto.execution:name=QueryManager',
+	  		'query_execution' => 'com.facebook.presto.execution:name=QueryExecution',
+	  		'node_scheduler' => 'com.facebook.presto.execution:name=NodeScheduler',
+	  		'task_executor' => 'com.facebook.presto.execution:name=TaskExecutor',
+	  		'task_manager' => 'com.facebook.presto.execution:name=TaskManager'
+	  	}
+
+
+		def path(path) 
+			c = path.split(/:/)
+			target = c[0]
+			mbean = @@MBEAN_ALIAS[target] || target
+			json_obj = get_metrics(mbean)
+			return json_obj if c.size <= 1
+			query_list = (c[1] || '').split(/,/)
+			result = {}
+			query_list.each{|q|
+				path_elems = q.split('/')
+				target_elem = extract_path(json_obj, path_elems, 0)
+				result[q] = target_elem unless target_elem.nil?
+			}
+			result
+		end
+
+
+		def extract_path(json_obj, path, depth) 
+			return nil if json_obj.nil?
+			if depth >= path.length 
+				json_obj
+			else 
+				if json_obj.kind_of?(Array)
+				   # Handle key, value pairs of GC information
+				   value = json_obj.find{|e| 
+				   	e.is_a?(Hash) && e['key'] == path[depth]
+				   }
+				   extract_path(value['value'], path, depth+1)
+				else
+					extract_path(json_obj[path[depth]], path, depth+1)
+				end
+			end
+		end
+
+		def query 
+			Query.new(self)
+		end
+
+		def get_mbean(mbean)
+			JSON.parse(get_mbean_json(mbean))
+	  	end
+
+	  	def get(path, default='{}')
+	  		resp = HTTParty.get("#{@endpoint}#{path}")
+	  		if resp.code == 200
+	  			resp.body 	  		
+	  		else
+	  			default
+	  		end
+	  	end
+
+	  	def get_mbean_json(mbean) 
+	  		get("#{@mbean_path}/#{mbean}")
+	  	end 
+
+	  	def get_query_json(path='', default='[]')
+			get("#{@query_path}/#{path}",default)
+	  	end
+
+	  	def get_attributes(mbean) 
+			json = get_mbean(mbean)
+			json['attributes'] || []
+	  	end
+
+	  	def get_attribute(mbean, attr_name) 
+			get_attributes(mbean).find {|obj| obj['name'] == attr_name } || {}
+	  	end
+
+	  	def map_to_canonical_name(name_arr) 
+	  		name_arr.map{|name| to_canonical_name(name)	}
+		end
+
+		def to_canonical_name(name) 
+	  		name.to_s.downcase.gsub(/_/, '')
+	  	end
+
+  	    def underscore(str)
+	    	str.gsub(/::/, '/').
+		    gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+	    	gsub(/([a-z\d])([A-Z])/,'\1_\2').
+		    tr('-', '_').
+		    downcase
+		end
+
+	  	def get_metrics(mbean, target_attr=[]) 
+	  		kv = Hash.new
+	  		arr = target_attr.kind_of?(Array) ? target_attr : [target_attr]
+	  		c_target_attr = map_to_canonical_name(arr).to_set
+	  		get_attributes(mbean)
+	  		 	.reject {|attr| attr['name'].nil? || attr['value'].nil? }
+	  		 	.each {|attr| 
+	  		 		c_name = to_canonical_name(attr['name'])
+	  		 		if c_target_attr.empty? || c_target_attr.include?(c_name)
+	  		 			key = @caml_case ? attr['name'] : underscore(attr['name'])
+	  		 			kv[key] = attr['value'] 
+	  		 		end
+	  		 	}
+	  		kv
+	  	end
+
+
+	  	def memory_usage_metrics(target_attr=[])
+	  		get_metrics('java.lang:type=Memory', target_attr)
+	  	end
+
+	  	def gc_cms_metrics(target_attr=[])
+	  		get_metrics('java.lang:type=GarbageCollector,name=ConcurrentMarkSweep', target_attr)
+	  	end
+
+	  	def gc_parnew_metrics(target_attr=[])
+	  		get_metrics('java.lang:type=GarbageCollector,name=ParNew', target_attr)
+	  	end
+
+	  	def os_metrics(target_attr=[])
+	  		get_metrics('java.lang:type=OperatingSystem', target_attr)
+	  	end
+
+	  	def query_manager_metrics(target_attr=[])
+	  		get_metrics('com.facebook.presto.execution:name=QueryManager', target_attr)
+	  	end
+
+	  	def query_execution_metrics(target_attr=[])
+	  		get_metrics('com.facebook.presto.execution:name=QueryExecution', target_attr)
+	  	end
+
+	  	def node_scheduler_metrics(target_attr=[])
+	  		get_metrics('com.facebook.presto.execution:name=NodeScheduler', target_attr)
+	  	end
+
+	  	def task_executor_metrics(target_attr=[])
+	  		get_metrics('com.facebook.presto.execution:name=TaskExecutor', target_attr)
+	  	end
+
+	  	def task_manager_metrics(target_attr=[])
+	  		get_metrics('com.facebook.presto.execution:name=TaskManager', target_attr)
+	  	end
+
+		private :map_to_canonical_name, :to_canonical_name, :underscore
+  	end 
+  end
+end
+

data/lib/presto/metrics/query.rb

@@ -0,0 +1,167 @@
+require 'presto/metrics/client'
+require 'presto-client'
+# Use httpclient for faster json retrieval
+Faraday.default_adapter = :httpclient
+require 'presto/client/models'
+require 'pp'
+include Presto::Client::Models
+
+module Presto
+  module Metrics
+
+  	class Query 
+  		def initialize(client)
+  			@client = client
+  		end
+
+  		def format_table(tbl, label:[], align:[], sep:' ')
+  			# Compute col length
+  			col = {}
+  			label.each_with_index{|l, i| col[i] = l.length}
+  			tbl.each{|row|
+  				row.each_with_index {|cell, i| 
+  					l = cell.to_s.size if cell 
+  					l ||= 0
+  					col[i] ||= l
+  					col[i] = [[col[i], l].max, 150].min
+  				}
+  			}
+  			# print label
+  			line = []
+  			label.each_with_index{|l, i|
+  				line << l.to_s[0..col[i]].ljust(col[i])
+  			}
+  			puts line.join(sep)
+  			tbl.each{|row|
+  				line = []
+  				row.each_with_index{|cell, i|
+  					str = cell.to_s[0..col[i]]
+  					a = align[i] || 'l'
+  					case a 
+  					when 'r'
+  						line << str.rjust(col[i])
+  					when 'l'
+  						line << str.ljust(col[i])
+  					else
+  						line << str.ljust(col[i])
+  					end
+  				}
+  				puts line.join(sep)
+  			}
+  			0
+  		end
+
+	  	def list
+	  		ql = query_list
+	  		tbl = ql.map {|q|
+	  			s = q['session'] || {}
+	  			query = q['query'].gsub(/[\t\r\n]/, ' ').gsub(/ {1,}/, ' ').strip
+	  			[q['queryId'], q['elapsedTime'], q['state'], q['runningDrivers'], q['completedDrivers'], q['totalDrivers'], s['user'], s['catalog'], s['schema'], s['source'], query]
+	  		}.sort_by{|row| row[0]}.reverse
+
+	  		format_table(
+	  			tbl, 
+	  			:label => %w|query time state r f t user catalog schema source sql|,
+	  			:align => %w|r     r    r     r r r l    l       r      l      l  |
+	  		)
+	  	end
+
+	  	def find(queryId)
+	  		JSON.parse(@client.get_query_json(queryId, "{}"))
+	  	end
+
+	  	def task_list(queryId)
+	  		qj = find(queryId) || {}
+        root_stage = qj['outputStage'] || []
+        tasks = root_stage['tasks'] || []
+        tasks << find_tasks(root_stage['subStages'])
+        tasks.flatten
+      end
+
+      def tasks(queryId)
+        tl = task_list(queryId)
+        stats = tl.map {|t|
+          s = t['stats']
+          host = (t['self'] || '').sub(/http:\/\/([a-z0-9\-.]+[\/:][0-9]+)\/.*/, '\1')
+          [t['taskId'], host, t['state'], s['rawInputPositions'], s['rawInputDataSize'], s['queuedDrivers'], s['runningDrivers'], s['completedDrivers']]
+        }
+        format_table(stats,
+                     :label => %w|task_id host    state processed_rows size|,
+                     :align => %w|l       l       l     r          r|
+        )
+      end
+
+	  	def query_list(path="")
+	  		JSON.parse(@client.get_query_json(path))
+	  	end
+
+      def running_list
+        ql = query_list.select{|q| q['state'] == 'RUNNING'}.sort_by{|row| row[0]}.reverse
+        ql.each{|q|
+          tasks(q['queryId'])
+        }
+        ql.size
+			end
+
+			private :count_total_processed_rows
+
+			def count_total_processed_rows(stage)
+				rows = stage.tasks.map{|t| t.stats.processed_input_positions }.inject(0, :+)
+				rows + stage.sub_stages.map{|ss| count_total_processed_rows(ss) }.inject(0, :+)
+			end
+
+			def query_progress
+				running_queries = query_list.map{|q| QueryInfo.decode(q) }.select{|q| q.state == :running }
+		    query_info = running_queries.map{|q|
+					queryInfo = find(q.query_id)
+					QueryInfo.decode(queryInfo)
+				}
+
+				result = {}
+				query_info.each{|q|
+				   os = q.output_stage
+					 result[q.query_id] = count_total_processed_rows(os)
+				}
+				result
+			end
+
+	  	def metrics 
+	  		ql = query_list
+	  		ql.map{|qi|
+	  			h = {}
+	  			h['query_id'] = qi['queryId'] || ''
+	  			h['state'] = qi['state'] || ''
+	  			session = qi['session'] || {}
+	  			h['source'] = session['source'] || ''
+	  			h['user'] = session['user'] || h['source'].gsub(/[^a-zA-Z0-9]/,'')
+	  			h['running_drivers'] = qi['runningDrivers'] || 0
+	  			h['queued_drivers'] = qi['queuedDrivers'] || 0
+	  			h['completed_drivers'] = qi['completedDrivers'] || 0
+	  			h['total_drivers'] = qi['totalDrivers'] || 0
+	  			h['elapsed_time'] = qi['elapsedTime'] || '0.0m'
+	  			h['create_time'] = qi['createTime']
+	  			h['running_time'] = qi['endTime'] || Time.now.utc.iso8601(3)
+	  			#if(h['state'] == "FAILED")
+	  			#	h['errorCode'] = find(h['query_id'])['errorCode'] || {}
+	  			#end
+	  			h
+	  		}
+      end
+
+
+
+
+      def find_tasks(sub_stages)
+        task_list = []
+        return task_list unless sub_stages
+        sub_stages.each{|ss|
+          tl = ss['tasks']
+          task_list << tl if tl
+          task_list << find_tasks(ss['subStages'])
+        }
+        task_list.flatten()
+      end
+
+    end
+  end
+end

data/lib/presto/metrics/version.rb

@@ -1,5 +1,5 @@
 module Presto
   module Metrics
-    VERSION = "0.2.6"
+    VERSION = '0.3.0'
   end
 end

data/lib/presto/metrics.rb

@@ -1,222 +1,3 @@
-require "presto/metrics/version"
-require 'httparty'
-require 'json'
-require 'set'
-require 'time'
-
-module Presto
-  module Metrics
-
-  	class Query 
-  		def initialize(client)
-  			@client = client
-  		end
-
-	  	def list
-	  		query_list().each {|q|
-	  			s = q['session'] || {}
-	  			query = q['query'].gsub(/[\r\n]/, " ")[0..50]
-	  			c = [q['queryId'], s['user'], s['catalog'], s['schema'], s['source'], query]
-	  			puts c.join("\t")
-	  		}
-	  		0
-	  	end
-
-	  	def find(id)
-	  		JSON.parse(@client.get_query_json(id))
-	  	end
-
-	  	def query_list(path="")
-	  		JSON.parse(@client.get_query_json(path))
-	  	end
-
-	  	def metrics 
-	  		ql = query_list
-	  		ql.map{|qi|
-	  			h = {}
-	  			h['query_id'] = qi['queryId'] || ""
-	  			h['state'] = qi['state'] || ""
-	  			session = qi['session'] || {}
-	  			h['source'] = session['source'] || ""
-	  			h['user'] = session['user'] || h['source'].gsub(/[^a-zA-Z0-9]/,'')
-	  			h['running_drivers'] = qi['runningDrivers'] || 0
-	  			h['queued_drivers'] = qi['queuedDrivers'] || 0
-	  			h['completed_drivers'] = qi['completedDrivers'] || 0
-	  			h['total_drivers'] = qi['totalDrivers'] || 0
-	  			h['elapsed_time'] = qi['elapsedTime'] || "0.0m"
-	  			h['create_time'] = qi['createTime']
-	  			h['running_time'] = qi['endTime'] || Time.now.utc.iso8601(3)
-	  			#if(h['state'] == "FAILED")
-	  			#	h['errorCode'] = find(h['query_id'])['errorCode'] || {}
-	  			#end
-	  			h
-	  		}
-	  	end
-
-  	end
-
-  	class Client 
-	  	def initialize(opts={})
- 			@host = opts[:host] || "localhost"
-  			@port = opts[:port] || "8080"
-		 	@endpoint = opts[:endpoint] || "http://#{@host}:#{@port}"
-		 	@mbean_path = opts[:mbean_path] || "/v1/jmx/mbean"
-		 	@query_path = opts[:query_path] || "/v1/query"
-		 	@caml_case = opts[:caml_case] || false
-		end
-
-	  	@@MBEAN_ALIAS = {
-	  		"memory" => "java.lang:type=Memory",
-	  		"gc_cms" => "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep",
-	  		"gc_parnew" => "java.lang:type=GarbageCollector,name=ParNew",
-	  		"os" => "java.lang:type=OperatingSystem",
-	  		"query_manager" => "com.facebook.presto.execution:name=QueryManager",
-	  		"query_execution" => "com.facebook.presto.execution:name=QueryExecution", 
-	  		"node_scheduler" => "com.facebook.presto.execution:name=NodeScheduler",
-	  		"task_executor" => "com.facebook.presto.execution:name=TaskExecutor",
-	  		"task_manager" => "com.facebook.presto.execution:name=TaskManager"
-	  	}
-
-
-		def path(path) 
-			c = path.split(/:/)
-			target = c[0]
-			mbean = @@MBEAN_ALIAS[target] || target
-			json_obj = get_metrics(mbean)
-			return json_obj if c.size <= 1
-			query_list = (c[1] || "").split(/,/)
-			result = {}
-			query_list.each{|q|
-				path_elems = q.split("/")
-				target_elem = extract_path(json_obj, path_elems, 0)
-				result[q] = target_elem unless target_elem.nil?
-			}
-			result
-		end
-
-
-		def extract_path(json_obj, path, depth) 
-			return nil if json_obj.nil?
-			if depth >= path.length 
-				json_obj
-			else 
-				if json_obj.kind_of?(Array)
-				   # Handle key, value pairs of GC information
-				   value = json_obj.find{|e| 
-				   	e.is_a?(Hash) && e['key'] == path[depth]
-				   }
-				   extract_path(value['value'], path, depth+1)
-				else
-					extract_path(json_obj[path[depth]], path, depth+1)
-				end
-			end
-		end
-
-		def query 
-			Query.new(self)
-		end
-
-		def get_mbean(mbean)
-			JSON.parse(get_mbean_json(mbean))
-	  	end
-
-	  	def get(path, default="{}") 
-	  		resp = HTTParty.get("#{@endpoint}#{path}")
-	  		if resp.code == 200
-	  			resp.body 	  		
-	  		else
-	  			default
-	  		end
-	  	end
-
-	  	def get_mbean_json(mbean) 
-	  		get("#{@mbean_path}/#{mbean}")
-	  	end 
-
-	  	def get_query_json(path="")
-			get("#{@query_path}/#{path}","[]")
-	  	end
-
-	  	def get_attributes(mbean) 
-			json = get_mbean(mbean)
-			json['attributes'] || []
-	  	end
-
-	  	def get_attribute(mbean, attr_name) 
-			get_attributes(mbean).find {|obj| obj['name'] == attr_name } || {}
-	  	end
-
-	  	def map_to_canonical_name(name_arr) 
-	  		name_arr.map{|name| to_canonical_name(name)	}
-		end
-
-		def to_canonical_name(name) 
-	  		name.to_s.downcase.gsub(/_/, "")
-	  	end
-
-  	    def underscore(str)
-	    	str.gsub(/::/, '/').
-		    gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
-	    	gsub(/([a-z\d])([A-Z])/,'\1_\2').
-		    tr("-", "_").
-		    downcase
-		end
-
-	  	def get_metrics(mbean, target_attr=[]) 
-	  		kv = Hash.new
-	  		arr = target_attr.kind_of?(Array) ? target_attr : [target_attr]
-	  		c_target_attr = map_to_canonical_name(arr).to_set
-	  		get_attributes(mbean)
-	  		 	.reject {|attr| attr['name'].nil? || attr['value'].nil? }
-	  		 	.each {|attr| 
-	  		 		c_name = to_canonical_name(attr['name'])
-	  		 		if c_target_attr.empty? || c_target_attr.include?(c_name)
-	  		 			key = @caml_case ? attr['name'] : underscore(attr['name'])
-	  		 			kv[key] = attr['value'] 
-	  		 		end
-	  		 	}
-	  		kv
-	  	end
-
-
-	  	def memory_usage_metrics(target_attr=[])
-	  		get_metrics("java.lang:type=Memory", target_attr)
-	  	end
-
-	  	def gc_cms_metrics(target_attr=[])
-	  		get_metrics("java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", target_attr)
-	  	end
-
-	  	def gc_parnew_metrics(target_attr=[])
-	  		get_metrics("java.lang:type=GarbageCollector,name=ParNew", target_attr)
-	  	end
-
-	  	def os_metrics(target_attr=[])
-	  		get_metrics("java.lang:type=OperatingSystem", target_attr)
-	  	end
-
-	  	def query_manager_metrics(target_attr=[])
-	  		get_metrics("com.facebook.presto.execution:name=QueryManager", target_attr)
-	  	end
-
-	  	def query_execution_metrics(target_attr=[])
-	  		get_metrics("com.facebook.presto.execution:name=QueryExecution", target_attr)
-	  	end
-
-	  	def node_scheduler_metrics(target_attr=[])
-	  		get_metrics("com.facebook.presto.execution:name=NodeScheduler", target_attr)
-	  	end
-
-	  	def task_executor_metrics(target_attr=[])
-	  		get_metrics("com.facebook.presto.execution:name=TaskExecutor", target_attr)
-	  	end
-
-	  	def task_manager_metrics(target_attr=[])
-	  		get_metrics("com.facebook.presto.execution:name=TaskManager", target_attr)
-	  	end
-
-		private :map_to_canonical_name, :to_canonical_name, :underscore
-  	end 
-  end
-end
-
+require 'presto/metrics/version'
+require 'presto/metrics/client'
+require 'presto/metrics/query'

data/presto-metrics.gemspec

@@ -21,6 +21,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"
+  spec.add_development_dependency "presto-client", '~> 0.4.4'
+  spec.add_development_dependency "httpclient"
 
   spec.add_runtime_dependency "httparty"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: presto-metrics
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.3.0
 platform: ruby
 authors:
 - Taro L. Saito
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-05 00:00:00.000000000 Z
+date: 2015-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,34 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: presto-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.4
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: httparty
   requirement: !ruby/object:Gem::Requirement
@@ -82,6 +110,8 @@ files:
 - README.md
 - Rakefile
 - lib/presto/metrics.rb
+- lib/presto/metrics/client.rb
+- lib/presto/metrics/query.rb
 - lib/presto/metrics/version.rb
 - presto-metrics.gemspec
 - spec/presto/metrics_spec.rb