metricsgeek 0.0.5

data/LICENSE.md

@@ -0,0 +1,13 @@
+Copyright (c) 2013 Evan Chan
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,71 @@
+Easily grok and analyze your application metrics from a cluster, in real time, with this command line tool /
+gem. Compatible with HTTP JSON metrics routes as exposed by Coda Hale's metrics package, Jolokia, ruby-
+metrics, or any metrics package that exposes metrics via an HTTP route as JSON.
+
+The only assumption is that the JSON consists of deeper and deeper levels of hashes.
+
+* Get metrics in real time, no need to wait minutes for a central service to index your logs
+* Easily see load and traffic imbalances across a cluster
+* Select and filter metrics easily using wildcards
+* Group and summarize metrics
+
+Quick start
+===========
+Install this gem:
+
+    gem install metricsgeek
+
+To get a list of metrics keys from your servers:
+
+    metricsgeek --list_keys --from "server[1..8].abc.com"
+
+To get two metrics, one a pattern, from your servers:
+
+    metricsgeek --select jvm.uptime,com.abc.*.latency.mean --from "server[1..8].abc.com"
+
+Port and Route
+==============
+The port defaults to 7000 and the route defaults to /metricz.  Both of these can be set via the --port and
+--route options.
+
+Host selection
+==============
+You can select multiple hosts to query in two ways.
+
+First, individual hosts can be passed, comma separated, to the --from option.
+
+Second, you can pass in a range of numbers within brackets, and this is automatically expanded to separate
+hosts.  For example,
+
+    --from server[1..3,5,8..11].dc
+
+expands to
+
+    --from server1.dc,server2.dc,server3.dc,server5.dc,server8.dc,server9.dc,server10.dc,server11.dc
+
+Metrics selection
+=================
+You can use the --list_keys option together with --from to list all the metrics keys available for querying.
+This is done by flattening the JSON output from all the routes, with successive levels of JSON separated by
+dots in the flat metric key.
+
+The --select option takes one or more metric keys separated by commas.  So,
+
+    --select jvm.uptime,my.app.*.latency
+
+Note that you can use the * wildcard character to select multiple metrics keys, as well as [12] style
+character selection.
+
+Grouping metrics
+================
+Intead of displaying metrics from every host on a separate line, you can group the metrics together from all hosts using one of four functions: sum, min, max, avg.  For example, to compute the average uptime of all hosts:
+
+    metricsgeek --select jvm.uptime --from "server[1..8].dc" --group avg
+
+Or, to sum up the POST rate from metric com.abc.webservice.posts.m1 across the cluster:
+
+    metricsgeek --select com.abc.webservice.posts.m1 --from "server[1..8].dc" --group sum
+
+How to contribute
+=================
+Pull requests are welcome!

data/bin/metricsgeek

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+require "trollop"
+require "metricsgeek"
+
+opts = Trollop::options do
+  opt :select, "metrics keys or key glob patterns, comma separated", :type => :string
+  opt :from, "host names, comma separated, may contain [1..10] ranges", :type => :string
+  opt :port, "port #", :default => 7000
+  opt :route, "JSON metrics route", :default => "metricz"
+  opt :list_keys, "Just list the keys"
+  opt :group, "Group metrics from hosts using min/max/avg/sum", :type => :string
+  opt :timeout, "Request timeout in seconds", :type => :int
+end
+
+class Array
+  def sum
+    self.inject(0) { |a, s| a + s }
+  end
+
+  def avg
+    self.sum * 1.0 / self.length
+  end
+end
+
+def format(number)
+  "%7.3f" % [number]
+end
+
+expanded_hosts = opts[:from].split(",").map { |expr| MetricsDownloader.expand_host_expr(expr) }.flatten
+urls = MetricsDownloader.create_urls_from_host_params(expanded_hosts, opts[:port], opts[:route])
+json_trees = MetricsDownloader.download_and_parse_json_from_urls(urls, opts[:timeout])
+
+if opts[:list_keys]
+  MetricsParser.list_keys_for_hashes(json_trees).each { |key| puts key }
+else
+  keys = opts[:select].split(",")
+  host_key_value = expanded_hosts.zip(json_trees).map do |host, json_tree|
+    matching_key_values = MetricsParser.glob_key_values(json_tree, keys)
+    matching_key_values.map do |flatkey, value|
+      {:host => host, :key => flatkey, :value => value}
+    end
+  end.flatten
+  host_key_value.group_by { |hash| hash[:key] }.each do |key, hashes|
+    puts "\n\nFor key: #{key}"
+    if opts[:group]
+      values = hashes.map { |h| h[:value] }
+      opts[:group].split(",").each { |func| puts "  #{func}: #{format(values.send(func.to_sym))}" }
+    else
+      hashes.each { |h| puts "  #{h[:host]}\t#{format(h[:value])}"}
+    end
+  end
+end

data/lib/metrics_downloader.rb

@@ -0,0 +1,48 @@
+require "json"
+require "yaml"
+require "rest_client"
+
+module MetricsDownloader
+  # @returns a list of parsed JSON hashes from each url
+  def self.download_and_parse_json_from_urls(urls, timeout = 5)
+    threads = urls.map do |url|
+      Thread.new do
+        Thread.current[:output] = begin
+          body = RestClient::Request.execute(:method => :get, :url => url,
+                                             :timeout => timeout,
+                                             :open_timeout => timeout)
+          JSON.parse(body)
+        rescue
+          # TODO: log the exception?
+          STDERR.puts "WARNING: Unable to parse JSON from #{url}"
+          {}
+        end
+      end
+    end
+
+    threads.map { |t| t.join; t[:output] }
+  end
+
+  # @param host_list A list of host strings, without the port, optionally with http:// prefix
+  # @route the string after the / for getting metrics, defaults to "metricz/"
+  # @returns a list of URLs from which to retrieve data
+  def self.create_urls_from_host_params(host_list, port, route = "metricz/")
+    host_list.map do |host|
+      host = host[7..-1] if host.start_with?("http:")
+      "http://#{host}:#{port}/#{route}"
+    end
+  end
+
+  # @param host_expr A host string with an embedded range [1..18] or individual numbers,
+  #                  [1..5,7,9]  =>  1,2,3,4,5,7,9 will be substituted
+  # @returns An expanded list of hosts, basically with the range expanded into numbers.
+  def self.expand_host_expr(host_expr)
+    if host_expr =~ /(\[([0-9.,]+)\])/
+      embedded_expr = $1
+      numbers = $2.split(",").map { |item| item.include?("..") ? eval(item).to_a : [item.to_i] }.flatten
+      numbers.map { |n| host_expr.gsub(embedded_expr, n.to_s) }
+    else
+      [host_expr]
+    end
+  end
+end

data/lib/metrics_parser.rb

@@ -0,0 +1,41 @@
+# Core functions for parsing metrics from JSON web output, such as that by Coda Hale's metrics library
+
+module MetricsParser
+  # @param tree A hash tree such as that generated by JSON.parse()
+  # @param prefix A list of namespace strings to prefix the generated keys by
+  # @returns A flattened hash where the key is each level of the tree joined by '.'
+  def self.flatten_tree(json_tree, prefix=[])
+    flat_tree = {}
+    pairs = json_tree.each do |key, value|
+      newkey = (prefix + [key]).join(".")
+      flat_tree[newkey] = value
+      if value.is_a?(Hash)
+        flat_tree.update(MetricsParser.flatten_tree(value, prefix + [key]))
+      end
+    end
+    flat_tree
+  end
+
+  # @returns a list of sorted, flattened hash keys for the input JSON hash tree
+  def self.list_keys_for_hash(json_tree)
+    self.flatten_tree(json_tree).keys.sort
+  end
+
+  # @returns a set of flattened hash keys that's the union of the individual flat hash keys for each JSON blob
+  def self.list_keys_for_hashes(json_trees)
+    json_trees.map { |tree| self.flatten_tree(tree).keys }.flatten.uniq.sort
+  end
+
+  # @param json_tree As returned by JSON,parse()
+  # @param key_patterns a list of glob pattern strings for matching keys
+  # @returns a hash of flat_key to value for all keys matching any of the key_patterns
+  def self.glob_key_values(json_tree, key_patterns)
+    matches = {}
+    flat_tree = flatten_tree(json_tree)
+    key_patterns.each do |key_pattern|
+      matching_keys = flat_tree.keys.select { |key| File.fnmatch(key_pattern, key) }
+      matches.merge!(Hash[ matching_keys.map { |key| [key, flat_tree[key]] } ])
+    end
+    matches
+  end
+end

data/lib/metricsgeek.rb

@@ -0,0 +1,2 @@
+require "metrics_downloader"
+require "metrics_parser"

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification
+name: metricsgeek
+version: !ruby/object:Gem::Version
+  version: 0.0.5
+  prerelease: 
+platform: ruby
+authors:
+- Evan Chan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-16 00:00:00.000000000Z
+dependencies: []
+description: ! 'Easily grok and analyze your application metrics from a cluster, in
+  real time, with this command line tool /
+
+  gem. Compatible with HTTP JSON metrics routes as exposed by Coda Hale''s metrics
+  package, Jolokia, ruby-
+
+  metrics, or any metrics package that exposes metrics via an HTTP route as JSON.
+
+'
+email: velvia@gmail.com
+executables:
+- metricsgeek
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/metrics_downloader.rb
+- lib/metrics_parser.rb
+- lib/metricsgeek.rb
+- bin/metricsgeek
+- LICENSE.md
+- README.md
+homepage: http://github.com/velvia/metricsgeek
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Easily grok and analyze application metrics from a cluster.
+test_files: []