fintop 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
@@ -0,0 +1,7 @@
1
+ # 0.0.1 (4/1/2014)
2
+
3
+ - Initial version of Fintop. Probes stats and threads endpoints of Finagle
4
+ servers built on [TwitterServer](http://twitter.github.io/twitter-server/)
5
+ with either [Ostrich](https://github.com/twitter/ostrich) or
6
+ [MetricsTM](https://github.com/twitter/commons/tree/master/src/java/com/twitter/common/metrics).
7
+ Prints basic statistics in top-like tabular output.
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fintop.gemspec
4
+ gemspec
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Evan Meagher
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,30 @@
1
+ # fintop
2
+
3
+ A top-like utility for monitoring [Finagle](http://github.com/twitter/finagle) servers.
4
+
5
+ ## Installation
6
+
7
+ The `fintop` gem hasn't been published yet, so is currently only usable from
8
+ source. To install `fintop` locally, clone this repository and run
9
+ `rake install` from within the repository's root directory.
10
+
11
+ ## Usage
12
+
13
+ `fintop` is intended to be used in a similar fashion as the `top` suite of
14
+ monitoring programs. For example, given two Finagle servers running locally,
15
+ running `fintop` could look like this:
16
+
17
+ $ fintop
18
+ Finagle processes: 2, Threads: 44 total, 30 runnable, 14 waiting
19
+
20
+ PID PORT CPU #TH #NOND #RUN #WAIT #TWAIT TXKB RXKB
21
+ 14909 1110 4.0 22 1 15 4 3 351 363
22
+ 14905 9990 4.0 22 1 15 4 3 325 823
23
+
24
+ ## Contributing
25
+
26
+ 1. [Fork the repository](http://github.com/evnm/fintop/fork)
27
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
28
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
29
+ 4. Push to the branch (`git push origin my-new-feature`)
30
+ 5. [Create new Pull Request](https://help.github.com/articles/creating-a-pull-request)
@@ -0,0 +1,8 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rake/testtask'
3
+
4
+ Rake::TestTask.new do |t|
5
+ t.libs << 'test'
6
+ t.test_files = FileList['test/test*.rb']
7
+ t.verbose = true
8
+ end
@@ -0,0 +1,9 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'fintop/printer'
4
+ require 'fintop/probe'
5
+ require 'fintop/threads'
6
+
7
+ finagle_procs = Fintop::Probe.apply
8
+ Fintop::Printer.apply(finagle_procs)
9
+
@@ -0,0 +1,24 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'fintop/version'
5
+
6
+ Gem::Specification.new do |s|
7
+ s.name = 'fintop'
8
+ s.version = Fintop::VERSION
9
+ s.authors = ['Evan Meagher']
10
+ s.email = ['evan.meagher@gmail.com']
11
+ s.summary = %q{A top-like utility for monitoring Finagle servers}
12
+ s.license = 'MIT'
13
+
14
+ s.files = `git ls-files -z`.split("\x0")
15
+ s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
16
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
17
+ s.require_paths = ['lib']
18
+
19
+ s.add_development_dependency 'bundler', '~> 1.5'
20
+ s.add_development_dependency 'rake'
21
+ s.add_development_dependency 'test-unit', '~> 2.5'
22
+
23
+ s.add_dependency 'json'
24
+ end
@@ -0,0 +1,8 @@
1
+ require 'fintop/metrics'
2
+ require 'fintop/printer'
3
+ require 'fintop/probe'
4
+ require 'fintop/threads'
5
+ require 'fintop/version'
6
+
7
+ module Fintop
8
+ end
@@ -0,0 +1,78 @@
1
+ require 'json'
2
+ require 'net/http'
3
+
4
+ module Fintop
5
+ # Contains functions that gather Finagle application metrics.
6
+ module Metrics
7
+ extend self
8
+
9
+ # Query a Finagle server's stats endpoint to produce a hash of metrics.
10
+ #
11
+ # @param finp [FinagleProcess]
12
+ def apply(finp)
13
+ case finp.stats_lib
14
+ when :metrics_tm
15
+ scrape_metrics_tm(finp.admin_port)
16
+ when :ostrich
17
+ scrape_ostrich(finp.admin_port)
18
+ else
19
+ raise ArgumentError.new('invalid `FinagleProcess.stats_lib` symbol')
20
+ end
21
+ end
22
+
23
+ # Parses a JSON object of MetricsTM-style stats into a hash.
24
+ #
25
+ # @param json_str [String] a JSON string of MetricsRM-style stats data
26
+ def process_metrics_tm_json(json_str)
27
+ result = {}
28
+ JSON.parse(json_str).each { |k, v|
29
+ prefix, rest = k.split('/', 2)
30
+ (result[prefix] ||= {})[rest] = v
31
+ }
32
+ result
33
+ end
34
+
35
+ # Parses a JSON object of Ostrich-style stats into a hash.
36
+ #
37
+ # @param json_str [String] a JSON string of Ostrich-style stats data
38
+ def process_ostrich_json(json_str)
39
+ json = JSON.parse(json_str)
40
+
41
+ # Note: We currently only gather counter and gauge data.
42
+ result = {}
43
+ json['counters'].merge(json['gauges']).each { |k, v|
44
+ # Do a dance in order to get around the fact that Ostrich doesn't prefix
45
+ # JVM-related metrics with "jvm/".
46
+ until_first_slash, slash_rest = k.split('/', 2)
47
+ until_first_underscore, us_rest = until_first_slash.split('_', 2)
48
+ prefix = until_first_underscore
49
+ rest = us_rest.to_s + slash_rest.to_s
50
+ (result[prefix] ||= {})[rest] = v
51
+ }
52
+ result
53
+ end
54
+
55
+ # Fetch metrics from a MetricsTM stats endpoint (/admin/metrics.json).
56
+ # Returns a nested hash keyed by the metric prefix.
57
+ # i.e. { jvm => { uptime => 18251.0, fd_count = 203.0, ... }, ... }
58
+ #
59
+ # @param admin_port [Fixnum]
60
+ def scrape_metrics_tm(admin_port)
61
+ json_str = Net::HTTP.get(
62
+ URI.parse("http://localhost:#{admin_port}/admin/metrics.json")
63
+ )
64
+
65
+ process_metrics_tm_json(json_str)
66
+ end
67
+
68
+ # Fetch metrics from an Ostrich stats endpoint (/stats.json).
69
+ # Returns a nested hash keyed by the metric prefix.
70
+ # i.e. { jvm => { uptime => 18251.0, fd_count = 203.0, ... }, ... }
71
+ #
72
+ # @param admin_port [Fixnum]
73
+ def scrape_ostrich(admin_port)
74
+ json_str = Net::HTTP.get(URI.parse("http://localhost:#{admin_port}/stats.json"))
75
+ process_ostrich_json(json_str)
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,107 @@
1
+ require 'fintop/metrics'
2
+ require 'pp'
3
+
4
+ module Fintop
5
+ # Contains functions that gather and print operational data on local
6
+ # Finagle processes.
7
+ module Printer
8
+ extend self
9
+
10
+ # Given an array of Fintop::Probe::FinagleProcess objects, gather data
11
+ # and print output.
12
+ #
13
+ # @param finagle_procs [Array<FinagleProcess>]
14
+ def apply(finagle_procs)
15
+ if finagle_procs.empty?
16
+ puts 'Finagle processes: 0'
17
+ exit
18
+ end
19
+
20
+ # Create a hash of pid=>ThreadsData objects.
21
+ threads_data_hash = Hash[
22
+ finagle_procs.map { |finp|
23
+ [finp.pid, Fintop::ThreadsData.new(finp.admin_port)]
24
+ }
25
+ ]
26
+
27
+ metrics_hash = Hash[
28
+ finagle_procs.map { |finp|
29
+ [finp.pid, Fintop::Metrics.apply(finp)]
30
+ }
31
+ ]
32
+
33
+ print_header(threads_data_hash)
34
+
35
+ finagle_procs.each { |finp|
36
+ threads_data = threads_data_hash[finp.pid]
37
+ metrics = metrics_hash[finp.pid]
38
+
39
+ tx_bytes, rx_bytes = [0, 0]
40
+ metrics.values.each { |scoped_metrics|
41
+ scoped_metrics.each { |k, v|
42
+ if not (k =~ /\/sent_bytes$/).nil?
43
+ tx_bytes = v / 1024
44
+ elsif not (k =~ /\/received_bytes$/).nil?
45
+ rx_bytes = v / 1024
46
+ end
47
+ }
48
+ }
49
+
50
+ printf(
51
+ @@row_format_str,
52
+ finp.pid,
53
+ finp.admin_port,
54
+ metrics['jvm']['num_cpus'].to_f,
55
+ threads_data.num_threads,
56
+ threads_data.num_non_daemon,
57
+ threads_data.num_runnable,
58
+ threads_data.num_waiting,
59
+ threads_data.num_timed_waiting,
60
+ tx_bytes,
61
+ rx_bytes
62
+ )
63
+ }
64
+ end
65
+
66
+ private
67
+
68
+ @@row_format_str = "%-7s %-6s %-5s %-5s %-6s %-6s %-7s %-8s %-10s %-10s\n"
69
+
70
+ # Print a total process/thread synopsis and column headers.
71
+ #
72
+ # @param threads_data_hash [Hash<Fixnum, ThreadsData>] a hash of pids and
73
+ # their corresponding ThreadsData objects.
74
+ def print_header(threads_data_hash)
75
+ total_threads = threads_data_hash.values.map { |t|
76
+ t.num_threads
77
+ }.inject(:+)
78
+
79
+ runnable_threads = threads_data_hash.values.map { |t|
80
+ t.num_runnable
81
+ }.inject(:+)
82
+
83
+ waiting_threads = threads_data_hash.values.map { |t|
84
+ t.num_waiting + t.num_timed_waiting
85
+ }.inject(:+)
86
+
87
+ puts "Finagle processes: #{threads_data_hash.size}, "\
88
+ "Threads: #{total_threads} total, "\
89
+ "#{runnable_threads} runnable, "\
90
+ "#{waiting_threads} waiting"
91
+ puts
92
+ printf(
93
+ @@row_format_str,
94
+ 'PID',
95
+ 'PORT',
96
+ 'CPU',
97
+ '#TH',
98
+ '#NOND',
99
+ '#RUN',
100
+ '#WAIT',
101
+ '#TWAIT',
102
+ 'TXKB',
103
+ 'RXKB'
104
+ )
105
+ end
106
+ end
107
+ end
@@ -0,0 +1,68 @@
1
+ require 'net/http'
2
+ require 'timeout'
3
+
4
+ module Fintop
5
+ # Contains functions for probing a system to discover Finagle server
6
+ # processes running locally.
7
+ module Probe
8
+ extend self
9
+
10
+ class FinagleProcess
11
+ attr_reader :pid
12
+ attr_reader :admin_port
13
+ attr_reader :stats_lib
14
+
15
+ def initialize(pid, admin_port, stats_lib)
16
+ @pid = pid
17
+ @admin_port = admin_port
18
+ @stats_lib = stats_lib
19
+ end
20
+ end
21
+
22
+ # Probe localhost for Finagle servers.
23
+ #
24
+ # Probing targets are Java processes listening on a TCP socket serving
25
+ # the path "/admin". Function returns an array of FinagleProcess objects
26
+ def apply
27
+ # Invoke jps and filter out nailgun servers and the jps process itself.
28
+ jps_cmd_str = '$JAVA_HOME/bin/jps | '\
29
+ 'grep -v NGServer | '\
30
+ 'grep -v Jps | '\
31
+ "awk '{print $1}'"
32
+
33
+ finagle_pids = `#{jps_cmd_str}`.split.map { |pid|
34
+ # Filter for the processes that are listening on a TCP port.
35
+ lsof_cmd_str = "lsof -P -i tcp -a -p #{pid} | grep LISTEN | awk '{print $9}'"
36
+ port_match = /\d+/.match(`#{lsof_cmd_str}`)
37
+ port_match && [pid, port_match[0]]
38
+ }.compact.map { |pid, admin_port|
39
+ # Probe the possible ping endpoints to determine which (if any) stats
40
+ # library is in use.
41
+ begin
42
+ # Manual timeout to bail out of hung requests to un-listened-on ports.
43
+ Timeout::timeout(0.3) do
44
+ if is_resolvable(admin_port, '/admin/metrics.json')
45
+ FinagleProcess.new(pid, admin_port, :metrics_tm)
46
+ elsif is_resolvable(admin_port, '/stats.json')
47
+ FinagleProcess.new(pid, admin_port, :ostrich)
48
+ else
49
+ nil
50
+ end
51
+ end
52
+ rescue Errno::ECONNREFUSED, Timeout::Error
53
+ nil
54
+ end
55
+ }.compact
56
+ end
57
+
58
+ # Returns true if localhost:admin_port/path is resolvable.
59
+ #
60
+ # @param admin_port [Fixnum]
61
+ # @param path [String]
62
+ def is_resolvable(admin_port, path)
63
+ Net::HTTP.start('localhost', admin_port) { |http|
64
+ http.head(path).code.to_i == 200
65
+ }
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,43 @@
1
+ require 'json'
2
+ require 'net/http'
3
+
4
+ module Fintop
5
+ # Container class for data gathered from a Finagle server's
6
+ # thread-dump endpoint.
7
+ class ThreadsData
8
+ attr_reader :num_threads
9
+ attr_reader :num_runnable
10
+ attr_reader :num_waiting
11
+ attr_reader :num_timed_waiting
12
+ attr_reader :num_non_daemon
13
+
14
+ # Initialize a ThreadsData object for a Finagle server's on a given
15
+ # port's "/admin/threads" endpoint.
16
+ #
17
+ # @param admin_port [Fixnum]
18
+ def initialize(admin_port)
19
+ json_str = Net::HTTP.get(
20
+ URI.parse("http://localhost:#{admin_port}/admin/threads")
21
+ )
22
+
23
+ @threads = JSON.parse(json_str)['threads'].to_a
24
+ @num_threads = @threads.size
25
+
26
+ @num_runnable = @threads.count { |tid, thread|
27
+ thread['state'] == 'RUNNABLE'
28
+ }
29
+
30
+ @num_waiting = @threads.count { |tid, thread|
31
+ thread['state'] == 'WAITING'
32
+ }
33
+
34
+ @num_timed_waiting = @threads.count { |tid, thread|
35
+ thread['state'] == 'TIMED_WAITING'
36
+ }
37
+
38
+ @num_non_daemon = @threads.count { |tid, thread|
39
+ thread['daemon'] == false
40
+ }
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,3 @@
1
+ module Fintop
2
+ VERSION = '0.0.1'
3
+ end
@@ -0,0 +1,7 @@
1
+ TEST_DIR = File.dirname(__FILE__)
2
+ %w(lib test).each do |dir|
3
+ $LOAD_PATH.unshift "#{TEST_DIR}/../#{dir}"
4
+ end
5
+
6
+ require 'test/unit'
7
+ require 'rubygems'
@@ -0,0 +1,50 @@
1
+ require 'test_helper'
2
+ require 'fintop/metrics'
3
+
4
+ class MetricsTest < Test::Unit::TestCase
5
+ def test_process_ostrich_json_parses_valid_json_string
6
+ json_str = %q{{"counters":{"srv\/http\/received_bytes":37},"gauges":{"jvm_uptime":219,"srv\/http\/connections":1}}}
7
+ expected = {
8
+ 'srv'=>{'http/received_bytes'=>37,'http/connections'=>1},
9
+ 'jvm'=>{'uptime'=>219}
10
+ }
11
+ output = Fintop::Metrics.process_ostrich_json(json_str)
12
+
13
+ assert_equal(output, expected)
14
+ end
15
+
16
+ def test_process_ostrich_json_throws_on_empty_string
17
+ assert_raise JSON::ParserError do
18
+ Fintop::Metrics.process_ostrich_json("")
19
+ end
20
+ end
21
+
22
+ def test_process_ostrich_json_throws_on_invalid_json_string
23
+ assert_raise JSON::ParserError do
24
+ Fintop::Metrics.process_ostrich_json("{j")
25
+ end
26
+ end
27
+
28
+ def test_process_metrics_tm_json_parses_valid_json_string
29
+ json_str = %q{{"jvm\/uptime":183.0,"srv\/http\/connections":1.0,"srv\/http\/received_bytes":96}}
30
+ expected = {
31
+ 'srv'=>{'http/received_bytes'=>96,'http/connections'=>1.0},
32
+ 'jvm'=>{'uptime'=>183.0}
33
+ }
34
+ output = Fintop::Metrics.process_metrics_tm_json(json_str)
35
+
36
+ assert_equal(output, expected)
37
+ end
38
+
39
+ def test_process_metrics_tm_json_throws_on_empty_string
40
+ assert_raise JSON::ParserError do
41
+ Fintop::Metrics.process_metrics_tm_json("")
42
+ end
43
+ end
44
+
45
+ def test_process_metrics_tm_json_throws_on_invalid_json_string
46
+ assert_raise JSON::ParserError do
47
+ Fintop::Metrics.process_metrics_tm_json("{j")
48
+ end
49
+ end
50
+ end
metadata ADDED
@@ -0,0 +1,141 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fintop
3
+ version: !ruby/object:Gem::Version
4
+ hash: 29
5
+ prerelease:
6
+ segments:
7
+ - 0
8
+ - 0
9
+ - 1
10
+ version: 0.0.1
11
+ platform: ruby
12
+ authors:
13
+ - Evan Meagher
14
+ autorequire:
15
+ bindir: bin
16
+ cert_chain: []
17
+
18
+ date: 2014-04-01 00:00:00 -07:00
19
+ default_executable:
20
+ dependencies:
21
+ - !ruby/object:Gem::Dependency
22
+ name: bundler
23
+ prerelease: false
24
+ requirement: &id001 !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ hash: 5
30
+ segments:
31
+ - 1
32
+ - 5
33
+ version: "1.5"
34
+ type: :development
35
+ version_requirements: *id001
36
+ - !ruby/object:Gem::Dependency
37
+ name: rake
38
+ prerelease: false
39
+ requirement: &id002 !ruby/object:Gem::Requirement
40
+ none: false
41
+ requirements:
42
+ - - ">="
43
+ - !ruby/object:Gem::Version
44
+ hash: 3
45
+ segments:
46
+ - 0
47
+ version: "0"
48
+ type: :development
49
+ version_requirements: *id002
50
+ - !ruby/object:Gem::Dependency
51
+ name: test-unit
52
+ prerelease: false
53
+ requirement: &id003 !ruby/object:Gem::Requirement
54
+ none: false
55
+ requirements:
56
+ - - ~>
57
+ - !ruby/object:Gem::Version
58
+ hash: 9
59
+ segments:
60
+ - 2
61
+ - 5
62
+ version: "2.5"
63
+ type: :development
64
+ version_requirements: *id003
65
+ - !ruby/object:Gem::Dependency
66
+ name: json
67
+ prerelease: false
68
+ requirement: &id004 !ruby/object:Gem::Requirement
69
+ none: false
70
+ requirements:
71
+ - - ">="
72
+ - !ruby/object:Gem::Version
73
+ hash: 3
74
+ segments:
75
+ - 0
76
+ version: "0"
77
+ type: :runtime
78
+ version_requirements: *id004
79
+ description:
80
+ email:
81
+ - evan.meagher@gmail.com
82
+ executables:
83
+ - fintop
84
+ extensions: []
85
+
86
+ extra_rdoc_files: []
87
+
88
+ files:
89
+ - .gitignore
90
+ - CHANGELOG.md
91
+ - Gemfile
92
+ - LICENSE.txt
93
+ - README.md
94
+ - Rakefile
95
+ - bin/fintop
96
+ - fintop.gemspec
97
+ - lib/fintop.rb
98
+ - lib/fintop/metrics.rb
99
+ - lib/fintop/printer.rb
100
+ - lib/fintop/probe.rb
101
+ - lib/fintop/threads.rb
102
+ - lib/fintop/version.rb
103
+ - test/test_helper.rb
104
+ - test/test_metrics.rb
105
+ has_rdoc: true
106
+ homepage:
107
+ licenses:
108
+ - MIT
109
+ post_install_message:
110
+ rdoc_options: []
111
+
112
+ require_paths:
113
+ - lib
114
+ required_ruby_version: !ruby/object:Gem::Requirement
115
+ none: false
116
+ requirements:
117
+ - - ">="
118
+ - !ruby/object:Gem::Version
119
+ hash: 3
120
+ segments:
121
+ - 0
122
+ version: "0"
123
+ required_rubygems_version: !ruby/object:Gem::Requirement
124
+ none: false
125
+ requirements:
126
+ - - ">="
127
+ - !ruby/object:Gem::Version
128
+ hash: 3
129
+ segments:
130
+ - 0
131
+ version: "0"
132
+ requirements: []
133
+
134
+ rubyforge_project:
135
+ rubygems_version: 1.6.2
136
+ signing_key:
137
+ specification_version: 3
138
+ summary: A top-like utility for monitoring Finagle servers
139
+ test_files:
140
+ - test/test_helper.rb
141
+ - test/test_metrics.rb