bipbip 0.6.7 → 0.6.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -1
- data/bin/bipbip +5 -6
- data/lib/bipbip.rb +6 -2
- data/lib/bipbip/agent.rb +2 -5
- data/lib/bipbip/config.rb +4 -7
- data/lib/bipbip/helper.rb +1 -2
- data/lib/bipbip/plugin.rb +21 -29
- data/lib/bipbip/plugin/apache2.rb +4 -6
- data/lib/bipbip/plugin/command.rb +10 -13
- data/lib/bipbip/plugin/elasticsearch.rb +56 -59
- data/lib/bipbip/plugin/fastcgi_php_apc.rb +4 -6
- data/lib/bipbip/plugin/fastcgi_php_fpm.rb +5 -7
- data/lib/bipbip/plugin/fastcgi_php_opcache.rb +15 -17
- data/lib/bipbip/plugin/gearman.rb +4 -6
- data/lib/bipbip/plugin/log_parser.rb +11 -15
- data/lib/bipbip/plugin/memcached.rb +5 -7
- data/lib/bipbip/plugin/mongodb.rb +39 -37
- data/lib/bipbip/plugin/monit.rb +12 -13
- data/lib/bipbip/plugin/mysql.rb +21 -23
- data/lib/bipbip/plugin/network.rb +3 -5
- data/lib/bipbip/plugin/nginx.rb +11 -13
- data/lib/bipbip/plugin/php_apc.rb +4 -6
- data/lib/bipbip/plugin/postfix.rb +2 -4
- data/lib/bipbip/plugin/puppet.rb +16 -18
- data/lib/bipbip/plugin/redis.rb +9 -11
- data/lib/bipbip/plugin/resque.rb +9 -9
- data/lib/bipbip/plugin/socket_redis.rb +7 -8
- data/lib/bipbip/storage.rb +5 -8
- data/lib/bipbip/storage/copperegg.rb +8 -11
- data/lib/bipbip/version.rb +1 -1
- data/lib/interruptible_sleep.rb +1 -1
- metadata +17 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 334c4a2cf9c0c11478f524f0e6316d71de27a055
|
4
|
+
data.tar.gz: 6c48b47f6ca4668bde7aa7345d7e4e7eb8532613
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bed5bf43747499e9874fa1cff47eb3774f002ce8fda0bda28d7243a7462a530060182f2deead017cbf4723a2bbf74122f7c3a30ed7ada2bb483dc7f78de816ad
|
7
|
+
data.tar.gz: 05e7335084e0c941c9c26ab3181f4dce7d9b405b1b7930023afcacac7f1f3d3f276d8963b3451a50998bd0a2b3a2cbb6677bb3bc576f2d9065316cb1e4fd5795
|
data/README.md
CHANGED
@@ -1,8 +1,12 @@
|
|
1
|
-
bipbip
|
1
|
+
bipbip
|
2
2
|
======
|
3
3
|
Agent to collect server metrics and send them to the [CopperEgg RevealMetrics](http://copperegg.com/) platform.
|
4
4
|
Plugins for different metrics available.
|
5
5
|
|
6
|
+
[](https://travis-ci.org/cargomedia/bipbip)
|
7
|
+
[](https://rubygems.org/gems/bipbip)
|
8
|
+
|
9
|
+
|
6
10
|
Installation
|
7
11
|
------------
|
8
12
|
```
|
data/bin/bipbip
CHANGED
@@ -11,30 +11,29 @@ options = {}
|
|
11
11
|
optparse = OptionParser.new do |opts|
|
12
12
|
opts.banner = "Usage: #{name} [options]"
|
13
13
|
|
14
|
-
opts.on(
|
14
|
+
opts.on('-c', '--config PATH', 'Configuration file') do |c|
|
15
15
|
options[:config] = c
|
16
16
|
end
|
17
17
|
|
18
|
-
opts.on(
|
18
|
+
opts.on('-v', '--version', 'Version') do |_v|
|
19
19
|
puts "#{name} v#{version}"
|
20
20
|
exit
|
21
21
|
end
|
22
22
|
|
23
|
-
opts.on(
|
23
|
+
opts.on('-h', '--help', 'Show this help') do |_h|
|
24
24
|
puts opts
|
25
25
|
exit
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
29
|
-
|
30
29
|
begin
|
31
30
|
optparse.parse!
|
32
31
|
unless options[:config]
|
33
|
-
puts
|
32
|
+
puts 'Missing options: config file [-c PATH]'
|
34
33
|
exit
|
35
34
|
end
|
36
35
|
rescue OptionParser::InvalidOption, OptionParser::MissingArgument
|
37
|
-
puts
|
36
|
+
puts $ERROR_INFO.to_s
|
38
37
|
puts optparse
|
39
38
|
exit
|
40
39
|
end
|
data/lib/bipbip.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
module Bipbip
|
2
|
-
require 'rubygems'
|
2
|
+
require 'rubygems' # For ruby < 1.9
|
3
3
|
|
4
4
|
require 'copperegg/revealmetrics'
|
5
5
|
require 'yaml'
|
@@ -28,6 +28,10 @@ module Bipbip
|
|
28
28
|
end
|
29
29
|
|
30
30
|
def self.fqdn
|
31
|
-
@fqdn ||=
|
31
|
+
@fqdn ||= begin
|
32
|
+
Socket.gethostbyname(Socket.gethostname).first
|
33
|
+
rescue
|
34
|
+
Socket.gethostname
|
35
|
+
end
|
32
36
|
end
|
33
37
|
end
|
data/lib/bipbip/agent.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
module Bipbip
|
2
|
-
|
3
2
|
class Agent
|
4
3
|
include InterruptibleSleep
|
5
4
|
|
@@ -22,9 +21,7 @@ module Bipbip
|
|
22
21
|
Bipbip.logger.info 'Startup...'
|
23
22
|
Bipbip.logger.warn 'No storages configured' if @storages.empty?
|
24
23
|
|
25
|
-
if @plugins.empty?
|
26
|
-
raise 'No services configured'
|
27
|
-
end
|
24
|
+
fail 'No services configured' if @plugins.empty?
|
28
25
|
|
29
26
|
@storages.each do |storage|
|
30
27
|
@plugins.each do |plugin|
|
@@ -38,7 +35,7 @@ module Bipbip
|
|
38
35
|
start_plugin(plugin, @storages)
|
39
36
|
end
|
40
37
|
|
41
|
-
|
38
|
+
loop do
|
42
39
|
thread = ThreadsWait.new(@threads).next_wait
|
43
40
|
@threads.delete(thread)
|
44
41
|
plugin = thread['plugin']
|
data/lib/bipbip/config.rb
CHANGED
@@ -1,7 +1,5 @@
|
|
1
1
|
module Bipbip
|
2
|
-
|
3
2
|
class Config
|
4
|
-
|
5
3
|
attr_reader :plugins
|
6
4
|
attr_reader :storages
|
7
5
|
attr_reader :logger
|
@@ -16,11 +14,11 @@ module Bipbip
|
|
16
14
|
'frequency' => 60,
|
17
15
|
'include' => nil,
|
18
16
|
'services' => [],
|
19
|
-
'tags' => []
|
17
|
+
'tags' => []
|
20
18
|
}.merge(config)
|
21
19
|
|
22
20
|
logger = Logger.new(config['logfile'])
|
23
|
-
logger.level = Logger
|
21
|
+
logger.level = Logger.const_get(config['loglevel'])
|
24
22
|
|
25
23
|
plugins_config = config['services'].to_a
|
26
24
|
if config['include']
|
@@ -35,14 +33,14 @@ module Bipbip
|
|
35
33
|
metric_group = service['metric_group']
|
36
34
|
frequency = service['frequency'].nil? ? config['frequency'] : service['frequency']
|
37
35
|
tags = config['tags'].to_a + service['tags'].to_a
|
38
|
-
plugin_config = service.reject { |key,
|
36
|
+
plugin_config = service.reject { |key, _value| %w(plugin frequency tags metric_group).include?(key) }
|
39
37
|
Bipbip::Plugin.factory(plugin_name, plugin_config, frequency, tags, metric_group)
|
40
38
|
end
|
41
39
|
|
42
40
|
storages_config = config['storages'].to_a
|
43
41
|
storages = storages_config.map do |storage|
|
44
42
|
storage_name = storage['name'].to_s
|
45
|
-
storage_config = storage.reject { |key,
|
43
|
+
storage_config = storage.reject { |key, _value| ['name'].include?(key) }
|
46
44
|
Bipbip::Storage.factory(storage_name, storage_config)
|
47
45
|
end
|
48
46
|
|
@@ -57,6 +55,5 @@ module Bipbip
|
|
57
55
|
@storages = storages || []
|
58
56
|
@logger = logger || Logger.new(STDOUT)
|
59
57
|
end
|
60
|
-
|
61
58
|
end
|
62
59
|
end
|
data/lib/bipbip/helper.rb
CHANGED
data/lib/bipbip/plugin.rb
CHANGED
@@ -1,7 +1,5 @@
|
|
1
1
|
module Bipbip
|
2
|
-
|
3
2
|
class Plugin
|
4
|
-
|
5
3
|
class MeasurementTimeout < RuntimeError
|
6
4
|
end
|
7
5
|
|
@@ -15,7 +13,7 @@ module Bipbip
|
|
15
13
|
|
16
14
|
def self.factory(name, config, frequency, tags, metric_group = nil)
|
17
15
|
require "bipbip/plugin/#{Bipbip::Helper.name_to_filename(name)}"
|
18
|
-
Plugin
|
16
|
+
Plugin.const_get(Bipbip::Helper.name_to_classname(name)).new(name, config, frequency, tags, metric_group)
|
19
17
|
end
|
20
18
|
|
21
19
|
# @param [Bipbip::Plugin] plugin
|
@@ -34,31 +32,27 @@ module Bipbip
|
|
34
32
|
|
35
33
|
# @param [Array] storages
|
36
34
|
def run(storages)
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
run_measurement(time, storages)
|
43
|
-
end
|
44
|
-
interruptible_sleep (frequency - (Time.now - time))
|
35
|
+
timeout = frequency * 2
|
36
|
+
loop do
|
37
|
+
time = Time.now
|
38
|
+
Timeout.timeout(timeout, MeasurementTimeout) do
|
39
|
+
run_measurement(time, storages)
|
45
40
|
end
|
46
|
-
|
47
|
-
log(Logger::ERROR, "Measurement timeout of #{timeout} seconds reached.")
|
48
|
-
retry
|
49
|
-
rescue StandardError => e
|
50
|
-
log_exception(Logger::ERROR, e)
|
51
|
-
interruptible_sleep frequency
|
52
|
-
retry
|
53
|
-
rescue Exception => e
|
54
|
-
log_exception(Logger::FATAL, e)
|
55
|
-
raise e
|
41
|
+
interruptible_sleep (frequency - (Time.now - time))
|
56
42
|
end
|
43
|
+
rescue MeasurementTimeout => e
|
44
|
+
log(Logger::ERROR, "Measurement timeout of #{timeout} seconds reached.")
|
45
|
+
retry
|
46
|
+
rescue StandardError => e
|
47
|
+
log_exception(Logger::ERROR, e)
|
48
|
+
interruptible_sleep frequency
|
49
|
+
retry
|
50
|
+
rescue Exception => e
|
51
|
+
log_exception(Logger::FATAL, e)
|
52
|
+
raise e
|
57
53
|
end
|
58
54
|
|
59
|
-
|
60
|
-
@frequency
|
61
|
-
end
|
55
|
+
attr_reader :frequency
|
62
56
|
|
63
57
|
def source_identifier
|
64
58
|
identifier = Bipbip.fqdn + '::' + metric_group
|
@@ -73,20 +67,18 @@ module Bipbip
|
|
73
67
|
end
|
74
68
|
|
75
69
|
def metrics_schema
|
76
|
-
|
70
|
+
fail 'Missing method metrics_schema'
|
77
71
|
end
|
78
72
|
|
79
73
|
def monitor
|
80
|
-
|
74
|
+
fail 'Missing method monitor'
|
81
75
|
end
|
82
76
|
|
83
77
|
private
|
84
78
|
|
85
79
|
def run_measurement(time, storages)
|
86
80
|
data = monitor
|
87
|
-
if data.empty?
|
88
|
-
raise "#{name} #{source_identifier}: Empty data"
|
89
|
-
end
|
81
|
+
fail "#{name} #{source_identifier}: Empty data" if data.empty?
|
90
82
|
log(Logger::DEBUG, "Data: #{data}")
|
91
83
|
storages.each do |storage|
|
92
84
|
storage.store_sample(self, time, data)
|
@@ -1,11 +1,9 @@
|
|
1
1
|
module Bipbip
|
2
|
-
|
3
2
|
class Plugin::Apache2 < Plugin
|
4
|
-
|
5
3
|
def metrics_schema
|
6
4
|
[
|
7
|
-
|
8
|
-
|
5
|
+
{ name: 'request_per_sec', type: 'gauge', unit: 'Requests' },
|
6
|
+
{ name: 'busy_workers', type: 'gauge', unit: 'Workers' }
|
9
7
|
]
|
10
8
|
end
|
11
9
|
|
@@ -13,7 +11,7 @@ module Bipbip
|
|
13
11
|
uri = URI.parse(config['url'])
|
14
12
|
response = Net::HTTP.get_response(uri)
|
15
13
|
|
16
|
-
|
14
|
+
fail "Invalid response from server at #{config['url']}" unless response.code == '200'
|
17
15
|
|
18
16
|
astats = response.body.split(/\r*\n/)
|
19
17
|
|
@@ -23,7 +21,7 @@ module Bipbip
|
|
23
21
|
ainfo[name] = value
|
24
22
|
end
|
25
23
|
|
26
|
-
{:
|
24
|
+
{ request_per_sec: ainfo['ReqPerSec'].to_f, busy_workers: ainfo['BusyWorkers'].to_i }
|
27
25
|
end
|
28
26
|
end
|
29
27
|
end
|
@@ -2,9 +2,7 @@ require 'open3'
|
|
2
2
|
require 'json'
|
3
3
|
|
4
4
|
module Bipbip
|
5
|
-
|
6
5
|
class Plugin::Command < Plugin
|
7
|
-
|
8
6
|
attr_accessor :schema
|
9
7
|
|
10
8
|
def metrics_schema
|
@@ -19,24 +17,24 @@ module Bipbip
|
|
19
17
|
|
20
18
|
def metric_value(value)
|
21
19
|
value = value['value'] if detect_operation_mode(value) == :advanced
|
22
|
-
value = 1 if value == 'true'
|
23
|
-
value = 0 if value == 'false'
|
20
|
+
value = 1 if value == 'true' || value == true
|
21
|
+
value = 0 if value == 'false' || value == false
|
24
22
|
value
|
25
23
|
end
|
26
24
|
|
27
25
|
def find_schema
|
28
26
|
command_output.map do |metric, value|
|
29
27
|
case detect_operation_mode(value)
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
28
|
+
when :simple
|
29
|
+
{ name: "#{metric}", type: 'gauge' }
|
30
|
+
when :advanced
|
31
|
+
{ name: "#{metric}", type: value['type'], unit: value['unit'] }
|
34
32
|
end
|
35
33
|
end
|
36
34
|
end
|
37
35
|
|
38
36
|
def detect_operation_mode(value)
|
39
|
-
{true => :advanced, false => :simple}.fetch(value.is_a?(Hash))
|
37
|
+
{ true => :advanced, false => :simple }.fetch(value.is_a?(Hash))
|
40
38
|
end
|
41
39
|
|
42
40
|
def command_output
|
@@ -47,21 +45,20 @@ module Bipbip
|
|
47
45
|
command = config['command'].to_s
|
48
46
|
|
49
47
|
output_stdout = output_stderr = exit_code = nil
|
50
|
-
Open3.popen3(command)
|
48
|
+
Open3.popen3(command) do |_stdin, stdout, stderr, wait_thr|
|
51
49
|
output_stdout = stdout.read.chomp
|
52
50
|
output_stderr = stderr.read.chomp
|
53
51
|
exit_code = wait_thr.value
|
54
|
-
|
52
|
+
end
|
55
53
|
|
56
54
|
unless exit_code.success?
|
57
55
|
message = ['Command execution failed:', command]
|
58
56
|
message.push 'STDOUT:', output_stdout unless output_stdout.empty?
|
59
57
|
message.push 'STDERR:', output_stderr unless output_stderr.empty?
|
60
|
-
|
58
|
+
fail message.join("\n")
|
61
59
|
end
|
62
60
|
|
63
61
|
output_stdout
|
64
62
|
end
|
65
|
-
|
66
63
|
end
|
67
64
|
end
|
@@ -3,89 +3,86 @@ class ElasticsearchClient < Elasticsearch::Transport::Client
|
|
3
3
|
end
|
4
4
|
|
5
5
|
module Bipbip
|
6
|
-
|
7
6
|
class Plugin::Elasticsearch < Plugin
|
8
|
-
|
9
7
|
def metrics_schema
|
10
8
|
[
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
9
|
+
{ name: 'store_size', type: 'gauge', unit: 'b' },
|
10
|
+
{ name: 'docs_count', type: 'gauge', unit: 'Docs' },
|
11
|
+
{ name: 'docs_deleted', type: 'counter', unit: 'Deleted' },
|
12
|
+
{ name: 'segments_count', type: 'gauge', unit: 'Segments' },
|
13
|
+
|
14
|
+
{ name: 'search_query_total', type: 'counter', unit: 'Queries' },
|
15
|
+
{ name: 'search_query_time', type: 'counter', unit: 'Seconds' },
|
16
|
+
{ name: 'search_fetch_total', type: 'counter', unit: 'Fetches' },
|
17
|
+
{ name: 'search_fetch_time', type: 'counter', unit: 'Seconds' },
|
18
|
+
|
19
|
+
{ name: 'get_total', type: 'counter', unit: 'Gets' },
|
20
|
+
{ name: 'get_time', type: 'counter', unit: 'Seconds' },
|
21
|
+
{ name: 'get_exists_total', type: 'counter', unit: 'Exists' },
|
22
|
+
{ name: 'get_exists_time', type: 'counter', unit: 'Seconds' },
|
23
|
+
{ name: 'get_missing_total', type: 'counter', unit: 'Missing' },
|
24
|
+
{ name: 'get_missing_time', type: 'counter', unit: 'Seconds' },
|
25
|
+
|
26
|
+
{ name: 'indexing_index_total', type: 'counter', unit: 'Indexes' },
|
27
|
+
{ name: 'indexing_index_time', type: 'counter', unit: 'Seconds' },
|
28
|
+
{ name: 'indexing_delete_total', type: 'counter', unit: 'Deletes' },
|
29
|
+
{ name: 'indexing_delete_time', type: 'counter', unit: 'Seconds' },
|
30
|
+
|
31
|
+
{ name: 'cache_filter_size', type: 'gauge', unit: 'b' },
|
32
|
+
{ name: 'cache_filter_evictions', type: 'gauge', unit: 'b' },
|
33
|
+
{ name: 'cache_field_size', type: 'gauge', unit: 'b' },
|
34
|
+
{ name: 'cache_field_evictions', type: 'gauge', unit: 'b' }
|
37
35
|
]
|
38
36
|
end
|
39
37
|
|
40
38
|
def monitor
|
41
39
|
@stats = nil
|
42
40
|
{
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
41
|
+
'store_size' => stats_sum(%w(indices store size_in_bytes)),
|
42
|
+
'docs_count' => stats_sum(%w(indices docs count)),
|
43
|
+
'docs_deleted' => stats_sum(%w(indices docs deleted)),
|
44
|
+
'segments_count' => stats_sum(%w(indices segments count)),
|
45
|
+
|
46
|
+
'search_query_total' => stats_sum(%w(indices search query_total)),
|
47
|
+
'search_query_time' => stats_sum(%w(indices search query_time_in_millis)) / 1000,
|
48
|
+
'search_fetch_total' => stats_sum(%w(indices search fetch_total)),
|
49
|
+
'search_fetch_time' => stats_sum(%w(indices search fetch_time_in_millis)) / 1000,
|
50
|
+
|
51
|
+
'get_total' => stats_sum(%w(indices get total)),
|
52
|
+
'get_time' => stats_sum(%w(indices get time_in_millis)) / 1000,
|
53
|
+
'get_exists_total' => stats_sum(%w(indices get exists_total)),
|
54
|
+
'get_exists_time' => stats_sum(%w(indices get exists_time_in_millis)) / 1000,
|
55
|
+
'get_missing_total' => stats_sum(%w(indices get missing_total)),
|
56
|
+
'get_missing_time' => stats_sum(%w(indices get missing_time_in_millis)) / 1000,
|
57
|
+
|
58
|
+
'indexing_index_total' => stats_sum(%w(indices indexing index_total)),
|
59
|
+
'indexing_index_time' => stats_sum(%w(indices indexing index_time_in_millis)) / 1000,
|
60
|
+
'indexing_delete_total' => stats_sum(%w(indices indexing delete_total)),
|
61
|
+
'indexing_delete_time' => stats_sum(%w(indices indexing delete_time_in_millis)) / 1000,
|
62
|
+
|
63
|
+
'cache_filter_size' => stats_sum(%w(indices filter_cache memory_size_in_bytes)),
|
64
|
+
'cache_filter_evictions' => stats_sum(%w(indices filter_cache evictions)),
|
65
|
+
'cache_field_size' => stats_sum(%w(indices fielddata memory_size_in_bytes)),
|
66
|
+
'cache_field_evictions' => stats_sum(%w(indices fielddata evictions))
|
69
67
|
}
|
70
68
|
end
|
71
69
|
|
72
70
|
private
|
73
71
|
|
74
72
|
def connection
|
75
|
-
ElasticsearchClient.new(
|
73
|
+
ElasticsearchClient.new(host: [config['hostname'], config['port']].join(':'))
|
76
74
|
end
|
77
75
|
|
78
76
|
def nodes_stats
|
79
|
-
connection.nodes.stats(
|
77
|
+
connection.nodes.stats(node_id: '_local')
|
80
78
|
end
|
81
79
|
|
82
80
|
def stats_sum(keys)
|
83
81
|
sum = 0
|
84
|
-
(@stats ||= nodes_stats)['nodes'].each do |
|
85
|
-
sum += keys.inject(stats) { |
|
82
|
+
(@stats ||= nodes_stats)['nodes'].each do |_node, stats|
|
83
|
+
sum += keys.inject(stats) { |a, e| (a.is_a?(Hash) && !a[e].nil?) ? a[e] : 0 }
|
86
84
|
end
|
87
85
|
sum
|
88
86
|
end
|
89
|
-
|
90
87
|
end
|
91
88
|
end
|