bipbip 0.6.7 → 0.6.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +5 -1
- data/bin/bipbip +5 -6
- data/lib/bipbip.rb +6 -2
- data/lib/bipbip/agent.rb +2 -5
- data/lib/bipbip/config.rb +4 -7
- data/lib/bipbip/helper.rb +1 -2
- data/lib/bipbip/plugin.rb +21 -29
- data/lib/bipbip/plugin/apache2.rb +4 -6
- data/lib/bipbip/plugin/command.rb +10 -13
- data/lib/bipbip/plugin/elasticsearch.rb +56 -59
- data/lib/bipbip/plugin/fastcgi_php_apc.rb +4 -6
- data/lib/bipbip/plugin/fastcgi_php_fpm.rb +5 -7
- data/lib/bipbip/plugin/fastcgi_php_opcache.rb +15 -17
- data/lib/bipbip/plugin/gearman.rb +4 -6
- data/lib/bipbip/plugin/log_parser.rb +11 -15
- data/lib/bipbip/plugin/memcached.rb +5 -7
- data/lib/bipbip/plugin/mongodb.rb +39 -37
- data/lib/bipbip/plugin/monit.rb +12 -13
- data/lib/bipbip/plugin/mysql.rb +21 -23
- data/lib/bipbip/plugin/network.rb +3 -5
- data/lib/bipbip/plugin/nginx.rb +11 -13
- data/lib/bipbip/plugin/php_apc.rb +4 -6
- data/lib/bipbip/plugin/postfix.rb +2 -4
- data/lib/bipbip/plugin/puppet.rb +16 -18
- data/lib/bipbip/plugin/redis.rb +9 -11
- data/lib/bipbip/plugin/resque.rb +9 -9
- data/lib/bipbip/plugin/socket_redis.rb +7 -8
- data/lib/bipbip/storage.rb +5 -8
- data/lib/bipbip/storage/copperegg.rb +8 -11
- data/lib/bipbip/version.rb +1 -1
- data/lib/interruptible_sleep.rb +1 -1
- metadata +17 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 334c4a2cf9c0c11478f524f0e6316d71de27a055
|
4
|
+
data.tar.gz: 6c48b47f6ca4668bde7aa7345d7e4e7eb8532613
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bed5bf43747499e9874fa1cff47eb3774f002ce8fda0bda28d7243a7462a530060182f2deead017cbf4723a2bbf74122f7c3a30ed7ada2bb483dc7f78de816ad
|
7
|
+
data.tar.gz: 05e7335084e0c941c9c26ab3181f4dce7d9b405b1b7930023afcacac7f1f3d3f276d8963b3451a50998bd0a2b3a2cbb6677bb3bc576f2d9065316cb1e4fd5795
|
data/README.md
CHANGED
@@ -1,8 +1,12 @@
|
|
1
|
-
bipbip
|
1
|
+
bipbip
|
2
2
|
======
|
3
3
|
Agent to collect server metrics and send them to the [CopperEgg RevealMetrics](http://copperegg.com/) platform.
|
4
4
|
Plugins for different metrics available.
|
5
5
|
|
6
|
+
[![Build Status](https://img.shields.io/travis/cargomedia/bipbip/master.svg)](https://travis-ci.org/cargomedia/bipbip)
|
7
|
+
[![Gem Version](https://img.shields.io/gem/v/bipbip.svg)](https://rubygems.org/gems/bipbip)
|
8
|
+
|
9
|
+
|
6
10
|
Installation
|
7
11
|
------------
|
8
12
|
```
|
data/bin/bipbip
CHANGED
@@ -11,30 +11,29 @@ options = {}
|
|
11
11
|
optparse = OptionParser.new do |opts|
|
12
12
|
opts.banner = "Usage: #{name} [options]"
|
13
13
|
|
14
|
-
opts.on(
|
14
|
+
opts.on('-c', '--config PATH', 'Configuration file') do |c|
|
15
15
|
options[:config] = c
|
16
16
|
end
|
17
17
|
|
18
|
-
opts.on(
|
18
|
+
opts.on('-v', '--version', 'Version') do |_v|
|
19
19
|
puts "#{name} v#{version}"
|
20
20
|
exit
|
21
21
|
end
|
22
22
|
|
23
|
-
opts.on(
|
23
|
+
opts.on('-h', '--help', 'Show this help') do |_h|
|
24
24
|
puts opts
|
25
25
|
exit
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
29
|
-
|
30
29
|
begin
|
31
30
|
optparse.parse!
|
32
31
|
unless options[:config]
|
33
|
-
puts
|
32
|
+
puts 'Missing options: config file [-c PATH]'
|
34
33
|
exit
|
35
34
|
end
|
36
35
|
rescue OptionParser::InvalidOption, OptionParser::MissingArgument
|
37
|
-
puts
|
36
|
+
puts $ERROR_INFO.to_s
|
38
37
|
puts optparse
|
39
38
|
exit
|
40
39
|
end
|
data/lib/bipbip.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
module Bipbip
|
2
|
-
require 'rubygems'
|
2
|
+
require 'rubygems' # For ruby < 1.9
|
3
3
|
|
4
4
|
require 'copperegg/revealmetrics'
|
5
5
|
require 'yaml'
|
@@ -28,6 +28,10 @@ module Bipbip
|
|
28
28
|
end
|
29
29
|
|
30
30
|
def self.fqdn
|
31
|
-
@fqdn ||=
|
31
|
+
@fqdn ||= begin
|
32
|
+
Socket.gethostbyname(Socket.gethostname).first
|
33
|
+
rescue
|
34
|
+
Socket.gethostname
|
35
|
+
end
|
32
36
|
end
|
33
37
|
end
|
data/lib/bipbip/agent.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
module Bipbip
|
2
|
-
|
3
2
|
class Agent
|
4
3
|
include InterruptibleSleep
|
5
4
|
|
@@ -22,9 +21,7 @@ module Bipbip
|
|
22
21
|
Bipbip.logger.info 'Startup...'
|
23
22
|
Bipbip.logger.warn 'No storages configured' if @storages.empty?
|
24
23
|
|
25
|
-
if @plugins.empty?
|
26
|
-
raise 'No services configured'
|
27
|
-
end
|
24
|
+
fail 'No services configured' if @plugins.empty?
|
28
25
|
|
29
26
|
@storages.each do |storage|
|
30
27
|
@plugins.each do |plugin|
|
@@ -38,7 +35,7 @@ module Bipbip
|
|
38
35
|
start_plugin(plugin, @storages)
|
39
36
|
end
|
40
37
|
|
41
|
-
|
38
|
+
loop do
|
42
39
|
thread = ThreadsWait.new(@threads).next_wait
|
43
40
|
@threads.delete(thread)
|
44
41
|
plugin = thread['plugin']
|
data/lib/bipbip/config.rb
CHANGED
@@ -1,7 +1,5 @@
|
|
1
1
|
module Bipbip
|
2
|
-
|
3
2
|
class Config
|
4
|
-
|
5
3
|
attr_reader :plugins
|
6
4
|
attr_reader :storages
|
7
5
|
attr_reader :logger
|
@@ -16,11 +14,11 @@ module Bipbip
|
|
16
14
|
'frequency' => 60,
|
17
15
|
'include' => nil,
|
18
16
|
'services' => [],
|
19
|
-
'tags' => []
|
17
|
+
'tags' => []
|
20
18
|
}.merge(config)
|
21
19
|
|
22
20
|
logger = Logger.new(config['logfile'])
|
23
|
-
logger.level = Logger
|
21
|
+
logger.level = Logger.const_get(config['loglevel'])
|
24
22
|
|
25
23
|
plugins_config = config['services'].to_a
|
26
24
|
if config['include']
|
@@ -35,14 +33,14 @@ module Bipbip
|
|
35
33
|
metric_group = service['metric_group']
|
36
34
|
frequency = service['frequency'].nil? ? config['frequency'] : service['frequency']
|
37
35
|
tags = config['tags'].to_a + service['tags'].to_a
|
38
|
-
plugin_config = service.reject { |key,
|
36
|
+
plugin_config = service.reject { |key, _value| %w(plugin frequency tags metric_group).include?(key) }
|
39
37
|
Bipbip::Plugin.factory(plugin_name, plugin_config, frequency, tags, metric_group)
|
40
38
|
end
|
41
39
|
|
42
40
|
storages_config = config['storages'].to_a
|
43
41
|
storages = storages_config.map do |storage|
|
44
42
|
storage_name = storage['name'].to_s
|
45
|
-
storage_config = storage.reject { |key,
|
43
|
+
storage_config = storage.reject { |key, _value| ['name'].include?(key) }
|
46
44
|
Bipbip::Storage.factory(storage_name, storage_config)
|
47
45
|
end
|
48
46
|
|
@@ -57,6 +55,5 @@ module Bipbip
|
|
57
55
|
@storages = storages || []
|
58
56
|
@logger = logger || Logger.new(STDOUT)
|
59
57
|
end
|
60
|
-
|
61
58
|
end
|
62
59
|
end
|
data/lib/bipbip/helper.rb
CHANGED
data/lib/bipbip/plugin.rb
CHANGED
@@ -1,7 +1,5 @@
|
|
1
1
|
module Bipbip
|
2
|
-
|
3
2
|
class Plugin
|
4
|
-
|
5
3
|
class MeasurementTimeout < RuntimeError
|
6
4
|
end
|
7
5
|
|
@@ -15,7 +13,7 @@ module Bipbip
|
|
15
13
|
|
16
14
|
def self.factory(name, config, frequency, tags, metric_group = nil)
|
17
15
|
require "bipbip/plugin/#{Bipbip::Helper.name_to_filename(name)}"
|
18
|
-
Plugin
|
16
|
+
Plugin.const_get(Bipbip::Helper.name_to_classname(name)).new(name, config, frequency, tags, metric_group)
|
19
17
|
end
|
20
18
|
|
21
19
|
# @param [Bipbip::Plugin] plugin
|
@@ -34,31 +32,27 @@ module Bipbip
|
|
34
32
|
|
35
33
|
# @param [Array] storages
|
36
34
|
def run(storages)
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
run_measurement(time, storages)
|
43
|
-
end
|
44
|
-
interruptible_sleep (frequency - (Time.now - time))
|
35
|
+
timeout = frequency * 2
|
36
|
+
loop do
|
37
|
+
time = Time.now
|
38
|
+
Timeout.timeout(timeout, MeasurementTimeout) do
|
39
|
+
run_measurement(time, storages)
|
45
40
|
end
|
46
|
-
|
47
|
-
log(Logger::ERROR, "Measurement timeout of #{timeout} seconds reached.")
|
48
|
-
retry
|
49
|
-
rescue StandardError => e
|
50
|
-
log_exception(Logger::ERROR, e)
|
51
|
-
interruptible_sleep frequency
|
52
|
-
retry
|
53
|
-
rescue Exception => e
|
54
|
-
log_exception(Logger::FATAL, e)
|
55
|
-
raise e
|
41
|
+
interruptible_sleep (frequency - (Time.now - time))
|
56
42
|
end
|
43
|
+
rescue MeasurementTimeout => e
|
44
|
+
log(Logger::ERROR, "Measurement timeout of #{timeout} seconds reached.")
|
45
|
+
retry
|
46
|
+
rescue StandardError => e
|
47
|
+
log_exception(Logger::ERROR, e)
|
48
|
+
interruptible_sleep frequency
|
49
|
+
retry
|
50
|
+
rescue Exception => e
|
51
|
+
log_exception(Logger::FATAL, e)
|
52
|
+
raise e
|
57
53
|
end
|
58
54
|
|
59
|
-
|
60
|
-
@frequency
|
61
|
-
end
|
55
|
+
attr_reader :frequency
|
62
56
|
|
63
57
|
def source_identifier
|
64
58
|
identifier = Bipbip.fqdn + '::' + metric_group
|
@@ -73,20 +67,18 @@ module Bipbip
|
|
73
67
|
end
|
74
68
|
|
75
69
|
def metrics_schema
|
76
|
-
|
70
|
+
fail 'Missing method metrics_schema'
|
77
71
|
end
|
78
72
|
|
79
73
|
def monitor
|
80
|
-
|
74
|
+
fail 'Missing method monitor'
|
81
75
|
end
|
82
76
|
|
83
77
|
private
|
84
78
|
|
85
79
|
def run_measurement(time, storages)
|
86
80
|
data = monitor
|
87
|
-
if data.empty?
|
88
|
-
raise "#{name} #{source_identifier}: Empty data"
|
89
|
-
end
|
81
|
+
fail "#{name} #{source_identifier}: Empty data" if data.empty?
|
90
82
|
log(Logger::DEBUG, "Data: #{data}")
|
91
83
|
storages.each do |storage|
|
92
84
|
storage.store_sample(self, time, data)
|
@@ -1,11 +1,9 @@
|
|
1
1
|
module Bipbip
|
2
|
-
|
3
2
|
class Plugin::Apache2 < Plugin
|
4
|
-
|
5
3
|
def metrics_schema
|
6
4
|
[
|
7
|
-
|
8
|
-
|
5
|
+
{ name: 'request_per_sec', type: 'gauge', unit: 'Requests' },
|
6
|
+
{ name: 'busy_workers', type: 'gauge', unit: 'Workers' }
|
9
7
|
]
|
10
8
|
end
|
11
9
|
|
@@ -13,7 +11,7 @@ module Bipbip
|
|
13
11
|
uri = URI.parse(config['url'])
|
14
12
|
response = Net::HTTP.get_response(uri)
|
15
13
|
|
16
|
-
|
14
|
+
fail "Invalid response from server at #{config['url']}" unless response.code == '200'
|
17
15
|
|
18
16
|
astats = response.body.split(/\r*\n/)
|
19
17
|
|
@@ -23,7 +21,7 @@ module Bipbip
|
|
23
21
|
ainfo[name] = value
|
24
22
|
end
|
25
23
|
|
26
|
-
{:
|
24
|
+
{ request_per_sec: ainfo['ReqPerSec'].to_f, busy_workers: ainfo['BusyWorkers'].to_i }
|
27
25
|
end
|
28
26
|
end
|
29
27
|
end
|
@@ -2,9 +2,7 @@ require 'open3'
|
|
2
2
|
require 'json'
|
3
3
|
|
4
4
|
module Bipbip
|
5
|
-
|
6
5
|
class Plugin::Command < Plugin
|
7
|
-
|
8
6
|
attr_accessor :schema
|
9
7
|
|
10
8
|
def metrics_schema
|
@@ -19,24 +17,24 @@ module Bipbip
|
|
19
17
|
|
20
18
|
def metric_value(value)
|
21
19
|
value = value['value'] if detect_operation_mode(value) == :advanced
|
22
|
-
value = 1 if value == 'true'
|
23
|
-
value = 0 if value == 'false'
|
20
|
+
value = 1 if value == 'true' || value == true
|
21
|
+
value = 0 if value == 'false' || value == false
|
24
22
|
value
|
25
23
|
end
|
26
24
|
|
27
25
|
def find_schema
|
28
26
|
command_output.map do |metric, value|
|
29
27
|
case detect_operation_mode(value)
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
28
|
+
when :simple
|
29
|
+
{ name: "#{metric}", type: 'gauge' }
|
30
|
+
when :advanced
|
31
|
+
{ name: "#{metric}", type: value['type'], unit: value['unit'] }
|
34
32
|
end
|
35
33
|
end
|
36
34
|
end
|
37
35
|
|
38
36
|
def detect_operation_mode(value)
|
39
|
-
{true => :advanced, false => :simple}.fetch(value.is_a?(Hash))
|
37
|
+
{ true => :advanced, false => :simple }.fetch(value.is_a?(Hash))
|
40
38
|
end
|
41
39
|
|
42
40
|
def command_output
|
@@ -47,21 +45,20 @@ module Bipbip
|
|
47
45
|
command = config['command'].to_s
|
48
46
|
|
49
47
|
output_stdout = output_stderr = exit_code = nil
|
50
|
-
Open3.popen3(command)
|
48
|
+
Open3.popen3(command) do |_stdin, stdout, stderr, wait_thr|
|
51
49
|
output_stdout = stdout.read.chomp
|
52
50
|
output_stderr = stderr.read.chomp
|
53
51
|
exit_code = wait_thr.value
|
54
|
-
|
52
|
+
end
|
55
53
|
|
56
54
|
unless exit_code.success?
|
57
55
|
message = ['Command execution failed:', command]
|
58
56
|
message.push 'STDOUT:', output_stdout unless output_stdout.empty?
|
59
57
|
message.push 'STDERR:', output_stderr unless output_stderr.empty?
|
60
|
-
|
58
|
+
fail message.join("\n")
|
61
59
|
end
|
62
60
|
|
63
61
|
output_stdout
|
64
62
|
end
|
65
|
-
|
66
63
|
end
|
67
64
|
end
|
@@ -3,89 +3,86 @@ class ElasticsearchClient < Elasticsearch::Transport::Client
|
|
3
3
|
end
|
4
4
|
|
5
5
|
module Bipbip
|
6
|
-
|
7
6
|
class Plugin::Elasticsearch < Plugin
|
8
|
-
|
9
7
|
def metrics_schema
|
10
8
|
[
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
9
|
+
{ name: 'store_size', type: 'gauge', unit: 'b' },
|
10
|
+
{ name: 'docs_count', type: 'gauge', unit: 'Docs' },
|
11
|
+
{ name: 'docs_deleted', type: 'counter', unit: 'Deleted' },
|
12
|
+
{ name: 'segments_count', type: 'gauge', unit: 'Segments' },
|
13
|
+
|
14
|
+
{ name: 'search_query_total', type: 'counter', unit: 'Queries' },
|
15
|
+
{ name: 'search_query_time', type: 'counter', unit: 'Seconds' },
|
16
|
+
{ name: 'search_fetch_total', type: 'counter', unit: 'Fetches' },
|
17
|
+
{ name: 'search_fetch_time', type: 'counter', unit: 'Seconds' },
|
18
|
+
|
19
|
+
{ name: 'get_total', type: 'counter', unit: 'Gets' },
|
20
|
+
{ name: 'get_time', type: 'counter', unit: 'Seconds' },
|
21
|
+
{ name: 'get_exists_total', type: 'counter', unit: 'Exists' },
|
22
|
+
{ name: 'get_exists_time', type: 'counter', unit: 'Seconds' },
|
23
|
+
{ name: 'get_missing_total', type: 'counter', unit: 'Missing' },
|
24
|
+
{ name: 'get_missing_time', type: 'counter', unit: 'Seconds' },
|
25
|
+
|
26
|
+
{ name: 'indexing_index_total', type: 'counter', unit: 'Indexes' },
|
27
|
+
{ name: 'indexing_index_time', type: 'counter', unit: 'Seconds' },
|
28
|
+
{ name: 'indexing_delete_total', type: 'counter', unit: 'Deletes' },
|
29
|
+
{ name: 'indexing_delete_time', type: 'counter', unit: 'Seconds' },
|
30
|
+
|
31
|
+
{ name: 'cache_filter_size', type: 'gauge', unit: 'b' },
|
32
|
+
{ name: 'cache_filter_evictions', type: 'gauge', unit: 'b' },
|
33
|
+
{ name: 'cache_field_size', type: 'gauge', unit: 'b' },
|
34
|
+
{ name: 'cache_field_evictions', type: 'gauge', unit: 'b' }
|
37
35
|
]
|
38
36
|
end
|
39
37
|
|
40
38
|
def monitor
|
41
39
|
@stats = nil
|
42
40
|
{
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
41
|
+
'store_size' => stats_sum(%w(indices store size_in_bytes)),
|
42
|
+
'docs_count' => stats_sum(%w(indices docs count)),
|
43
|
+
'docs_deleted' => stats_sum(%w(indices docs deleted)),
|
44
|
+
'segments_count' => stats_sum(%w(indices segments count)),
|
45
|
+
|
46
|
+
'search_query_total' => stats_sum(%w(indices search query_total)),
|
47
|
+
'search_query_time' => stats_sum(%w(indices search query_time_in_millis)) / 1000,
|
48
|
+
'search_fetch_total' => stats_sum(%w(indices search fetch_total)),
|
49
|
+
'search_fetch_time' => stats_sum(%w(indices search fetch_time_in_millis)) / 1000,
|
50
|
+
|
51
|
+
'get_total' => stats_sum(%w(indices get total)),
|
52
|
+
'get_time' => stats_sum(%w(indices get time_in_millis)) / 1000,
|
53
|
+
'get_exists_total' => stats_sum(%w(indices get exists_total)),
|
54
|
+
'get_exists_time' => stats_sum(%w(indices get exists_time_in_millis)) / 1000,
|
55
|
+
'get_missing_total' => stats_sum(%w(indices get missing_total)),
|
56
|
+
'get_missing_time' => stats_sum(%w(indices get missing_time_in_millis)) / 1000,
|
57
|
+
|
58
|
+
'indexing_index_total' => stats_sum(%w(indices indexing index_total)),
|
59
|
+
'indexing_index_time' => stats_sum(%w(indices indexing index_time_in_millis)) / 1000,
|
60
|
+
'indexing_delete_total' => stats_sum(%w(indices indexing delete_total)),
|
61
|
+
'indexing_delete_time' => stats_sum(%w(indices indexing delete_time_in_millis)) / 1000,
|
62
|
+
|
63
|
+
'cache_filter_size' => stats_sum(%w(indices filter_cache memory_size_in_bytes)),
|
64
|
+
'cache_filter_evictions' => stats_sum(%w(indices filter_cache evictions)),
|
65
|
+
'cache_field_size' => stats_sum(%w(indices fielddata memory_size_in_bytes)),
|
66
|
+
'cache_field_evictions' => stats_sum(%w(indices fielddata evictions))
|
69
67
|
}
|
70
68
|
end
|
71
69
|
|
72
70
|
private
|
73
71
|
|
74
72
|
def connection
|
75
|
-
ElasticsearchClient.new(
|
73
|
+
ElasticsearchClient.new(host: [config['hostname'], config['port']].join(':'))
|
76
74
|
end
|
77
75
|
|
78
76
|
def nodes_stats
|
79
|
-
connection.nodes.stats(
|
77
|
+
connection.nodes.stats(node_id: '_local')
|
80
78
|
end
|
81
79
|
|
82
80
|
def stats_sum(keys)
|
83
81
|
sum = 0
|
84
|
-
(@stats ||= nodes_stats)['nodes'].each do |
|
85
|
-
sum += keys.inject(stats) { |
|
82
|
+
(@stats ||= nodes_stats)['nodes'].each do |_node, stats|
|
83
|
+
sum += keys.inject(stats) { |a, e| (a.is_a?(Hash) && !a[e].nil?) ? a[e] : 0 }
|
86
84
|
end
|
87
85
|
sum
|
88
86
|
end
|
89
|
-
|
90
87
|
end
|
91
88
|
end
|