fluent-plugin-munin 0.3.1 → 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +17 -3
- data/example.conf +43 -0
- data/example2.conf +32 -0
- data/fluent-plugin-munin.gemspec +2 -2
- data/lib/fluent/plugin/in_munin.rb +53 -43
- metadata +27 -9
data/README.md
CHANGED
@@ -44,17 +44,22 @@ gem install fluent-plugin-munin
|
|
44
44
|
`````
|
45
45
|
|
46
46
|
### Output Sample
|
47
|
-
record_hostname: no, nest_result: no
|
47
|
+
record_hostname: no, nest_result: no #DEFAULT
|
48
48
|
`````
|
49
49
|
input.munin.cpu: {"service":"cpu","user":"113183","nice":"340","system":"26584","idle":"74205345","iowait":"26134","irq":"1","softirq":"506","steal":"0","guest":"0"}
|
50
50
|
`````
|
51
51
|
|
52
|
+
tag_prefix: input.${hostname}-munin, record_hostname: yes, nest_result: no
|
53
|
+
`````
|
54
|
+
input.myhost.example.com-munin.cpu: {"hostname":"myhost.example.com","service":"cpu","user":"113183","nice":"340","system":"26584","idle":"74205345","iowait":"26134","irq":"1","softirq":"506","steal":"0","guest":"0"}
|
55
|
+
`````
|
56
|
+
|
52
57
|
record_hostname: yes, nest_result: no
|
53
58
|
`````
|
54
59
|
input.munin.cpu: {"hostname":"myhost.example.com","service":"cpu","user":"113183","nice":"340","system":"26584","idle":"74205345","iowait":"26134","irq":"1","softirq":"506","steal":"0","guest":"0"}
|
55
60
|
`````
|
56
61
|
|
57
|
-
record_hostname: yes, nest_result: no, convert_type: yes
|
62
|
+
record_hostname: yes, nest_result: no, convert_type: yes #RECOMMEND
|
58
63
|
`````
|
59
64
|
input.munin.cpu: {"hostname":"myhost.example.com","service":"cpu","user":113183,"nice":340,"system":26584,"idle":74205345,"iowait":26134,"irq":1,"softirq":506,"steal":0,"guest":0}
|
60
65
|
`````
|
@@ -71,8 +76,17 @@ record_hostname: yes, nest_result: yes, convert_type: yes
|
|
71
76
|
> db.cpu.find({ "data.iowait" : { $gt : 200000 } })
|
72
77
|
`````
|
73
78
|
|
79
|
+
### Example
|
80
|
+
|
81
|
+
* Example1: how to send munin metrics to treasuredata.<br>
|
82
|
+
https://github.com/y-ken/fluent-plugin-munin/blob/master/example.conf
|
83
|
+
|
84
|
+
* Example2: how to send munin metrics to mongoDB.<br>
|
85
|
+
https://github.com/y-ken/fluent-plugin-munin/blob/master/example2.conf
|
86
|
+
|
87
|
+
|
74
88
|
## TODO
|
75
|
-
|
89
|
+
Pull requests are very welcome!!
|
76
90
|
|
77
91
|
## Copyright
|
78
92
|
|
data/example.conf
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
It is a sample to send munin metrics to treasuredata.
|
2
|
+
|
3
|
+
# appearing plugins:
|
4
|
+
# munin: http://rubygems.org/gems/fluent-plugin-munin
|
5
|
+
# rewrite_tag_filter: http://rubygems.org/gems/fluent-plugin-rewrite-tag-filter
|
6
|
+
# tdlog: http://rubygems.org/gems/fluent-plugin-td
|
7
|
+
|
8
|
+
# recieve all metrics from munin.
|
9
|
+
<source>
|
10
|
+
type munin
|
11
|
+
interval 60s
|
12
|
+
tag_prefix rewrite.munin
|
13
|
+
record_hostname yes
|
14
|
+
convert_type yes
|
15
|
+
</source>
|
16
|
+
|
17
|
+
# rewrite tag for "df" to pass the treasuredata table name length limitation.
|
18
|
+
# It could avoid the error below.
|
19
|
+
# "Name must be 3 to 256 characters, got 2 characters."
|
20
|
+
<match rewrite.munin.*>
|
21
|
+
type rewrite_tag_filter
|
22
|
+
rewriterule1 service ^df$ td.munin.diskfree
|
23
|
+
rewriterule2 service ^(.+)$ td.munin.$1
|
24
|
+
</match>
|
25
|
+
|
26
|
+
<match td.*.*>
|
27
|
+
type tdlog
|
28
|
+
apikey YOUR_API_KEY
|
29
|
+
auto_create_table
|
30
|
+
buffer_type file
|
31
|
+
buffer_path /var/log/td-agent/buffer/td
|
32
|
+
use_ssl true
|
33
|
+
</match>
|
34
|
+
|
35
|
+
# treasure data stored sample.
|
36
|
+
# $ td tables
|
37
|
+
# +----------+---------------------+------+-------+--------+
|
38
|
+
# | Database | Table | Type | Count | Schema |
|
39
|
+
# +----------+---------------------+------+-------+--------+
|
40
|
+
# | munin | cpu | log | 5 | |
|
41
|
+
# | munin | diskfree | log | 5 | |
|
42
|
+
# +----------+---------------------+------+-------+--------+
|
43
|
+
# 2 rows in set
|
data/example2.conf
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
It is a sample to send munin metrics to mongoDB.
|
2
|
+
|
3
|
+
# appearing plugins:
|
4
|
+
# munin: http://rubygems.org/gems/fluent-plugin-munin
|
5
|
+
# mongo: http://rubygems.org/gems/fluent-plugin-mongo
|
6
|
+
|
7
|
+
# recieve all metrics from munin.
|
8
|
+
<source>
|
9
|
+
type munin
|
10
|
+
interval 60s
|
11
|
+
tag_prefix mongo.munin
|
12
|
+
record_hostname yes
|
13
|
+
convert_type yes
|
14
|
+
</source>
|
15
|
+
|
16
|
+
<match mongo.*.*>
|
17
|
+
type mongo
|
18
|
+
host localhost
|
19
|
+
port 27017
|
20
|
+
database mongo
|
21
|
+
tag_mapped
|
22
|
+
</match>
|
23
|
+
|
24
|
+
# mongoDB stored sample.
|
25
|
+
# $ mongo
|
26
|
+
# MongoDB shell version: 2.2.0
|
27
|
+
# connecting to: test
|
28
|
+
# > use munin
|
29
|
+
# switched to db munin
|
30
|
+
# > show collections
|
31
|
+
# cpu
|
32
|
+
# df
|
data/fluent-plugin-munin.gemspec
CHANGED
@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-munin"
|
6
|
-
s.version = "0.3.
|
6
|
+
s.version = "0.3.2"
|
7
7
|
s.authors = ["Kentaro Yoshida"]
|
8
8
|
s.email = ["y.ken.studio@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/y-ken/fluent-plugin-munin"
|
10
|
-
s.summary = %q{Fluentd Input plugin to fetch munin-node metrics data with custom intervals.}
|
10
|
+
s.summary = %q{Fluentd Input plugin to fetch munin-node metrics data with custom intervals. It supports all of munin plugins.}
|
11
11
|
|
12
12
|
s.files = `git ls-files`.split("\n")
|
13
13
|
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
@@ -19,15 +19,31 @@ module Fluent
|
|
19
19
|
|
20
20
|
def configure(conf)
|
21
21
|
super
|
22
|
-
|
22
|
+
|
23
23
|
@interval = Config.time_value(@interval)
|
24
|
-
service_list = get_service_list
|
25
|
-
@services = @service == 'all' ? service_list : @service.split(',')
|
26
24
|
@nest_result = Config.bool_value(@nest_result) || false
|
27
25
|
@convert_type = Config.bool_value(@convert_type) || false
|
28
26
|
@record_hostname = Config.bool_value(@record_hostname) || false
|
29
|
-
|
30
|
-
|
27
|
+
|
28
|
+
self
|
29
|
+
end
|
30
|
+
|
31
|
+
def configure_munin
|
32
|
+
retry_interval = 30
|
33
|
+
max_retry_interval = retry_interval * 2 ** (8 - 1)
|
34
|
+
begin
|
35
|
+
@munin = get_connection
|
36
|
+
@hostname = @munin.nodes.join(',')
|
37
|
+
service_list = get_service_list
|
38
|
+
@services = @service == 'all' ? service_list : @service.split(',')
|
39
|
+
$log.info "munin: munin-node ready ", :hostname=>@hostname, :service_list=>service_list
|
40
|
+
$log.info "munin: activating service ", :service=>@services
|
41
|
+
rescue => e
|
42
|
+
$log.warn "munin: connect failed ", :error_class=>e.class, :error=>e.message, :retry_interval=>retry_interval
|
43
|
+
sleep retry_interval
|
44
|
+
retry_interval *= 2 if retry_interval < max_retry_interval
|
45
|
+
retry
|
46
|
+
end
|
31
47
|
end
|
32
48
|
|
33
49
|
def start
|
@@ -41,20 +57,9 @@ module Fluent
|
|
41
57
|
end
|
42
58
|
|
43
59
|
def run
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
record = Hash.new
|
48
|
-
record.store('hostname', @hostname) if @record_hostname
|
49
|
-
record.store('service', key)
|
50
|
-
if (@nest_result)
|
51
|
-
record.store(@nest_key, fetch(key))
|
52
|
-
else
|
53
|
-
record.merge!(fetch(key).to_hash)
|
54
|
-
end
|
55
|
-
Engine.emit(tag, Engine.now, record)
|
56
|
-
end
|
57
|
-
disconnect
|
60
|
+
configure_munin
|
61
|
+
loop do
|
62
|
+
emit_collected_service
|
58
63
|
sleep @interval
|
59
64
|
end
|
60
65
|
end
|
@@ -68,36 +73,34 @@ module Fluent
|
|
68
73
|
@munin.connection.close
|
69
74
|
end
|
70
75
|
|
71
|
-
def
|
72
|
-
@munin
|
73
|
-
|
74
|
-
return @munin.nodes.join(',')
|
75
|
-
rescue Munin::ConnectionError
|
76
|
-
@munin = get_connection
|
77
|
-
retry
|
78
|
-
end
|
76
|
+
def get_service_list
|
77
|
+
@munin = get_connection
|
78
|
+
return @munin.list
|
79
79
|
end
|
80
80
|
|
81
|
-
def
|
82
|
-
@
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
81
|
+
def emit_collected_service
|
82
|
+
@services.each do |key|
|
83
|
+
tag = "#{@tag_prefix}.#{key}".gsub(/(\${[a-z]+}|__[A-Z]+__)/, get_placeholder)
|
84
|
+
record = Hash.new
|
85
|
+
record.store('hostname', @hostname) if @record_hostname
|
86
|
+
record.store('service', key)
|
87
|
+
if (@nest_result)
|
88
|
+
record.store(@nest_key, fetch(key))
|
89
|
+
else
|
90
|
+
record.merge!(fetch(key).to_hash)
|
91
|
+
end
|
92
|
+
Engine.emit(tag, Engine.now, record)
|
88
93
|
end
|
94
|
+
disconnect
|
95
|
+
rescue => e
|
96
|
+
$log.warn "munin: fetch failed ", :error_class=>e.class, :error=>e.message
|
89
97
|
end
|
90
98
|
|
91
99
|
def fetch(key)
|
92
|
-
@munin
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
return values[key]
|
97
|
-
rescue Munin::ConnectionError
|
98
|
-
@munin = get_connection
|
99
|
-
retry
|
100
|
-
end
|
100
|
+
@munin = get_connection
|
101
|
+
values = @munin.fetch(key)
|
102
|
+
return convert_type(values[key]) if @convert_type
|
103
|
+
return values[key]
|
101
104
|
end
|
102
105
|
|
103
106
|
def convert_type(ary)
|
@@ -113,5 +116,12 @@ module Fluent
|
|
113
116
|
end
|
114
117
|
return data
|
115
118
|
end
|
119
|
+
|
120
|
+
def get_placeholder
|
121
|
+
return {
|
122
|
+
'__HOSTNAME__' => @hostname,
|
123
|
+
'${hostname}' => @hostname,
|
124
|
+
}
|
125
|
+
end
|
116
126
|
end
|
117
127
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-munin
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2013-07-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
16
|
-
requirement:
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,15 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :development
|
23
23
|
prerelease: false
|
24
|
-
version_requirements:
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
25
30
|
- !ruby/object:Gem::Dependency
|
26
31
|
name: fluentd
|
27
|
-
requirement:
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
28
33
|
none: false
|
29
34
|
requirements:
|
30
35
|
- - ! '>='
|
@@ -32,10 +37,15 @@ dependencies:
|
|
32
37
|
version: '0'
|
33
38
|
type: :runtime
|
34
39
|
prerelease: false
|
35
|
-
version_requirements:
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
36
46
|
- !ruby/object:Gem::Dependency
|
37
47
|
name: munin-ruby
|
38
|
-
requirement:
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
39
49
|
none: false
|
40
50
|
requirements:
|
41
51
|
- - ! '>='
|
@@ -43,7 +53,12 @@ dependencies:
|
|
43
53
|
version: '0'
|
44
54
|
type: :runtime
|
45
55
|
prerelease: false
|
46
|
-
version_requirements:
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
47
62
|
description:
|
48
63
|
email:
|
49
64
|
- y.ken.studio@gmail.com
|
@@ -55,6 +70,8 @@ files:
|
|
55
70
|
- Gemfile
|
56
71
|
- README.md
|
57
72
|
- Rakefile
|
73
|
+
- example.conf
|
74
|
+
- example2.conf
|
58
75
|
- fluent-plugin-munin.gemspec
|
59
76
|
- lib/fluent/plugin/in_munin.rb
|
60
77
|
- test/helper.rb
|
@@ -79,10 +96,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
79
96
|
version: '0'
|
80
97
|
requirements: []
|
81
98
|
rubyforge_project:
|
82
|
-
rubygems_version: 1.8.
|
99
|
+
rubygems_version: 1.8.23
|
83
100
|
signing_key:
|
84
101
|
specification_version: 3
|
85
102
|
summary: Fluentd Input plugin to fetch munin-node metrics data with custom intervals.
|
103
|
+
It supports all of munin plugins.
|
86
104
|
test_files:
|
87
105
|
- test/helper.rb
|
88
106
|
- test/plugin/test_in_munin.rb
|