fluent-plugin-prometheus 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +15 -0
- data/.rspec +2 -0
- data/.travis.yml +14 -0
- data/Gemfile +4 -0
- data/Gemfile.fluentd.0.10 +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +350 -0
- data/Rakefile +7 -0
- data/fluent-plugin-prometheus.gemspec +22 -0
- data/lib/fluent/plugin/filter_prometheus.rb +24 -0
- data/lib/fluent/plugin/in_prometheus.rb +61 -0
- data/lib/fluent/plugin/in_prometheus_monitor.rb +98 -0
- data/lib/fluent/plugin/out_prometheus.rb +24 -0
- data/lib/fluent/plugin/prometheus.rb +195 -0
- data/misc/fluentd_sample.conf +135 -0
- data/misc/nginx_proxy.conf +22 -0
- data/misc/prometheus.conf +10 -0
- data/spec/fluent/plugin/filter_prometheus_spec.rb +34 -0
- data/spec/fluent/plugin/out_prometheus_spec.rb +30 -0
- data/spec/fluent/plugin/prometheus_spec.rb +76 -0
- data/spec/fluent/plugin/shared.rb +210 -0
- data/spec/spec_helper.rb +8 -0
- metadata +155 -0
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'fluent/plugin/prometheus'
|
2
|
+
|
3
|
+
module Fluent
|
4
|
+
class PrometheusFilter < Filter
|
5
|
+
Plugin.register_filter('prometheus', self)
|
6
|
+
include Fluent::Prometheus
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
super
|
10
|
+
@registry = ::Prometheus::Client.registry
|
11
|
+
end
|
12
|
+
|
13
|
+
def configure(conf)
|
14
|
+
super
|
15
|
+
labels = Fluent::Prometheus.parse_labels_elements(conf)
|
16
|
+
@metrics = Fluent::Prometheus.parse_metrics_elements(conf, @registry, labels)
|
17
|
+
end
|
18
|
+
|
19
|
+
def filter_stream(tag, es)
|
20
|
+
instrument(tag, es, @metrics)
|
21
|
+
es
|
22
|
+
end
|
23
|
+
end if defined?(Filter)
|
24
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'fluent/plugin/prometheus'
|
2
|
+
require 'webrick'
|
3
|
+
|
4
|
+
module Fluent
|
5
|
+
class PrometheusInput < Input
|
6
|
+
Plugin.register_input('prometheus', self)
|
7
|
+
|
8
|
+
config_param :bind, :string, :default => '0.0.0.0'
|
9
|
+
config_param :port, :integer, :default => 24231
|
10
|
+
config_param :metrics_path, :string, :default => '/metrics'
|
11
|
+
|
12
|
+
attr_reader :registry
|
13
|
+
|
14
|
+
def initialize
|
15
|
+
super
|
16
|
+
@registry = ::Prometheus::Client.registry
|
17
|
+
end
|
18
|
+
|
19
|
+
def configure(conf)
|
20
|
+
super
|
21
|
+
end
|
22
|
+
|
23
|
+
def start
|
24
|
+
@server = WEBrick::HTTPServer.new(
|
25
|
+
BindAddress: @bind,
|
26
|
+
Port: @port,
|
27
|
+
Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
|
28
|
+
AccessLog: [],
|
29
|
+
)
|
30
|
+
@server.mount(@metrics_path, MonitorServlet, self)
|
31
|
+
@thread = Thread.new { @server.start }
|
32
|
+
end
|
33
|
+
|
34
|
+
def shutdown
|
35
|
+
if @server
|
36
|
+
@server.shutdown
|
37
|
+
@server = nil
|
38
|
+
end
|
39
|
+
if @thread
|
40
|
+
@thread.join
|
41
|
+
@thread = nil
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
|
46
|
+
def initialize(server, prometheus)
|
47
|
+
@prometheus = prometheus
|
48
|
+
end
|
49
|
+
|
50
|
+
def do_GET(req, res)
|
51
|
+
res.status = 200
|
52
|
+
res['Content-Type'] = ::Prometheus::Client::Formats::Text::CONTENT_TYPE
|
53
|
+
res.body = ::Prometheus::Client::Formats::Text.marshal(@prometheus.registry)
|
54
|
+
rescue
|
55
|
+
res.status = 500
|
56
|
+
res['Content-Type'] = 'text/plain'
|
57
|
+
res.body = $!.to_s
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
require 'fluent/plugin/prometheus'
|
2
|
+
require 'webrick'
|
3
|
+
|
4
|
+
module Fluent
|
5
|
+
class PrometheusMonitorInput < Input
|
6
|
+
Plugin.register_input('prometheus_monitor', self)
|
7
|
+
|
8
|
+
config_param :interval, :time, :default => 5
|
9
|
+
attr_reader :registry
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
super
|
13
|
+
@registry = ::Prometheus::Client.registry
|
14
|
+
end
|
15
|
+
|
16
|
+
def configure(conf)
|
17
|
+
super
|
18
|
+
hostname = Socket.gethostname
|
19
|
+
expander = Fluent::Prometheus.placeholder_expnader(log)
|
20
|
+
expander.prepare_placeholders(Time.now.to_i, {}, {'hostname' => hostname})
|
21
|
+
@base_labels = Fluent::Prometheus.parse_labels_elements(conf)
|
22
|
+
@base_labels.each do |key, value|
|
23
|
+
@base_labels[key] = expander.expand(value)
|
24
|
+
end
|
25
|
+
|
26
|
+
@monitor_agent = MonitorAgentInput.new
|
27
|
+
|
28
|
+
buffer_queue_length = @registry.gauge(
|
29
|
+
:fluentd_status_buffer_queue_length,
|
30
|
+
'Current buffer queue length.')
|
31
|
+
buffer_total_queued_size = @registry.gauge(
|
32
|
+
:fluentd_status_buffer_total_bytes,
|
33
|
+
'Current total size of ququed buffers.')
|
34
|
+
retry_counts = @registry.gauge(
|
35
|
+
:fluentd_status_retry_count,
|
36
|
+
'Current retry counts.')
|
37
|
+
|
38
|
+
@monitor_info = {
|
39
|
+
'buffer_queue_length' => buffer_queue_length,
|
40
|
+
'buffer_total_queued_size' => buffer_total_queued_size,
|
41
|
+
'retry_count' => retry_counts,
|
42
|
+
}
|
43
|
+
end
|
44
|
+
|
45
|
+
class TimerWatcher < Coolio::TimerWatcher
|
46
|
+
def initialize(interval, repeat, log, &callback)
|
47
|
+
@callback = callback
|
48
|
+
@log = log
|
49
|
+
super(interval, repeat)
|
50
|
+
end
|
51
|
+
|
52
|
+
def on_timer
|
53
|
+
@callback.call
|
54
|
+
rescue
|
55
|
+
@log.error $!.to_s
|
56
|
+
@log.error_backtrace
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def start
|
61
|
+
@loop = Coolio::Loop.new
|
62
|
+
@timer = TimerWatcher.new(@interval, true, log, &method(:update_monitor_info))
|
63
|
+
@loop.attach(@timer)
|
64
|
+
@thread = Thread.new(&method(:run))
|
65
|
+
end
|
66
|
+
|
67
|
+
def shutdown
|
68
|
+
@loop.watchers.each {|w| w.detach }
|
69
|
+
@loop.stop
|
70
|
+
@thread.join
|
71
|
+
end
|
72
|
+
|
73
|
+
def run
|
74
|
+
@loop.run
|
75
|
+
rescue
|
76
|
+
log.error "unexpected error", :error=>$!.to_s
|
77
|
+
log.error_backtrace
|
78
|
+
end
|
79
|
+
|
80
|
+
def update_monitor_info
|
81
|
+
@monitor_agent.plugins_info_all.each do |info|
|
82
|
+
@monitor_info.each do |name, metric|
|
83
|
+
if info[name]
|
84
|
+
metric.set(labels(info), info[name])
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def labels(plugin_info)
|
91
|
+
@base_labels.merge(
|
92
|
+
plugin_id: plugin_info["plugin_id"],
|
93
|
+
plugin_category: plugin_info["plugin_category"],
|
94
|
+
type: plugin_info["type"],
|
95
|
+
)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'fluent/plugin/prometheus'
|
2
|
+
|
3
|
+
module Fluent
|
4
|
+
class PrometheusOutput < Output
|
5
|
+
Plugin.register_output('prometheus', self)
|
6
|
+
include Fluent::Prometheus
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
super
|
10
|
+
@registry = ::Prometheus::Client.registry
|
11
|
+
end
|
12
|
+
|
13
|
+
def configure(conf)
|
14
|
+
super
|
15
|
+
labels = Fluent::Prometheus.parse_labels_elements(conf)
|
16
|
+
@metrics = Fluent::Prometheus.parse_metrics_elements(conf, @registry, labels)
|
17
|
+
end
|
18
|
+
|
19
|
+
def emit(tag, es, chain)
|
20
|
+
instrument(tag, es, @metrics)
|
21
|
+
chain.next
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,195 @@
|
|
1
|
+
require 'prometheus/client'
|
2
|
+
require 'prometheus/client/formats/text'
|
3
|
+
|
4
|
+
module Fluent
|
5
|
+
module Prometheus
|
6
|
+
class AlreadyRegisteredError < StandardError; end
|
7
|
+
|
8
|
+
def self.parse_labels_elements(conf)
|
9
|
+
labels = conf.elements.select { |e| e.name == 'labels' }
|
10
|
+
if labels.size > 1
|
11
|
+
raise ConfigError, "labels section must have at most 1"
|
12
|
+
end
|
13
|
+
|
14
|
+
base_labels = {}
|
15
|
+
unless labels.empty?
|
16
|
+
labels.first.each do |key, value|
|
17
|
+
labels.first.has_key?(key)
|
18
|
+
base_labels[key.to_sym] = value
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
base_labels
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.parse_metrics_elements(conf, registry, labels = {})
|
26
|
+
metrics = []
|
27
|
+
conf.elements.select { |element|
|
28
|
+
element.name == 'metric'
|
29
|
+
}.each { |element|
|
30
|
+
case element['type']
|
31
|
+
when 'summary'
|
32
|
+
metrics << Fluent::Prometheus::Summary.new(element, registry, labels)
|
33
|
+
when 'gauge'
|
34
|
+
metrics << Fluent::Prometheus::Gauge.new(element, registry, labels)
|
35
|
+
when 'counter'
|
36
|
+
metrics << Fluent::Prometheus::Counter.new(element, registry, labels)
|
37
|
+
else
|
38
|
+
raise ConfigError, "type option must be 'counter', 'gauge' or 'summary'"
|
39
|
+
end
|
40
|
+
}
|
41
|
+
metrics
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.placeholder_expnader(log)
|
45
|
+
# Use internal class in order to expand placeholder
|
46
|
+
if defined?(Fluent::Filter) # for v0.12, built-in PlaceholderExpander
|
47
|
+
begin
|
48
|
+
require 'fluent/plugin/filter_record_transformer'
|
49
|
+
return Fluent::RecordTransformerFilter::PlaceholderExpander.new(log)
|
50
|
+
rescue LoadError => e
|
51
|
+
raise ConfigError, "cannot find filter_record_transformer plugin: #{e.message}"
|
52
|
+
end
|
53
|
+
else # for v0.10, use PlaceholderExapander in fluent-plugin-record-reformer plugin
|
54
|
+
begin
|
55
|
+
require 'fluent/plugin/out_record_reformer.rb'
|
56
|
+
return Fluent::RecordReformerOutput::PlaceholderExpander.new(log)
|
57
|
+
rescue LoadError => e
|
58
|
+
raise ConfigError, "cannot find fluent-plugin-record-reformer: #{e.message}"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def configure(conf)
|
64
|
+
super
|
65
|
+
@placeholder_expander = Fluent::Prometheus.placeholder_expnader(log)
|
66
|
+
@hostname = Socket.gethostname
|
67
|
+
end
|
68
|
+
|
69
|
+
def instrument(tag, es, metrics)
|
70
|
+
placeholder_options = {
|
71
|
+
'tag' => tag,
|
72
|
+
'hostname' => @hostname,
|
73
|
+
}
|
74
|
+
|
75
|
+
es.each do |time, record|
|
76
|
+
@placeholder_expander.prepare_placeholders(time, record, placeholder_options)
|
77
|
+
metrics.each do |metric|
|
78
|
+
begin
|
79
|
+
metric.instrument(record, @placeholder_expander)
|
80
|
+
rescue => e
|
81
|
+
log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
|
82
|
+
router.emit_error_event(tag, time, record, e)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
class Metric
|
89
|
+
attr_reader :type
|
90
|
+
attr_reader :name
|
91
|
+
attr_reader :key
|
92
|
+
attr_reader :desc
|
93
|
+
|
94
|
+
def initialize(element, registry, labels)
|
95
|
+
['name', 'desc'].each do |key|
|
96
|
+
if element[key].nil?
|
97
|
+
raise ConfigError, "metric requires '#{key}' option"
|
98
|
+
end
|
99
|
+
end
|
100
|
+
@type = element['type']
|
101
|
+
@name = element['name']
|
102
|
+
@key = element['key']
|
103
|
+
@desc = element['desc']
|
104
|
+
|
105
|
+
@base_labels = Fluent::Prometheus.parse_labels_elements(element)
|
106
|
+
@base_labels = labels.merge(@base_labels)
|
107
|
+
end
|
108
|
+
|
109
|
+
def labels(record, expander)
|
110
|
+
label = {}
|
111
|
+
@base_labels.each do |k, v|
|
112
|
+
label[k] = expander.expand(v)
|
113
|
+
end
|
114
|
+
label
|
115
|
+
end
|
116
|
+
|
117
|
+
def self.get(registry, name, type, docstring)
|
118
|
+
metric = registry.get(name)
|
119
|
+
|
120
|
+
# should have same type, docstring
|
121
|
+
if metric.type != type
|
122
|
+
raise AlreadyRegisteredError, "#{name} has already been registered as #{type} type"
|
123
|
+
end
|
124
|
+
if metric.docstring != docstring
|
125
|
+
raise AlreadyRegisteredError, "#{name} has already been registered with different docstring"
|
126
|
+
end
|
127
|
+
|
128
|
+
metric
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
class Gauge < Metric
|
133
|
+
def initialize(element, registry, labels)
|
134
|
+
super
|
135
|
+
if @key.nil?
|
136
|
+
raise ConfigError, "gauge metric requires 'key' option"
|
137
|
+
end
|
138
|
+
|
139
|
+
begin
|
140
|
+
@gauge = registry.gauge(element['name'].to_sym, element['desc'])
|
141
|
+
rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
|
142
|
+
@gauge = Fluent::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
def instrument(record, expander)
|
147
|
+
if record[@key]
|
148
|
+
@gauge.set(labels(record, expander), record[@key])
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
class Counter < Metric
|
154
|
+
def initialize(element, registry, labels)
|
155
|
+
super
|
156
|
+
begin
|
157
|
+
@counter = registry.counter(element['name'].to_sym, element['desc'])
|
158
|
+
rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
|
159
|
+
@counter = Fluent::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
def instrument(record, expander)
|
164
|
+
# use record value of the key if key is specified, otherwise just increment
|
165
|
+
value = @key ? record[@key] : 1
|
166
|
+
|
167
|
+
# ignore if record value is nil
|
168
|
+
return if value.nil?
|
169
|
+
|
170
|
+
@counter.increment(labels(record, expander), value)
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
class Summary < Metric
|
175
|
+
def initialize(element, registry, labels)
|
176
|
+
super
|
177
|
+
if @key.nil?
|
178
|
+
raise ConfigError, "summary metric requires 'key' option"
|
179
|
+
end
|
180
|
+
|
181
|
+
begin
|
182
|
+
@summary = registry.summary(element['name'].to_sym, element['desc'])
|
183
|
+
rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
|
184
|
+
@summary = Fluent::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
def instrument(record, expander)
|
189
|
+
if record[@key]
|
190
|
+
@summary.add(labels(record, expander), record[@key])
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
@@ -0,0 +1,135 @@
|
|
1
|
+
## Prometheus Input Plugin Configuration
|
2
|
+
|
3
|
+
# input plugin that exports metrics
|
4
|
+
<source>
|
5
|
+
type prometheus
|
6
|
+
</source>
|
7
|
+
|
8
|
+
# input plugin that collects metrics from MonitorAgent
|
9
|
+
<source>
|
10
|
+
type prometheus_monitor
|
11
|
+
<labels>
|
12
|
+
host ${hostname}
|
13
|
+
</labels>
|
14
|
+
</source>
|
15
|
+
|
16
|
+
## Nginx Access Log Configuration
|
17
|
+
|
18
|
+
<source>
|
19
|
+
type tail
|
20
|
+
format nginx
|
21
|
+
tag nginx
|
22
|
+
path /var/log/nginx/access.log
|
23
|
+
pos_file /tmp/fluent_nginx.pos
|
24
|
+
types size:integer
|
25
|
+
</source>
|
26
|
+
|
27
|
+
<filter nginx>
|
28
|
+
type prometheus
|
29
|
+
|
30
|
+
# You can use counter type with specifying a key,
|
31
|
+
# and increments counter by the value
|
32
|
+
<metric>
|
33
|
+
name nginx_size_counter_bytes
|
34
|
+
type counter
|
35
|
+
desc nginx bytes sent
|
36
|
+
key size
|
37
|
+
<labels>
|
38
|
+
host ${hostname}
|
39
|
+
</labels>
|
40
|
+
</metric>
|
41
|
+
|
42
|
+
# You can use counter type without specifying a key
|
43
|
+
# This just increments counter by 1
|
44
|
+
<metric>
|
45
|
+
name nginx_record_counts
|
46
|
+
type counter
|
47
|
+
desc the number of emited records
|
48
|
+
<labels>
|
49
|
+
host ${hostname}
|
50
|
+
</labels>
|
51
|
+
</metric>
|
52
|
+
</filter>
|
53
|
+
|
54
|
+
<match nginx>
|
55
|
+
type copy
|
56
|
+
# for MonitorAgent sample
|
57
|
+
# <store>
|
58
|
+
# @id test_forward
|
59
|
+
# type forward
|
60
|
+
# buffer_type memory
|
61
|
+
# <server>
|
62
|
+
# host 127.0.0.1
|
63
|
+
# port 20000
|
64
|
+
# </server>
|
65
|
+
# </store>
|
66
|
+
<store>
|
67
|
+
type stdout
|
68
|
+
</store>
|
69
|
+
</match>
|
70
|
+
|
71
|
+
## Nginx Proxy Log Configuration
|
72
|
+
|
73
|
+
<source>
|
74
|
+
type tail
|
75
|
+
format ltsv
|
76
|
+
tag nginx_proxy
|
77
|
+
path /var/log/nginx/access_proxy.log
|
78
|
+
pos_file /tmp/fluent_nginx_proxy.pos
|
79
|
+
types size:integer,request_length:integer,bytes_sent:integer,body_bytes_sent:integer,request_time:float,upstream_response_time:float
|
80
|
+
</source>
|
81
|
+
|
82
|
+
<filter nginx_proxy>
|
83
|
+
type prometheus
|
84
|
+
|
85
|
+
# common labels for all metrics
|
86
|
+
<labels>
|
87
|
+
host ${hostname}
|
88
|
+
method ${request_method}
|
89
|
+
status ${status}
|
90
|
+
</labels>
|
91
|
+
|
92
|
+
<metric>
|
93
|
+
name nginx_proxy_request_length_total_bytes
|
94
|
+
type counter
|
95
|
+
desc nginx proxy request length bytes
|
96
|
+
key request_length
|
97
|
+
</metric>
|
98
|
+
<metric>
|
99
|
+
name nginx_proxy_bytes_sent_total_bytes
|
100
|
+
type counter
|
101
|
+
desc nginx proxy bytes sent
|
102
|
+
key bytes_sent
|
103
|
+
</metric>
|
104
|
+
<metric>
|
105
|
+
name nginx_proxy_request_duration_total_milliseconds
|
106
|
+
type counter
|
107
|
+
desc nginx proxy request time
|
108
|
+
key request_time
|
109
|
+
</metric>
|
110
|
+
<metric>
|
111
|
+
name nginx_proxy_upstream_response_duration_total_milliseconds
|
112
|
+
type counter
|
113
|
+
desc nginx proxy upstream response time
|
114
|
+
key upstream_response_time
|
115
|
+
</metric>
|
116
|
+
<metric>
|
117
|
+
name nginx_proxy_request_duration_milliseconds
|
118
|
+
type summary
|
119
|
+
desc nginx proxy request duration summary
|
120
|
+
key request_time
|
121
|
+
</metric>
|
122
|
+
<metric>
|
123
|
+
name nginx_proxy_upstream_duration_milliseconds
|
124
|
+
type summary
|
125
|
+
desc nginx proxy upstream response duration summary
|
126
|
+
key upstream_response_time
|
127
|
+
</metric>
|
128
|
+
</filter>
|
129
|
+
|
130
|
+
<match nginx_proxy>
|
131
|
+
type copy
|
132
|
+
<store>
|
133
|
+
type stdout
|
134
|
+
</store>
|
135
|
+
</match>
|