phihos-fluent-plugin-prometheus 2.0.3.pre.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/workflows/linux.yml +34 -0
- data/.gitignore +16 -0
- data/.rspec +2 -0
- data/.travis.yml +14 -0
- data/ChangeLog +43 -0
- data/Gemfile +4 -0
- data/LICENSE +202 -0
- data/README.md +537 -0
- data/Rakefile +7 -0
- data/fluent-plugin-prometheus.gemspec +22 -0
- data/lib/fluent/plugin/filter_prometheus.rb +50 -0
- data/lib/fluent/plugin/in_prometheus/async_wrapper.rb +47 -0
- data/lib/fluent/plugin/in_prometheus.rb +230 -0
- data/lib/fluent/plugin/in_prometheus_monitor.rb +107 -0
- data/lib/fluent/plugin/in_prometheus_output_monitor.rb +234 -0
- data/lib/fluent/plugin/in_prometheus_tail_monitor.rb +98 -0
- data/lib/fluent/plugin/out_prometheus.rb +49 -0
- data/lib/fluent/plugin/prometheus/data_store.rb +103 -0
- data/lib/fluent/plugin/prometheus/placeholder_expander.rb +132 -0
- data/lib/fluent/plugin/prometheus.rb +445 -0
- data/lib/fluent/plugin/prometheus_metrics.rb +77 -0
- data/misc/fluentd_sample.conf +170 -0
- data/misc/nginx_proxy.conf +22 -0
- data/misc/prometheus.yaml +13 -0
- data/misc/prometheus_alerts.yaml +59 -0
- data/spec/fluent/plugin/filter_prometheus_spec.rb +145 -0
- data/spec/fluent/plugin/in_prometheus_monitor_spec.rb +42 -0
- data/spec/fluent/plugin/in_prometheus_spec.rb +225 -0
- data/spec/fluent/plugin/in_prometheus_tail_monitor_spec.rb +42 -0
- data/spec/fluent/plugin/out_prometheus_spec.rb +166 -0
- data/spec/fluent/plugin/prometheus/placeholder_expander_spec.rb +110 -0
- data/spec/fluent/plugin/prometheus_metrics_spec.rb +138 -0
- data/spec/fluent/plugin/shared.rb +248 -0
- data/spec/spec_helper.rb +10 -0
- metadata +176 -0
@@ -0,0 +1,445 @@
|
|
1
|
+
require 'prometheus/client'
|
2
|
+
require 'prometheus/client/formats/text'
|
3
|
+
require 'fluent/plugin/prometheus/placeholder_expander'
|
4
|
+
require 'fluent/plugin/prometheus/data_store'
|
5
|
+
|
6
|
+
module Fluent
|
7
|
+
module Plugin
|
8
|
+
module PrometheusLabelParser
|
9
|
+
def configure(conf)
|
10
|
+
super
|
11
|
+
# Check if running with multiple workers
|
12
|
+
sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
|
13
|
+
owner.system_config
|
14
|
+
elsif self.respond_to?(:system_config)
|
15
|
+
self.system_config
|
16
|
+
else
|
17
|
+
nil
|
18
|
+
end
|
19
|
+
@multi_worker = sysconf && sysconf.workers ? (sysconf.workers > 1) : false
|
20
|
+
end
|
21
|
+
|
22
|
+
def parse_labels_elements(conf)
|
23
|
+
base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
|
24
|
+
|
25
|
+
if @multi_worker
|
26
|
+
base_labels[:worker_id] = fluentd_worker_id.to_s
|
27
|
+
end
|
28
|
+
|
29
|
+
base_labels
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
module Prometheus
|
34
|
+
class AlreadyRegisteredError < StandardError; end
|
35
|
+
|
36
|
+
def self.parse_labels_elements(conf)
|
37
|
+
labels = conf.elements.select { |e| e.name == 'labels' }
|
38
|
+
if labels.size > 1
|
39
|
+
raise ConfigError, "labels section must have at most 1"
|
40
|
+
end
|
41
|
+
|
42
|
+
base_labels = {}
|
43
|
+
unless labels.empty?
|
44
|
+
labels.first.each do |key, value|
|
45
|
+
labels.first.has_key?(key)
|
46
|
+
|
47
|
+
# use RecordAccessor only for $. and $[ syntax
|
48
|
+
# otherwise use the value as is or expand the value by RecordTransformer for ${} syntax
|
49
|
+
if value.start_with?('$.') || value.start_with?('$[')
|
50
|
+
base_labels[key.to_sym] = PluginHelper::RecordAccessor::Accessor.new(value)
|
51
|
+
else
|
52
|
+
base_labels[key.to_sym] = value
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
base_labels
|
58
|
+
end
|
59
|
+
|
60
|
+
def self.parse_metrics_elements(conf, registry, labels = {})
|
61
|
+
metrics = []
|
62
|
+
conf.elements.select { |element|
|
63
|
+
element.name == 'metric'
|
64
|
+
}.each { |element|
|
65
|
+
if element.has_key?('key') && (element['key'].start_with?('$.') || element['key'].start_with?('$['))
|
66
|
+
value = element['key']
|
67
|
+
element['key'] = PluginHelper::RecordAccessor::Accessor.new(value)
|
68
|
+
end
|
69
|
+
case element['type']
|
70
|
+
when 'summary'
|
71
|
+
metrics << Fluent::Plugin::Prometheus::Summary.new(element, registry, labels)
|
72
|
+
when 'gauge'
|
73
|
+
metrics << Fluent::Plugin::Prometheus::Gauge.new(element, registry, labels)
|
74
|
+
when 'counter'
|
75
|
+
metrics << Fluent::Plugin::Prometheus::Counter.new(element, registry, labels)
|
76
|
+
when 'histogram'
|
77
|
+
metrics << Fluent::Plugin::Prometheus::Histogram.new(element, registry, labels)
|
78
|
+
else
|
79
|
+
raise ConfigError, "type option must be 'counter', 'gauge', 'summary' or 'histogram'"
|
80
|
+
end
|
81
|
+
}
|
82
|
+
metrics
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.start_retention_threads(metrics, registry, thread_create, thread_running, log)
|
86
|
+
metrics.select { |metric| metric.has_retention? }.each do |metric|
|
87
|
+
thread_create.call("prometheus_retention_#{metric.name}".to_sym) do
|
88
|
+
while thread_running.call()
|
89
|
+
metric.remove_expired_metrics(registry, log)
|
90
|
+
sleep(metric.retention_check_interval)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def self.start_reset_threads(metrics, registry, thread_create, thread_running, log)
|
97
|
+
metrics.select { |metric| metric.has_reset? }.each do |metric|
|
98
|
+
thread_create.call("prometheus_reset_#{metric.name}".to_sym) do
|
99
|
+
while thread_running.call()
|
100
|
+
sleep(metric.reset_after)
|
101
|
+
metric.reset_values(registry, log)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def self.placeholder_expander(log)
|
108
|
+
Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
|
109
|
+
end
|
110
|
+
|
111
|
+
def stringify_keys(hash_to_stringify)
|
112
|
+
# Adapted from: https://www.jvt.me/posts/2019/09/07/ruby-hash-keys-string-symbol/
|
113
|
+
hash_to_stringify.map do |k,v|
|
114
|
+
value_or_hash = if v.instance_of? Hash
|
115
|
+
stringify_keys(v)
|
116
|
+
else
|
117
|
+
v
|
118
|
+
end
|
119
|
+
[k.to_s, value_or_hash]
|
120
|
+
end.to_h
|
121
|
+
end
|
122
|
+
|
123
|
+
def initialize
|
124
|
+
super
|
125
|
+
::Prometheus::Client.config.data_store = Fluent::Plugin::Prometheus::DataStore.new
|
126
|
+
end
|
127
|
+
|
128
|
+
def configure(conf)
|
129
|
+
super
|
130
|
+
@placeholder_values = {}
|
131
|
+
@placeholder_expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
|
132
|
+
@hostname = Socket.gethostname
|
133
|
+
end
|
134
|
+
|
135
|
+
def instrument_single(tag, time, record, metrics)
|
136
|
+
@placeholder_values[tag] ||= {
|
137
|
+
'tag' => tag,
|
138
|
+
'hostname' => @hostname,
|
139
|
+
'worker_id' => fluentd_worker_id,
|
140
|
+
}
|
141
|
+
|
142
|
+
record = stringify_keys(record)
|
143
|
+
placeholders = record.merge(@placeholder_values[tag])
|
144
|
+
expander = @placeholder_expander_builder.build(placeholders)
|
145
|
+
metrics.each do |metric|
|
146
|
+
begin
|
147
|
+
metric.instrument(record, expander)
|
148
|
+
rescue => e
|
149
|
+
log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
|
150
|
+
router.emit_error_event(tag, time, record, e)
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
def instrument(tag, es, metrics)
|
156
|
+
placeholder_values = {
|
157
|
+
'tag' => tag,
|
158
|
+
'hostname' => @hostname,
|
159
|
+
'worker_id' => fluentd_worker_id,
|
160
|
+
}
|
161
|
+
|
162
|
+
es.each do |time, record|
|
163
|
+
record = stringify_keys(record)
|
164
|
+
placeholders = record.merge(placeholder_values)
|
165
|
+
expander = @placeholder_expander_builder.build(placeholders)
|
166
|
+
metrics.each do |metric|
|
167
|
+
begin
|
168
|
+
metric.instrument(record, expander)
|
169
|
+
rescue => e
|
170
|
+
log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
|
171
|
+
router.emit_error_event(tag, time, record, e)
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
class Metric
|
178
|
+
attr_reader :type
|
179
|
+
attr_reader :name
|
180
|
+
attr_reader :key
|
181
|
+
attr_reader :desc
|
182
|
+
attr_reader :retention
|
183
|
+
attr_reader :retention_check_interval
|
184
|
+
attr_reader :reset_after
|
185
|
+
|
186
|
+
def initialize(element, registry, labels)
|
187
|
+
['name', 'desc'].each do |key|
|
188
|
+
if element[key].nil?
|
189
|
+
raise ConfigError, "metric requires '#{key}' option"
|
190
|
+
end
|
191
|
+
end
|
192
|
+
@type = element['type']
|
193
|
+
@name = element['name']
|
194
|
+
@key = element['key']
|
195
|
+
@desc = element['desc']
|
196
|
+
@retention = element['retention'].to_i
|
197
|
+
@retention_check_interval = element.fetch('retention_check_interval', 60).to_i
|
198
|
+
if has_retention?
|
199
|
+
@last_modified_store = LastModifiedStore.new
|
200
|
+
end
|
201
|
+
@topk = element['topk'].to_i
|
202
|
+
@reset_after = element['reset_after'].to_i
|
203
|
+
|
204
|
+
@base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(element)
|
205
|
+
@base_labels = labels.merge(@base_labels)
|
206
|
+
end
|
207
|
+
|
208
|
+
def labels(record, expander)
|
209
|
+
label = {}
|
210
|
+
@base_labels.each do |k, v|
|
211
|
+
if v.is_a?(String)
|
212
|
+
label[k] = expander.expand(v)
|
213
|
+
else
|
214
|
+
label[k] = v.call(record)
|
215
|
+
end
|
216
|
+
end
|
217
|
+
label
|
218
|
+
end
|
219
|
+
|
220
|
+
def self.get(registry, name, type, docstring)
|
221
|
+
metric = registry.get(name)
|
222
|
+
|
223
|
+
# should have same type, docstring
|
224
|
+
if metric.type != type
|
225
|
+
raise AlreadyRegisteredError, "#{name} has already been registered as #{type} type"
|
226
|
+
end
|
227
|
+
if metric.docstring != docstring
|
228
|
+
raise AlreadyRegisteredError, "#{name} has already been registered with different docstring"
|
229
|
+
end
|
230
|
+
|
231
|
+
metric
|
232
|
+
end
|
233
|
+
|
234
|
+
def set_value?(value)
|
235
|
+
if value
|
236
|
+
return true
|
237
|
+
end
|
238
|
+
false
|
239
|
+
end
|
240
|
+
|
241
|
+
def instrument(record, expander)
|
242
|
+
value = self.value(record)
|
243
|
+
if self.set_value?(value)
|
244
|
+
labels = labels(record, expander)
|
245
|
+
set_value(value, labels)
|
246
|
+
if has_retention?
|
247
|
+
@last_modified_store.set_last_updated(labels)
|
248
|
+
end
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
def has_retention?
|
253
|
+
@retention > 0
|
254
|
+
end
|
255
|
+
|
256
|
+
def has_reset?
|
257
|
+
reset_after > 0
|
258
|
+
end
|
259
|
+
|
260
|
+
def remove_expired_metrics(registry, log)
|
261
|
+
if has_retention?
|
262
|
+
metric = registry.get(@name)
|
263
|
+
|
264
|
+
expiration_time = Time.now - @retention
|
265
|
+
expired_label_sets = @last_modified_store.get_labels_not_modified_since(expiration_time)
|
266
|
+
|
267
|
+
expired_label_sets.each { |expired_label_set|
|
268
|
+
log.debug "Metric #{@name} with labels #{expired_label_set} expired. Removing..."
|
269
|
+
metric.remove(expired_label_set) # this method is supplied by the require at the top of this method
|
270
|
+
@last_modified_store.remove(expired_label_set)
|
271
|
+
}
|
272
|
+
else
|
273
|
+
log.warn('remove_expired_metrics should not be called when retention is not set for this metric!')
|
274
|
+
end
|
275
|
+
end
|
276
|
+
|
277
|
+
def reset_values(registry, log)
|
278
|
+
if has_reset?
|
279
|
+
metric = registry.get(@name)
|
280
|
+
log.debug "Resetting values nof metric #{@name}..."
|
281
|
+
metric.reset_values
|
282
|
+
else
|
283
|
+
log.warn('reset_store should not be called when reset_after is not set for this metric!')
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
287
|
+
class LastModifiedStore
|
288
|
+
def initialize
|
289
|
+
@internal_store = Hash.new
|
290
|
+
@lock = Monitor.new
|
291
|
+
end
|
292
|
+
|
293
|
+
def synchronize
|
294
|
+
@lock.synchronize { yield }
|
295
|
+
end
|
296
|
+
|
297
|
+
def set_last_updated(labels)
|
298
|
+
synchronize do
|
299
|
+
@internal_store[labels] = Time.now
|
300
|
+
end
|
301
|
+
end
|
302
|
+
|
303
|
+
def remove(labels)
|
304
|
+
synchronize do
|
305
|
+
@internal_store.delete(labels)
|
306
|
+
end
|
307
|
+
end
|
308
|
+
|
309
|
+
def get_labels_not_modified_since(time)
|
310
|
+
synchronize do
|
311
|
+
@internal_store.select { |k, v| v < time }.keys
|
312
|
+
end
|
313
|
+
end
|
314
|
+
end
|
315
|
+
end
|
316
|
+
|
317
|
+
class Gauge < Metric
|
318
|
+
def initialize(element, registry, labels)
|
319
|
+
super
|
320
|
+
if @key.nil?
|
321
|
+
raise ConfigError, "gauge metric requires 'key' option"
|
322
|
+
end
|
323
|
+
|
324
|
+
begin
|
325
|
+
@gauge = registry.gauge(
|
326
|
+
element['name'].to_sym,
|
327
|
+
docstring: element['desc'],
|
328
|
+
labels: @base_labels.keys,
|
329
|
+
store_settings: { topk: @topk }
|
330
|
+
)
|
331
|
+
rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
|
332
|
+
@gauge = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
def value(record)
|
337
|
+
if @key.is_a?(String)
|
338
|
+
record[@key]
|
339
|
+
else
|
340
|
+
@key.call(record)
|
341
|
+
end
|
342
|
+
end
|
343
|
+
|
344
|
+
def set_value(value, labels)
|
345
|
+
@gauge.set(value, labels: labels)
|
346
|
+
end
|
347
|
+
end
|
348
|
+
|
349
|
+
class Counter < Metric
|
350
|
+
def initialize(element, registry, labels)
|
351
|
+
super
|
352
|
+
begin
|
353
|
+
@counter = registry.counter(
|
354
|
+
element['name'].to_sym,
|
355
|
+
docstring: element['desc'],
|
356
|
+
labels: @base_labels.keys,
|
357
|
+
store_settings: { topk: @topk }
|
358
|
+
)
|
359
|
+
rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
|
360
|
+
@counter = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
|
361
|
+
end
|
362
|
+
end
|
363
|
+
|
364
|
+
def value(record)
|
365
|
+
if @key.nil?
|
366
|
+
1
|
367
|
+
elsif @key.is_a?(String)
|
368
|
+
record[@key]
|
369
|
+
else
|
370
|
+
@key.call(record)
|
371
|
+
end
|
372
|
+
end
|
373
|
+
|
374
|
+
def set_value?(value)
|
375
|
+
!value.nil?
|
376
|
+
end
|
377
|
+
|
378
|
+
def set_value(value, labels)
|
379
|
+
@counter.increment(by: value, labels: labels)
|
380
|
+
end
|
381
|
+
end
|
382
|
+
|
383
|
+
class Summary < Metric
|
384
|
+
def initialize(element, registry, labels)
|
385
|
+
super
|
386
|
+
if @key.nil?
|
387
|
+
raise ConfigError, "summary metric requires 'key' option"
|
388
|
+
end
|
389
|
+
|
390
|
+
begin
|
391
|
+
@summary = registry.summary(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
|
392
|
+
rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
|
393
|
+
@summary = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
|
394
|
+
end
|
395
|
+
end
|
396
|
+
|
397
|
+
def value(record)
|
398
|
+
if @key.is_a?(String)
|
399
|
+
record[@key]
|
400
|
+
else
|
401
|
+
@key.call(record)
|
402
|
+
end
|
403
|
+
end
|
404
|
+
|
405
|
+
def set_value(value, labels)
|
406
|
+
@summary.observe(value, labels: labels)
|
407
|
+
end
|
408
|
+
end
|
409
|
+
|
410
|
+
class Histogram < Metric
|
411
|
+
def initialize(element, registry, labels)
|
412
|
+
super
|
413
|
+
if @key.nil?
|
414
|
+
raise ConfigError, "histogram metric requires 'key' option"
|
415
|
+
end
|
416
|
+
|
417
|
+
begin
|
418
|
+
if element['buckets']
|
419
|
+
buckets = element['buckets'].split(/,/).map(&:strip).map do |e|
|
420
|
+
e[/\A\d+.\d+\Z/] ? e.to_f : e.to_i
|
421
|
+
end
|
422
|
+
@histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys, buckets: buckets)
|
423
|
+
else
|
424
|
+
@histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
|
425
|
+
end
|
426
|
+
rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
|
427
|
+
@histogram = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :histogram, element['desc'])
|
428
|
+
end
|
429
|
+
end
|
430
|
+
|
431
|
+
def value(record)
|
432
|
+
if @key.is_a?(String)
|
433
|
+
record[@key]
|
434
|
+
else
|
435
|
+
@key.call(record)
|
436
|
+
end
|
437
|
+
end
|
438
|
+
|
439
|
+
def set_value(value, labels)
|
440
|
+
@histogram.observe(value, labels: labels)
|
441
|
+
end
|
442
|
+
end
|
443
|
+
end
|
444
|
+
end
|
445
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
module Fluent::Plugin
|
2
|
+
|
3
|
+
##
|
4
|
+
# PromMetricsAggregator aggregates multiples metrics exposed using Prometheus text-based format
|
5
|
+
# see https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
|
6
|
+
|
7
|
+
|
8
|
+
class PrometheusMetrics
|
9
|
+
def initialize
|
10
|
+
@comments = []
|
11
|
+
@metrics = []
|
12
|
+
end
|
13
|
+
|
14
|
+
def to_string
|
15
|
+
(@comments + @metrics).join("\n")
|
16
|
+
end
|
17
|
+
|
18
|
+
def add_comment(comment)
|
19
|
+
@comments << comment
|
20
|
+
end
|
21
|
+
|
22
|
+
def add_metric_value(value)
|
23
|
+
@metrics << value
|
24
|
+
end
|
25
|
+
|
26
|
+
attr_writer :comments, :metrics
|
27
|
+
end
|
28
|
+
|
29
|
+
class PromMetricsAggregator
|
30
|
+
def initialize
|
31
|
+
@metrics = {}
|
32
|
+
end
|
33
|
+
|
34
|
+
def get_metric_name_from_comment(line)
|
35
|
+
tokens = line.split(' ')
|
36
|
+
if ['HELP', 'TYPE'].include?(tokens[1])
|
37
|
+
tokens[2]
|
38
|
+
else
|
39
|
+
''
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def add_metrics(metrics)
|
44
|
+
current_metric = ''
|
45
|
+
new_metric = false
|
46
|
+
lines = metrics.split("\n")
|
47
|
+
for line in lines
|
48
|
+
if line[0] == '#'
|
49
|
+
# Metric comment (# TYPE, # HELP)
|
50
|
+
parsed_metric = get_metric_name_from_comment(line)
|
51
|
+
if parsed_metric != ''
|
52
|
+
if parsed_metric != current_metric
|
53
|
+
# Starting a new metric comment block
|
54
|
+
new_metric = !@metrics.key?(parsed_metric)
|
55
|
+
if new_metric
|
56
|
+
@metrics[parsed_metric] = PrometheusMetrics.new()
|
57
|
+
end
|
58
|
+
current_metric = parsed_metric
|
59
|
+
end
|
60
|
+
|
61
|
+
if new_metric && parsed_metric == current_metric
|
62
|
+
# New metric, inject comments (# TYPE, # HELP)
|
63
|
+
@metrics[parsed_metric].add_comment(line)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
else
|
67
|
+
# Metric value, simply append line
|
68
|
+
@metrics[current_metric].add_metric_value(line)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def get_metrics
|
74
|
+
@metrics.map{|k,v| v.to_string()}.join("\n") + (@metrics.length ? "\n" : "")
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,170 @@
|
|
1
|
+
## Prometheus Input Plugin Configuration
|
2
|
+
|
3
|
+
# input plugin that exports metrics
|
4
|
+
<source>
|
5
|
+
@type prometheus
|
6
|
+
</source>
|
7
|
+
|
8
|
+
<source>
|
9
|
+
@type monitor_agent
|
10
|
+
</source>
|
11
|
+
|
12
|
+
<source>
|
13
|
+
@type forward
|
14
|
+
</source>
|
15
|
+
|
16
|
+
# input plugin that collects metrics from MonitorAgent
|
17
|
+
<source>
|
18
|
+
@type prometheus_monitor
|
19
|
+
<labels>
|
20
|
+
host ${hostname}
|
21
|
+
</labels>
|
22
|
+
</source>
|
23
|
+
|
24
|
+
# input plugin that collects metrics for output plugin
|
25
|
+
<source>
|
26
|
+
@type prometheus_output_monitor
|
27
|
+
<labels>
|
28
|
+
host ${hostname}
|
29
|
+
</labels>
|
30
|
+
</source>
|
31
|
+
|
32
|
+
# input plugin that collects metrics for in_tail plugin
|
33
|
+
<source>
|
34
|
+
@type prometheus_tail_monitor
|
35
|
+
<labels>
|
36
|
+
host ${hostname}
|
37
|
+
</labels>
|
38
|
+
</source>
|
39
|
+
|
40
|
+
## Nginx Access Log Configuration
|
41
|
+
|
42
|
+
<source>
|
43
|
+
@type tail
|
44
|
+
format nginx
|
45
|
+
tag nginx
|
46
|
+
path /var/log/nginx/access.log
|
47
|
+
pos_file /tmp/fluent_nginx.pos
|
48
|
+
types size:integer
|
49
|
+
</source>
|
50
|
+
|
51
|
+
<filter nginx>
|
52
|
+
@type prometheus
|
53
|
+
|
54
|
+
# You can use counter type with specifying a key,
|
55
|
+
# and increments counter by the value
|
56
|
+
<metric>
|
57
|
+
name nginx_size_counter_bytes
|
58
|
+
type counter
|
59
|
+
desc nginx bytes sent
|
60
|
+
key size
|
61
|
+
<labels>
|
62
|
+
host ${hostname}
|
63
|
+
foo bar
|
64
|
+
</labels>
|
65
|
+
</metric>
|
66
|
+
|
67
|
+
# You can use counter type without specifying a key
|
68
|
+
# This just increments counter by 1
|
69
|
+
<metric>
|
70
|
+
name nginx_record_counts
|
71
|
+
type counter
|
72
|
+
desc the number of emited records
|
73
|
+
<labels>
|
74
|
+
host ${hostname}
|
75
|
+
</labels>
|
76
|
+
</metric>
|
77
|
+
</filter>
|
78
|
+
|
79
|
+
<match nginx>
|
80
|
+
@type copy
|
81
|
+
# for MonitorAgent sample
|
82
|
+
<store>
|
83
|
+
@id test_forward
|
84
|
+
@type forward
|
85
|
+
buffer_type memory
|
86
|
+
flush_interval 1s
|
87
|
+
max_retry_wait 2s
|
88
|
+
<buffer>
|
89
|
+
# max_retry_wait 10s
|
90
|
+
flush_interval 1s
|
91
|
+
# retry_type periodic
|
92
|
+
disable_retry_limit
|
93
|
+
</buffer>
|
94
|
+
# retry_limit 3
|
95
|
+
disable_retry_limit
|
96
|
+
<server>
|
97
|
+
host 127.0.0.1
|
98
|
+
port 20000
|
99
|
+
</server>
|
100
|
+
</store>
|
101
|
+
<store>
|
102
|
+
@type stdout
|
103
|
+
</store>
|
104
|
+
</match>
|
105
|
+
|
106
|
+
## Nginx Proxy Log Configuration
|
107
|
+
|
108
|
+
<source>
|
109
|
+
@type tail
|
110
|
+
format ltsv
|
111
|
+
tag nginx_proxy
|
112
|
+
path /var/log/nginx/access_proxy.log
|
113
|
+
pos_file /tmp/fluent_nginx_proxy.pos
|
114
|
+
types size:integer,request_length:integer,bytes_sent:integer,body_bytes_sent:integer,request_time:float,upstream_response_time:float
|
115
|
+
</source>
|
116
|
+
|
117
|
+
<filter nginx_proxy>
|
118
|
+
@type prometheus
|
119
|
+
|
120
|
+
# common labels for all metrics
|
121
|
+
<labels>
|
122
|
+
host ${hostname}
|
123
|
+
method ${request_method}
|
124
|
+
status ${status}
|
125
|
+
</labels>
|
126
|
+
|
127
|
+
<metric>
|
128
|
+
name nginx_proxy_request_length_total_bytes
|
129
|
+
type counter
|
130
|
+
desc nginx proxy request length bytes
|
131
|
+
key request_length
|
132
|
+
</metric>
|
133
|
+
<metric>
|
134
|
+
name nginx_proxy_bytes_sent_total_bytes
|
135
|
+
type counter
|
136
|
+
desc nginx proxy bytes sent
|
137
|
+
key bytes_sent
|
138
|
+
</metric>
|
139
|
+
<metric>
|
140
|
+
name nginx_proxy_request_duration_total_milliseconds
|
141
|
+
type counter
|
142
|
+
desc nginx proxy request time
|
143
|
+
key request_time
|
144
|
+
</metric>
|
145
|
+
<metric>
|
146
|
+
name nginx_proxy_upstream_response_duration_total_milliseconds
|
147
|
+
type counter
|
148
|
+
desc nginx proxy upstream response time
|
149
|
+
key upstream_response_time
|
150
|
+
</metric>
|
151
|
+
<metric>
|
152
|
+
name nginx_proxy_request_duration_milliseconds
|
153
|
+
type summary
|
154
|
+
desc nginx proxy request duration summary
|
155
|
+
key request_time
|
156
|
+
</metric>
|
157
|
+
<metric>
|
158
|
+
name nginx_proxy_upstream_duration_milliseconds
|
159
|
+
type summary
|
160
|
+
desc nginx proxy upstream response duration summary
|
161
|
+
key upstream_response_time
|
162
|
+
</metric>
|
163
|
+
</filter>
|
164
|
+
|
165
|
+
<match nginx_proxy>
|
166
|
+
@type copy
|
167
|
+
<store>
|
168
|
+
@type stdout
|
169
|
+
</store>
|
170
|
+
</match>
|
@@ -0,0 +1,22 @@
|
|
1
|
+
log_format ltsv 'time:$time_iso8601\t'
|
2
|
+
'remote_addr:$remote_addr\t'
|
3
|
+
'request_method:$request_method\t'
|
4
|
+
'request_length:$request_length\t'
|
5
|
+
'request_uri:$request_uri\t'
|
6
|
+
'uri:$uri\t'
|
7
|
+
'status:$status\t'
|
8
|
+
'bytes_sent:$bytes_sent\t'
|
9
|
+
'body_bytes_sent:$body_bytes_sent\t'
|
10
|
+
'referer:$http_referer\t'
|
11
|
+
'useragent:$http_user_agent\t'
|
12
|
+
'request_time:$request_time\t'
|
13
|
+
'upstream_response_time:$upstream_response_time';
|
14
|
+
|
15
|
+
server {
|
16
|
+
access_log /var/log/nginx/access_proxy.log ltsv;
|
17
|
+
listen 9999;
|
18
|
+
location / {
|
19
|
+
proxy_pass https://www.google.com;
|
20
|
+
}
|
21
|
+
}
|
22
|
+
|