logstash-filter-metrics 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/.gitignore +4 -0
- data/Gemfile +3 -0
- data/Rakefile +6 -0
- data/lib/logstash/filters/metrics.rb +241 -0
- data/logstash-filter-metrics.gemspec +26 -0
- data/rakelib/publish.rake +9 -0
- data/rakelib/vendor.rake +169 -0
- data/spec/filters/metrics_spec.rb +233 -0
- metadata +74 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
ZDcwOTM5NjE4ZGNmOGQyMDQ4NTVkMDA2YjAwMzI0Y2NiMTBkZWViZA==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
ZDIyYmI3NzYyNmQwOGU3NGEzNmNkZmI0MTY5YzFlN2IxNjY3NDUxZQ==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
NGEzNWU1MThmNzA0OGQ0YWM1NDg1ODQ3M2YwNjE0MjcxYzY4Yjk4ZTYxZWZm
|
10
|
+
MzhmMzFlMzA2ZWEwZmViYWQ4YWIzY2FkM2FkYmY3Y2I5YWIyYWExOTAwMWIz
|
11
|
+
NzQ5ZGIwMDJlOWUzYThjZGI1MjFhMmQ0NDlhYWFiY2UxMzkwYTA=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
MjQ3YzQ4NDM0NzA4NTA4MWRiYTY2NWQ4YzAxMjZkMmNlZjJkZjdhMjQ5YWY0
|
14
|
+
ZWU0NTI3MWMxOTcxYmZhMjNmYmQ3Y2ViOGI3ODg3MDc2NjQ5YjdjZjI5Nzk4
|
15
|
+
ODJlYTQyZjU5OWFkYWRkMWViYWJlMjI0YzE2YmQ1OTJlZGZkYWE=
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,241 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "securerandom"
|
3
|
+
require "logstash/filters/base"
|
4
|
+
require "logstash/namespace"
|
5
|
+
|
6
|
+
# The metrics filter is useful for aggregating metrics.
|
7
|
+
#
|
8
|
+
# For example, if you have a field 'response' that is
|
9
|
+
# a http response code, and you want to count each
|
10
|
+
# kind of response, you can do this:
|
11
|
+
#
|
12
|
+
# filter {
|
13
|
+
# metrics {
|
14
|
+
# meter => [ "http.%{response}" ]
|
15
|
+
# add_tag => "metric"
|
16
|
+
# }
|
17
|
+
# }
|
18
|
+
#
|
19
|
+
# Metrics are flushed every 5 seconds by default or according to
|
20
|
+
# 'flush_interval'. Metrics appear as
|
21
|
+
# new events in the event stream and go through any filters
|
22
|
+
# that occur after as well as outputs.
|
23
|
+
#
|
24
|
+
# In general, you will want to add a tag to your metrics and have an output
|
25
|
+
# explicitly look for that tag.
|
26
|
+
#
|
27
|
+
# The event that is flushed will include every 'meter' and 'timer'
|
28
|
+
# metric in the following way:
|
29
|
+
#
|
30
|
+
# #### 'meter' values
|
31
|
+
#
|
32
|
+
# For a `meter => "something"` you will receive the following fields:
|
33
|
+
#
|
34
|
+
# * "thing.count" - the total count of events
|
35
|
+
# * "thing.rate_1m" - the 1-minute rate (sliding)
|
36
|
+
# * "thing.rate_5m" - the 5-minute rate (sliding)
|
37
|
+
# * "thing.rate_15m" - the 15-minute rate (sliding)
|
38
|
+
#
|
39
|
+
# #### 'timer' values
|
40
|
+
#
|
41
|
+
# For a `timer => [ "thing", "%{duration}" ]` you will receive the following fields:
|
42
|
+
#
|
43
|
+
# * "thing.count" - the total count of events
|
44
|
+
# * "thing.rate_1m" - the 1-minute rate of events (sliding)
|
45
|
+
# * "thing.rate_5m" - the 5-minute rate of events (sliding)
|
46
|
+
# * "thing.rate_15m" - the 15-minute rate of events (sliding)
|
47
|
+
# * "thing.min" - the minimum value seen for this metric
|
48
|
+
# * "thing.max" - the maximum value seen for this metric
|
49
|
+
# * "thing.stddev" - the standard deviation for this metric
|
50
|
+
# * "thing.mean" - the mean for this metric
|
51
|
+
# * "thing.pXX" - the XXth percentile for this metric (see `percentiles`)
|
52
|
+
#
|
53
|
+
# #### Example: computing event rate
|
54
|
+
#
|
55
|
+
# For a simple example, let's track how many events per second are running
|
56
|
+
# through logstash:
|
57
|
+
#
|
58
|
+
# input {
|
59
|
+
# generator {
|
60
|
+
# type => "generated"
|
61
|
+
# }
|
62
|
+
# }
|
63
|
+
#
|
64
|
+
# filter {
|
65
|
+
# if [type] == "generated" {
|
66
|
+
# metrics {
|
67
|
+
# meter => "events"
|
68
|
+
# add_tag => "metric"
|
69
|
+
# }
|
70
|
+
# }
|
71
|
+
# }
|
72
|
+
#
|
73
|
+
# output {
|
74
|
+
# # only emit events with the 'metric' tag
|
75
|
+
# if "metric" in [tags] {
|
76
|
+
# stdout {
|
77
|
+
# codec => line {
|
78
|
+
# format => "rate: %{events.rate_1m}"
|
79
|
+
# }
|
80
|
+
# }
|
81
|
+
# }
|
82
|
+
# }
|
83
|
+
#
|
84
|
+
# Running the above:
|
85
|
+
#
|
86
|
+
# % bin/logstash -f example.conf
|
87
|
+
# rate: 23721.983566819246
|
88
|
+
# rate: 24811.395722536377
|
89
|
+
# rate: 25875.892745934525
|
90
|
+
# rate: 26836.42375967113
|
91
|
+
#
|
92
|
+
# We see the output includes our 'events' 1-minute rate.
|
93
|
+
#
|
94
|
+
# In the real world, you would emit this to graphite or another metrics store,
|
95
|
+
# like so:
|
96
|
+
#
|
97
|
+
# output {
|
98
|
+
# graphite {
|
99
|
+
# metrics => [ "events.rate_1m", "%{events.rate_1m}" ]
|
100
|
+
# }
|
101
|
+
# }
|
102
|
+
class LogStash::Filters::Metrics < LogStash::Filters::Base
|
103
|
+
config_name "metrics"
|
104
|
+
milestone 1
|
105
|
+
|
106
|
+
# syntax: `meter => [ "name of metric", "name of metric" ]`
|
107
|
+
config :meter, :validate => :array, :default => []
|
108
|
+
|
109
|
+
# syntax: `timer => [ "name of metric", "%{time_value}" ]`
|
110
|
+
config :timer, :validate => :hash, :default => {}
|
111
|
+
|
112
|
+
# Don't track events that have @timestamp older than some number of seconds.
|
113
|
+
#
|
114
|
+
# This is useful if you want to only include events that are near real-time
|
115
|
+
# in your metrics.
|
116
|
+
#
|
117
|
+
# Example, to only count events that are within 10 seconds of real-time, you
|
118
|
+
# would do this:
|
119
|
+
#
|
120
|
+
# filter {
|
121
|
+
# metrics {
|
122
|
+
# meter => [ "hits" ]
|
123
|
+
# ignore_older_than => 10
|
124
|
+
# }
|
125
|
+
# }
|
126
|
+
config :ignore_older_than, :validate => :number, :default => 0
|
127
|
+
|
128
|
+
# The flush interval, when the metrics event is created. Must be a multiple of 5s.
|
129
|
+
config :flush_interval, :validate => :number, :default => 5
|
130
|
+
|
131
|
+
# The clear interval, when all counter are reset.
|
132
|
+
#
|
133
|
+
# If set to -1, the default value, the metrics will never be cleared.
|
134
|
+
# Otherwise, should be a multiple of 5s.
|
135
|
+
config :clear_interval, :validate => :number, :default => -1
|
136
|
+
|
137
|
+
# The rates that should be measured, in minutes.
|
138
|
+
# Possible values are 1, 5, and 15.
|
139
|
+
config :rates, :validate => :array, :default => [1, 5, 15]
|
140
|
+
|
141
|
+
# The percentiles that should be measured
|
142
|
+
config :percentiles, :validate => :array, :default => [1, 5, 10, 90, 95, 99, 100]
|
143
|
+
|
144
|
+
def register
|
145
|
+
require "metriks"
|
146
|
+
require "socket"
|
147
|
+
require "atomic"
|
148
|
+
require "thread_safe"
|
149
|
+
@last_flush = Atomic.new(0) # how many seconds ago the metrics where flushed.
|
150
|
+
@last_clear = Atomic.new(0) # how many seconds ago the metrics where cleared.
|
151
|
+
@random_key_preffix = SecureRandom.hex
|
152
|
+
unless (@rates - [1, 5, 15]).empty?
|
153
|
+
raise LogStash::ConfigurationError, "Invalid rates configuration. possible rates are 1, 5, 15. Rates: #{rates}."
|
154
|
+
end
|
155
|
+
@metric_meters = ThreadSafe::Cache.new { |h,k| h[k] = Metriks.meter metric_key(k) }
|
156
|
+
@metric_timers = ThreadSafe::Cache.new { |h,k| h[k] = Metriks.timer metric_key(k) }
|
157
|
+
end # def register
|
158
|
+
|
159
|
+
def filter(event)
|
160
|
+
return unless filter?(event)
|
161
|
+
|
162
|
+
# TODO(piavlo): This should probably be moved to base filter class.
|
163
|
+
if @ignore_older_than > 0 && Time.now - event.timestamp.time > @ignore_older_than
|
164
|
+
@logger.debug("Skipping metriks for old event", :event => event)
|
165
|
+
return
|
166
|
+
end
|
167
|
+
|
168
|
+
@meter.each do |m|
|
169
|
+
@metric_meters[event.sprintf(m)].mark
|
170
|
+
end
|
171
|
+
|
172
|
+
@timer.each do |name, value|
|
173
|
+
@metric_timers[event.sprintf(name)].update(event.sprintf(value).to_f)
|
174
|
+
end
|
175
|
+
end # def filter
|
176
|
+
|
177
|
+
def flush
|
178
|
+
# Add 5 seconds to @last_flush and @last_clear counters
|
179
|
+
# since this method is called every 5 seconds.
|
180
|
+
@last_flush.update { |v| v + 5 }
|
181
|
+
@last_clear.update { |v| v + 5 }
|
182
|
+
|
183
|
+
# Do nothing if there's nothing to do ;)
|
184
|
+
return unless should_flush?
|
185
|
+
|
186
|
+
event = LogStash::Event.new
|
187
|
+
event["message"] = Socket.gethostname
|
188
|
+
@metric_meters.each_pair do |name, metric|
|
189
|
+
flush_rates event, name, metric
|
190
|
+
metric.clear if should_clear?
|
191
|
+
end
|
192
|
+
|
193
|
+
@metric_timers.each_pair do |name, metric|
|
194
|
+
flush_rates event, name, metric
|
195
|
+
# These 4 values are not sliding, so they probably are not useful.
|
196
|
+
event["#{name}.min"] = metric.min
|
197
|
+
event["#{name}.max"] = metric.max
|
198
|
+
# timer's stddev currently returns variance, fix it.
|
199
|
+
event["#{name}.stddev"] = metric.stddev ** 0.5
|
200
|
+
event["#{name}.mean"] = metric.mean
|
201
|
+
|
202
|
+
@percentiles.each do |percentile|
|
203
|
+
event["#{name}.p#{percentile}"] = metric.snapshot.value(percentile / 100.0)
|
204
|
+
end
|
205
|
+
metric.clear if should_clear?
|
206
|
+
end
|
207
|
+
|
208
|
+
# Reset counter since metrics were flushed
|
209
|
+
@last_flush.value = 0
|
210
|
+
|
211
|
+
if should_clear?
|
212
|
+
#Reset counter since metrics were cleared
|
213
|
+
@last_clear.value = 0
|
214
|
+
@metric_meters.clear
|
215
|
+
@metric_timers.clear
|
216
|
+
end
|
217
|
+
|
218
|
+
filter_matched(event)
|
219
|
+
return [event]
|
220
|
+
end
|
221
|
+
|
222
|
+
private
|
223
|
+
def flush_rates(event, name, metric)
|
224
|
+
event["#{name}.count"] = metric.count
|
225
|
+
event["#{name}.rate_1m"] = metric.one_minute_rate if @rates.include? 1
|
226
|
+
event["#{name}.rate_5m"] = metric.five_minute_rate if @rates.include? 5
|
227
|
+
event["#{name}.rate_15m"] = metric.fifteen_minute_rate if @rates.include? 15
|
228
|
+
end
|
229
|
+
|
230
|
+
def metric_key(key)
|
231
|
+
"#{@random_key_preffix}_#{key}"
|
232
|
+
end
|
233
|
+
|
234
|
+
def should_flush?
|
235
|
+
@last_flush.value >= @flush_interval && (!@metric_meters.empty? || !@metric_timers.empty?)
|
236
|
+
end
|
237
|
+
|
238
|
+
def should_clear?
|
239
|
+
@clear_interval > 0 && @last_clear.value >= @clear_interval
|
240
|
+
end
|
241
|
+
end # class LogStash::Filters::Metrics
|
@@ -0,0 +1,26 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
|
3
|
+
s.name = 'logstash-filter-metrics'
|
4
|
+
s.version = '0.1.0'
|
5
|
+
s.licenses = ['Apache License (2.0)']
|
6
|
+
s.summary = "The metrics filter is useful for aggregating metrics."
|
7
|
+
s.description = "The metrics filter is useful for aggregating metrics."
|
8
|
+
s.authors = ["Elasticsearch"]
|
9
|
+
s.email = 'richard.pijnenburg@elasticsearch.com'
|
10
|
+
s.homepage = "http://logstash.net/"
|
11
|
+
s.require_paths = ["lib"]
|
12
|
+
|
13
|
+
# Files
|
14
|
+
s.files = `git ls-files`.split($\)
|
15
|
+
|
16
|
+
# Tests
|
17
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
18
|
+
|
19
|
+
# Special flag to let us know this is actually a logstash plugin
|
20
|
+
s.metadata = { "logstash_plugin" => "true", "group" => "filter" }
|
21
|
+
|
22
|
+
# Gem dependencies
|
23
|
+
s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
|
24
|
+
|
25
|
+
end
|
26
|
+
|
@@ -0,0 +1,9 @@
|
|
1
|
+
require "gem_publisher"
|
2
|
+
|
3
|
+
desc "Publish gem to RubyGems.org"
|
4
|
+
task :publish_gem do |t|
|
5
|
+
gem_file = Dir.glob(File.expand_path('../*.gemspec',File.dirname(__FILE__))).first
|
6
|
+
gem = GemPublisher.publish_if_updated(gem_file, :rubygems)
|
7
|
+
puts "Published #{gem}" if gem
|
8
|
+
end
|
9
|
+
|
data/rakelib/vendor.rake
ADDED
@@ -0,0 +1,169 @@
|
|
1
|
+
require "net/http"
|
2
|
+
require "uri"
|
3
|
+
require "digest/sha1"
|
4
|
+
|
5
|
+
def vendor(*args)
|
6
|
+
return File.join("vendor", *args)
|
7
|
+
end
|
8
|
+
|
9
|
+
directory "vendor/" => ["vendor"] do |task, args|
|
10
|
+
mkdir task.name
|
11
|
+
end
|
12
|
+
|
13
|
+
def fetch(url, sha1, output)
|
14
|
+
|
15
|
+
puts "Downloading #{url}"
|
16
|
+
actual_sha1 = download(url, output)
|
17
|
+
|
18
|
+
if actual_sha1 != sha1
|
19
|
+
fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
|
20
|
+
end
|
21
|
+
end # def fetch
|
22
|
+
|
23
|
+
def file_fetch(url, sha1)
|
24
|
+
filename = File.basename( URI(url).path )
|
25
|
+
output = "vendor/#{filename}"
|
26
|
+
task output => [ "vendor/" ] do
|
27
|
+
begin
|
28
|
+
actual_sha1 = file_sha1(output)
|
29
|
+
if actual_sha1 != sha1
|
30
|
+
fetch(url, sha1, output)
|
31
|
+
end
|
32
|
+
rescue Errno::ENOENT
|
33
|
+
fetch(url, sha1, output)
|
34
|
+
end
|
35
|
+
end.invoke
|
36
|
+
|
37
|
+
return output
|
38
|
+
end
|
39
|
+
|
40
|
+
def file_sha1(path)
|
41
|
+
digest = Digest::SHA1.new
|
42
|
+
fd = File.new(path, "r")
|
43
|
+
while true
|
44
|
+
begin
|
45
|
+
digest << fd.sysread(16384)
|
46
|
+
rescue EOFError
|
47
|
+
break
|
48
|
+
end
|
49
|
+
end
|
50
|
+
return digest.hexdigest
|
51
|
+
ensure
|
52
|
+
fd.close if fd
|
53
|
+
end
|
54
|
+
|
55
|
+
def download(url, output)
|
56
|
+
uri = URI(url)
|
57
|
+
digest = Digest::SHA1.new
|
58
|
+
tmp = "#{output}.tmp"
|
59
|
+
Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
|
60
|
+
request = Net::HTTP::Get.new(uri.path)
|
61
|
+
http.request(request) do |response|
|
62
|
+
fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
|
63
|
+
size = (response["content-length"].to_i || -1).to_f
|
64
|
+
count = 0
|
65
|
+
File.open(tmp, "w") do |fd|
|
66
|
+
response.read_body do |chunk|
|
67
|
+
fd.write(chunk)
|
68
|
+
digest << chunk
|
69
|
+
if size > 0 && $stdout.tty?
|
70
|
+
count += chunk.bytesize
|
71
|
+
$stdout.write(sprintf("\r%0.2f%%", count/size * 100))
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
$stdout.write("\r \r") if $stdout.tty?
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
File.rename(tmp, output)
|
80
|
+
|
81
|
+
return digest.hexdigest
|
82
|
+
rescue SocketError => e
|
83
|
+
puts "Failure while downloading #{url}: #{e}"
|
84
|
+
raise
|
85
|
+
ensure
|
86
|
+
File.unlink(tmp) if File.exist?(tmp)
|
87
|
+
end # def download
|
88
|
+
|
89
|
+
def untar(tarball, &block)
|
90
|
+
require "archive/tar/minitar"
|
91
|
+
tgz = Zlib::GzipReader.new(File.open(tarball))
|
92
|
+
# Pull out typesdb
|
93
|
+
tar = Archive::Tar::Minitar::Input.open(tgz)
|
94
|
+
tar.each do |entry|
|
95
|
+
path = block.call(entry)
|
96
|
+
next if path.nil?
|
97
|
+
parent = File.dirname(path)
|
98
|
+
|
99
|
+
mkdir_p parent unless File.directory?(parent)
|
100
|
+
|
101
|
+
# Skip this file if the output file is the same size
|
102
|
+
if entry.directory?
|
103
|
+
mkdir path unless File.directory?(path)
|
104
|
+
else
|
105
|
+
entry_mode = entry.instance_eval { @mode } & 0777
|
106
|
+
if File.exists?(path)
|
107
|
+
stat = File.stat(path)
|
108
|
+
# TODO(sissel): Submit a patch to archive-tar-minitar upstream to
|
109
|
+
# expose headers in the entry.
|
110
|
+
entry_size = entry.instance_eval { @size }
|
111
|
+
# If file sizes are same, skip writing.
|
112
|
+
next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
|
113
|
+
end
|
114
|
+
puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
|
115
|
+
File.open(path, "w") do |fd|
|
116
|
+
# eof? check lets us skip empty files. Necessary because the API provided by
|
117
|
+
# Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
|
118
|
+
# IO object. Something about empty files in this EntryStream causes
|
119
|
+
# IO.copy_stream to throw "can't convert nil into String" on JRuby
|
120
|
+
# TODO(sissel): File a bug about this.
|
121
|
+
while !entry.eof?
|
122
|
+
chunk = entry.read(16384)
|
123
|
+
fd.write(chunk)
|
124
|
+
end
|
125
|
+
#IO.copy_stream(entry, fd)
|
126
|
+
end
|
127
|
+
File.chmod(entry_mode, path)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
tar.close
|
131
|
+
File.unlink(tarball) if File.file?(tarball)
|
132
|
+
end # def untar
|
133
|
+
|
134
|
+
def ungz(file)
|
135
|
+
|
136
|
+
outpath = file.gsub('.gz', '')
|
137
|
+
tgz = Zlib::GzipReader.new(File.open(file))
|
138
|
+
begin
|
139
|
+
File.open(outpath, "w") do |out|
|
140
|
+
IO::copy_stream(tgz, out)
|
141
|
+
end
|
142
|
+
File.unlink(file)
|
143
|
+
rescue
|
144
|
+
File.unlink(outpath) if File.file?(outpath)
|
145
|
+
raise
|
146
|
+
end
|
147
|
+
tgz.close
|
148
|
+
end
|
149
|
+
|
150
|
+
desc "Process any vendor files required for this plugin"
|
151
|
+
task "vendor" do |task, args|
|
152
|
+
|
153
|
+
@files.each do |file|
|
154
|
+
download = file_fetch(file['url'], file['sha1'])
|
155
|
+
if download =~ /.tar.gz/
|
156
|
+
prefix = download.gsub('.tar.gz', '').gsub('vendor/', '')
|
157
|
+
untar(download) do |entry|
|
158
|
+
if !file['files'].nil?
|
159
|
+
next unless file['files'].include?(entry.full_name.gsub(prefix, ''))
|
160
|
+
out = entry.full_name.split("/").last
|
161
|
+
end
|
162
|
+
File.join('vendor', out)
|
163
|
+
end
|
164
|
+
elsif download =~ /.gz/
|
165
|
+
ungz(download)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
end
|
@@ -0,0 +1,233 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "logstash/filters/metrics"
|
3
|
+
|
4
|
+
describe LogStash::Filters::Metrics do
|
5
|
+
|
6
|
+
context "with basic meter config" do
|
7
|
+
context "when no events were received" do
|
8
|
+
it "should not flush" do
|
9
|
+
config = {"meter" => ["http.%{response}"]}
|
10
|
+
filter = LogStash::Filters::Metrics.new config
|
11
|
+
filter.register
|
12
|
+
|
13
|
+
events = filter.flush
|
14
|
+
insist { events }.nil?
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
context "when events are received" do
|
19
|
+
context "on the first flush" do
|
20
|
+
subject {
|
21
|
+
config = {"meter" => ["http.%{response}"]}
|
22
|
+
filter = LogStash::Filters::Metrics.new config
|
23
|
+
filter.register
|
24
|
+
filter.filter LogStash::Event.new({"response" => 200})
|
25
|
+
filter.filter LogStash::Event.new({"response" => 200})
|
26
|
+
filter.filter LogStash::Event.new({"response" => 404})
|
27
|
+
filter.flush
|
28
|
+
}
|
29
|
+
|
30
|
+
it "should flush counts" do
|
31
|
+
insist { subject.length } == 1
|
32
|
+
insist { subject.first["http.200.count"] } == 2
|
33
|
+
insist { subject.first["http.404.count"] } == 1
|
34
|
+
end
|
35
|
+
|
36
|
+
it "should include rates and percentiles" do
|
37
|
+
metrics = ["http.200.rate_1m", "http.200.rate_5m", "http.200.rate_15m",
|
38
|
+
"http.404.rate_1m", "http.404.rate_5m", "http.404.rate_15m"]
|
39
|
+
metrics.each do |metric|
|
40
|
+
insist { subject.first }.include? metric
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
context "on the second flush" do
|
46
|
+
it "should not reset counts" do
|
47
|
+
config = {"meter" => ["http.%{response}"]}
|
48
|
+
filter = LogStash::Filters::Metrics.new config
|
49
|
+
filter.register
|
50
|
+
filter.filter LogStash::Event.new({"response" => 200})
|
51
|
+
filter.filter LogStash::Event.new({"response" => 200})
|
52
|
+
filter.filter LogStash::Event.new({"response" => 404})
|
53
|
+
|
54
|
+
events = filter.flush
|
55
|
+
events = filter.flush
|
56
|
+
insist { events.length } == 1
|
57
|
+
insist { events.first["http.200.count"] } == 2
|
58
|
+
insist { events.first["http.404.count"] } == 1
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
context "when custom rates and percentiles are selected" do
|
64
|
+
context "on the first flush" do
|
65
|
+
subject {
|
66
|
+
config = {
|
67
|
+
"meter" => ["http.%{response}"],
|
68
|
+
"rates" => [1]
|
69
|
+
}
|
70
|
+
filter = LogStash::Filters::Metrics.new config
|
71
|
+
filter.register
|
72
|
+
filter.filter LogStash::Event.new({"response" => 200})
|
73
|
+
filter.filter LogStash::Event.new({"response" => 200})
|
74
|
+
filter.filter LogStash::Event.new({"response" => 404})
|
75
|
+
filter.flush
|
76
|
+
}
|
77
|
+
|
78
|
+
it "should include only the requested rates" do
|
79
|
+
rate_fields = subject.first.to_hash.keys.select {|field| field.start_with?("http.200.rate") }
|
80
|
+
insist { rate_fields.length } == 1
|
81
|
+
insist { rate_fields }.include? "http.200.rate_1m"
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
context "with multiple instances" do
|
88
|
+
it "counts should be independent" do
|
89
|
+
config_tag1 = {"meter" => ["http.%{response}"], "tags" => ["tag1"]}
|
90
|
+
config_tag2 = {"meter" => ["http.%{response}"], "tags" => ["tag2"]}
|
91
|
+
filter_tag1 = LogStash::Filters::Metrics.new config_tag1
|
92
|
+
filter_tag2 = LogStash::Filters::Metrics.new config_tag2
|
93
|
+
event_tag1 = LogStash::Event.new({"response" => 200, "tags" => [ "tag1" ]})
|
94
|
+
event_tag2 = LogStash::Event.new({"response" => 200, "tags" => [ "tag2" ]})
|
95
|
+
event2_tag2 = LogStash::Event.new({"response" => 200, "tags" => [ "tag2" ]})
|
96
|
+
filter_tag1.register
|
97
|
+
filter_tag2.register
|
98
|
+
|
99
|
+
[event_tag1, event_tag2, event2_tag2].each do |event|
|
100
|
+
filter_tag1.filter event
|
101
|
+
filter_tag2.filter event
|
102
|
+
end
|
103
|
+
|
104
|
+
events_tag1 = filter_tag1.flush
|
105
|
+
events_tag2 = filter_tag2.flush
|
106
|
+
|
107
|
+
insist { events_tag1.first["http.200.count"] } == 1
|
108
|
+
insist { events_tag2.first["http.200.count"] } == 2
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
context "with timer config" do
|
113
|
+
context "on the first flush" do
|
114
|
+
subject {
|
115
|
+
config = {"timer" => ["http.request_time", "%{request_time}"]}
|
116
|
+
filter = LogStash::Filters::Metrics.new config
|
117
|
+
filter.register
|
118
|
+
filter.filter LogStash::Event.new({"request_time" => 10})
|
119
|
+
filter.filter LogStash::Event.new({"request_time" => 20})
|
120
|
+
filter.filter LogStash::Event.new({"request_time" => 30})
|
121
|
+
filter.flush
|
122
|
+
}
|
123
|
+
|
124
|
+
it "should flush counts" do
|
125
|
+
insist { subject.length } == 1
|
126
|
+
insist { subject.first["http.request_time.count"] } == 3
|
127
|
+
end
|
128
|
+
|
129
|
+
it "should include rates and percentiles keys" do
|
130
|
+
metrics = ["rate_1m", "rate_5m", "rate_15m", "p1", "p5", "p10", "p90", "p95", "p99"]
|
131
|
+
metrics.each do |metric|
|
132
|
+
insist { subject.first }.include? "http.request_time.#{metric}"
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
it "should include min value" do
|
137
|
+
insist { subject.first['http.request_time.min'] } == 10.0
|
138
|
+
end
|
139
|
+
|
140
|
+
it "should include mean value" do
|
141
|
+
insist { subject.first['http.request_time.mean'] } == 20.0
|
142
|
+
end
|
143
|
+
|
144
|
+
it "should include stddev value" do
|
145
|
+
insist { subject.first['http.request_time.stddev'] } == Math.sqrt(10.0)
|
146
|
+
end
|
147
|
+
|
148
|
+
it "should include max value" do
|
149
|
+
insist { subject.first['http.request_time.max'] } == 30.0
|
150
|
+
end
|
151
|
+
|
152
|
+
it "should include percentile value" do
|
153
|
+
insist { subject.first['http.request_time.p99'] } == 30.0
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
context "when custom rates and percentiles are selected" do
|
159
|
+
context "on the first flush" do
|
160
|
+
subject {
|
161
|
+
config = {
|
162
|
+
"timer" => ["http.request_time", "request_time"],
|
163
|
+
"rates" => [1],
|
164
|
+
"percentiles" => [1, 2]
|
165
|
+
}
|
166
|
+
filter = LogStash::Filters::Metrics.new config
|
167
|
+
filter.register
|
168
|
+
filter.filter LogStash::Event.new({"request_time" => 1})
|
169
|
+
filter.flush
|
170
|
+
}
|
171
|
+
|
172
|
+
it "should flush counts" do
|
173
|
+
insist { subject.length } == 1
|
174
|
+
insist { subject.first["http.request_time.count"] } == 1
|
175
|
+
end
|
176
|
+
|
177
|
+
it "should include only the requested rates" do
|
178
|
+
rate_fields = subject.first.to_hash.keys.select {|field| field.start_with?("http.request_time.rate") }
|
179
|
+
insist { rate_fields.length } == 1
|
180
|
+
insist { rate_fields }.include? "http.request_time.rate_1m"
|
181
|
+
end
|
182
|
+
|
183
|
+
it "should include only the requested percentiles" do
|
184
|
+
percentile_fields = subject.first.to_hash.keys.select {|field| field.start_with?("http.request_time.p") }
|
185
|
+
insist { percentile_fields.length } == 2
|
186
|
+
insist { percentile_fields }.include? "http.request_time.p1"
|
187
|
+
insist { percentile_fields }.include? "http.request_time.p2"
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
|
193
|
+
context "when a custom flush_interval is set" do
|
194
|
+
it "should flush only when required" do
|
195
|
+
config = {"meter" => ["http.%{response}"], "flush_interval" => 15}
|
196
|
+
filter = LogStash::Filters::Metrics.new config
|
197
|
+
filter.register
|
198
|
+
filter.filter LogStash::Event.new({"response" => 200})
|
199
|
+
|
200
|
+
insist { filter.flush }.nil? # 5s
|
201
|
+
insist { filter.flush }.nil? # 10s
|
202
|
+
insist { filter.flush.length } == 1 # 15s
|
203
|
+
insist { filter.flush }.nil? # 20s
|
204
|
+
insist { filter.flush }.nil? # 25s
|
205
|
+
insist { filter.flush.length } == 1 # 30s
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
context "when a custom clear_interval is set" do
|
210
|
+
it "should clear the metrics after interval has passed" do
|
211
|
+
config = {"meter" => ["http.%{response}"], "clear_interval" => 15}
|
212
|
+
filter = LogStash::Filters::Metrics.new config
|
213
|
+
filter.register
|
214
|
+
filter.filter LogStash::Event.new({"response" => 200})
|
215
|
+
|
216
|
+
insist { filter.flush.first["http.200.count"] } == 1 # 5s
|
217
|
+
insist { filter.flush.first["http.200.count"] } == 1 # 10s
|
218
|
+
insist { filter.flush.first["http.200.count"] } == 1 # 15s
|
219
|
+
insist { filter.flush }.nil? # 20s
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
context "when invalid rates are set" do
|
224
|
+
subject {
|
225
|
+
config = {"meter" => ["http.%{response}"], "rates" => [90]}
|
226
|
+
filter = LogStash::Filters::Metrics.new config
|
227
|
+
}
|
228
|
+
|
229
|
+
it "should raise an error" do
|
230
|
+
insist {subject.register }.raises(LogStash::ConfigurationError)
|
231
|
+
end
|
232
|
+
end
|
233
|
+
end
|
metadata
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logstash-filter-metrics
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Elasticsearch
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-11-02 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: logstash
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ! '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.4.0
|
20
|
+
- - <
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 2.0.0
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 1.4.0
|
30
|
+
- - <
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 2.0.0
|
33
|
+
description: The metrics filter is useful for aggregating metrics.
|
34
|
+
email: richard.pijnenburg@elasticsearch.com
|
35
|
+
executables: []
|
36
|
+
extensions: []
|
37
|
+
extra_rdoc_files: []
|
38
|
+
files:
|
39
|
+
- .gitignore
|
40
|
+
- Gemfile
|
41
|
+
- Rakefile
|
42
|
+
- lib/logstash/filters/metrics.rb
|
43
|
+
- logstash-filter-metrics.gemspec
|
44
|
+
- rakelib/publish.rake
|
45
|
+
- rakelib/vendor.rake
|
46
|
+
- spec/filters/metrics_spec.rb
|
47
|
+
homepage: http://logstash.net/
|
48
|
+
licenses:
|
49
|
+
- Apache License (2.0)
|
50
|
+
metadata:
|
51
|
+
logstash_plugin: 'true'
|
52
|
+
group: filter
|
53
|
+
post_install_message:
|
54
|
+
rdoc_options: []
|
55
|
+
require_paths:
|
56
|
+
- lib
|
57
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - ! '>='
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: '0'
|
67
|
+
requirements: []
|
68
|
+
rubyforge_project:
|
69
|
+
rubygems_version: 2.4.1
|
70
|
+
signing_key:
|
71
|
+
specification_version: 4
|
72
|
+
summary: The metrics filter is useful for aggregating metrics.
|
73
|
+
test_files:
|
74
|
+
- spec/filters/metrics_spec.rb
|