logstash-filter-grok 3.4.4 → 4.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/lib/logstash/filters/grok.rb +7 -8
- data/lib/logstash/filters/grok/timeout_enforcer.rb +39 -41
- data/logstash-filter-grok.gemspec +3 -2
- data/spec/filters/grok_spec.rb +1 -1
- metadata +21 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d06febb2d4c48dd7c02d2d89ecbe2c379c345b980687d10e80542192f3b232d9
|
4
|
+
data.tar.gz: ddb45461cff71c3fee2c7cd9e84780e0c0c216b207c23cee42ed39d499be25bf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7db250b93474764efb1c769aab63fe9e8850fa9eac4e14ad4b5ec82bdb296d6fe6edc92e4604c7f70cc52199929b2ce31b523cd07f75abb7fd781da64f74b385
|
7
|
+
data.tar.gz: 55051f431618fa87a944984de8b62ec81daaed169e719fad74d84d323b034d776d76bf26d568f5b587cabd16008ff2154c6b245f640287df36cf26ed7b7a0f9c
|
data/CHANGELOG.md
CHANGED
@@ -287,19 +287,18 @@
|
|
287
287
|
@patterns[field] << grok
|
288
288
|
end
|
289
289
|
end # @match.each
|
290
|
+
@match_counter = metric.counter(:matches)
|
291
|
+
@failure_counter = metric.counter(:failures)
|
290
292
|
end # def register
|
291
293
|
|
292
294
|
public
|
293
295
|
def filter(event)
|
294
296
|
matched = false
|
295
|
-
done = false
|
296
297
|
|
297
|
-
@logger.debug? and @logger.debug("Running grok filter", :event => event)
|
298
|
+
@logger.debug? and @logger.debug("Running grok filter", :event => event)
|
298
299
|
|
299
300
|
@patterns.each do |field, groks|
|
300
|
-
|
301
|
-
|
302
|
-
if success
|
301
|
+
if match(groks, field, event)
|
303
302
|
matched = true
|
304
303
|
break if @break_on_match
|
305
304
|
end
|
@@ -307,10 +306,10 @@
|
|
307
306
|
end # @patterns.each
|
308
307
|
|
309
308
|
if matched
|
310
|
-
|
309
|
+
@match_counter.increment(1)
|
311
310
|
filter_matched(event)
|
312
311
|
else
|
313
|
-
|
312
|
+
@failure_counter.increment(1)
|
314
313
|
@tag_on_failure.each {|tag| event.tag(tag)}
|
315
314
|
end
|
316
315
|
|
@@ -345,7 +344,7 @@
|
|
345
344
|
groks.each do |grok|
|
346
345
|
# Convert anything else to string (number, hash, etc)
|
347
346
|
|
348
|
-
matched = @timeout_enforcer.grok_till_timeout(
|
347
|
+
matched = @timeout_enforcer.grok_till_timeout(grok, field, input)
|
349
348
|
if matched
|
350
349
|
grok.capture(matched) {|field, value| handle(field, value, event)}
|
351
350
|
break if @break_on_match
|
@@ -1,5 +1,3 @@
|
|
1
|
-
java_import java.util.concurrent.locks.ReentrantLock
|
2
|
-
|
3
1
|
class LogStash::Filters::Grok::TimeoutEnforcer
|
4
2
|
attr_reader :running
|
5
3
|
|
@@ -10,24 +8,30 @@ class LogStash::Filters::Grok::TimeoutEnforcer
|
|
10
8
|
|
11
9
|
# Stores running matches with their start time, this is used to cancel long running matches
|
12
10
|
# Is a map of Thread => start_time
|
13
|
-
@threads_to_start_time =
|
14
|
-
@
|
11
|
+
@threads_to_start_time = java.util.concurrent.ConcurrentHashMap.new
|
12
|
+
@cancel_mutex = Mutex.new
|
15
13
|
end
|
16
14
|
|
17
|
-
def grok_till_timeout(
|
15
|
+
def grok_till_timeout(grok, field, value)
|
18
16
|
begin
|
19
17
|
thread = java.lang.Thread.currentThread()
|
20
18
|
start_thread_groking(thread)
|
21
|
-
|
19
|
+
grok.execute(value)
|
22
20
|
rescue InterruptedRegexpError => e
|
23
21
|
raise ::LogStash::Filters::Grok::TimeoutException.new(grok, field, value)
|
24
22
|
ensure
|
25
|
-
stop_thread_groking(thread)
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
23
|
+
unless stop_thread_groking(thread)
|
24
|
+
@cancel_mutex.lock
|
25
|
+
begin
|
26
|
+
# Clear any interrupts from any previous invocations that were not caught by Joni
|
27
|
+
# It may appear that this should go in #stop_thread_groking but that would actually
|
28
|
+
# break functionality! If this were moved there we would clear the interrupt
|
29
|
+
# immediately after setting it in #cancel_timed_out, hence this MUST be here
|
30
|
+
java.lang.Thread.interrupted
|
31
|
+
ensure
|
32
|
+
@cancel_mutex.unlock
|
33
|
+
end
|
34
|
+
end
|
31
35
|
end
|
32
36
|
end
|
33
37
|
|
@@ -64,44 +68,38 @@ class LogStash::Filters::Grok::TimeoutEnforcer
|
|
64
68
|
|
65
69
|
def start_thread_groking(thread)
|
66
70
|
# Clear any interrupts from any previous invocations that were not caught by Joni
|
67
|
-
|
68
|
-
|
69
|
-
@threads_to_start_time[thread] = java.lang.System.nanoTime()
|
70
|
-
end
|
71
|
+
java.lang.Thread.interrupted
|
72
|
+
@threads_to_start_time.put(thread, java.lang.System.nanoTime)
|
71
73
|
end
|
72
74
|
|
75
|
+
# Returns falsy in case there was no Grok execution in progress for the thread
|
73
76
|
def stop_thread_groking(thread)
|
74
|
-
|
75
|
-
@threads_to_start_time.delete(thread)
|
76
|
-
end
|
77
|
+
@threads_to_start_time.remove(thread)
|
77
78
|
end
|
78
79
|
|
79
80
|
def cancel_timed_out!
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
81
|
+
now = java.lang.System.nanoTime # save ourselves some nanotime calls
|
82
|
+
@threads_to_start_time.entry_set.each do |entry|
|
83
|
+
start_time = entry.get_value
|
84
|
+
if start_time < now && now - start_time > @timeout_nanos
|
85
|
+
thread = entry.get_key
|
86
|
+
# Ensure that we never attempt to cancel this thread unless a Grok execution is in progress
|
87
|
+
# Theoretically there is a race condition here in case the entry's grok action changed
|
88
|
+
# between evaluating the above condition on the start_time and calling stop_thread_groking
|
89
|
+
# Practically this is impossible, since it would require a whole loop of writing to an
|
90
|
+
# output, pulling new input events and starting a new Grok execution in worker thread
|
91
|
+
# in between the above `if start_time < now && now - start_time > @timeout_nanos` and
|
92
|
+
# the call to `stop_thread_groking`.
|
93
|
+
if stop_thread_groking(thread)
|
94
|
+
@cancel_mutex.lock
|
95
|
+
begin
|
96
|
+
thread.interrupt()
|
97
|
+
ensure
|
98
|
+
@cancel_mutex.unlock
|
99
|
+
end
|
90
100
|
end
|
91
101
|
end
|
92
102
|
end
|
93
103
|
end
|
94
104
|
|
95
|
-
# We use this instead of a Mutex because JRuby mutexes are interruptible
|
96
|
-
# We actually don't want that behavior since we always clear the interrupt in
|
97
|
-
# grok_till_timeout
|
98
|
-
def synchronize
|
99
|
-
# The JRuby Mutex uses lockInterruptibly which is what we DO NOT want
|
100
|
-
@state_lock.lock()
|
101
|
-
yield
|
102
|
-
ensure
|
103
|
-
@state_lock.unlock()
|
104
|
-
end
|
105
|
-
|
106
|
-
|
107
105
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-grok'
|
4
|
-
s.version = '
|
4
|
+
s.version = '4.0.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Parses unstructured event data into fields"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -21,10 +21,11 @@ Gem::Specification.new do |s|
|
|
21
21
|
|
22
22
|
# Gem dependencies
|
23
23
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
24
|
+
s.add_runtime_dependency "logstash-core", ">= 5.6.0"
|
24
25
|
|
25
26
|
s.add_runtime_dependency 'jls-grok', '~> 0.11.3'
|
26
27
|
s.add_runtime_dependency 'stud', '~> 0.0.22'
|
27
28
|
s.add_runtime_dependency 'logstash-patterns-core'
|
28
29
|
|
29
|
-
s.add_development_dependency 'logstash-devutils'
|
30
|
+
s.add_development_dependency 'logstash-devutils', '= 1.3.6'
|
30
31
|
end
|
data/spec/filters/grok_spec.rb
CHANGED
@@ -889,7 +889,7 @@ describe LogStash::Filters::Grok do
|
|
889
889
|
insist { subject.to_json } =~ %r|"@version":"1"|
|
890
890
|
insist { subject.to_json } =~ %r|"username"|i
|
891
891
|
insist { subject.to_json } =~ %r|"testuser"|
|
892
|
-
insist { subject.to_json } =~ %r|"tags":\["ssh_failure"\]
|
892
|
+
insist { subject.to_json } =~ %r|"tags":\["ssh_failure"\]|
|
893
893
|
end
|
894
894
|
end
|
895
895
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-grok
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 4.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-11-
|
11
|
+
date: 2017-11-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -30,6 +30,20 @@ dependencies:
|
|
30
30
|
- - "<="
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2.99'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
35
|
+
requirements:
|
36
|
+
- - ">="
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: 5.6.0
|
39
|
+
name: logstash-core
|
40
|
+
prerelease: false
|
41
|
+
type: :runtime
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 5.6.0
|
33
47
|
- !ruby/object:Gem::Dependency
|
34
48
|
requirement: !ruby/object:Gem::Requirement
|
35
49
|
requirements:
|
@@ -75,17 +89,17 @@ dependencies:
|
|
75
89
|
- !ruby/object:Gem::Dependency
|
76
90
|
requirement: !ruby/object:Gem::Requirement
|
77
91
|
requirements:
|
78
|
-
- -
|
92
|
+
- - '='
|
79
93
|
- !ruby/object:Gem::Version
|
80
|
-
version:
|
94
|
+
version: 1.3.6
|
81
95
|
name: logstash-devutils
|
82
96
|
prerelease: false
|
83
97
|
type: :development
|
84
98
|
version_requirements: !ruby/object:Gem::Requirement
|
85
99
|
requirements:
|
86
|
-
- -
|
100
|
+
- - '='
|
87
101
|
- !ruby/object:Gem::Version
|
88
|
-
version:
|
102
|
+
version: 1.3.6
|
89
103
|
description: This gem is a Logstash plugin required to be installed on top of the
|
90
104
|
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
91
105
|
gem is not a stand-alone program
|
@@ -128,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
128
142
|
version: '0'
|
129
143
|
requirements: []
|
130
144
|
rubyforge_project:
|
131
|
-
rubygems_version: 2.6.
|
145
|
+
rubygems_version: 2.6.13
|
132
146
|
signing_key:
|
133
147
|
specification_version: 4
|
134
148
|
summary: Parses unstructured event data into fields
|