logstash-filter-grok 2.0.2 → 2.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -1
- data/README.md +3 -0
- data/lib/logstash/filters/grok.rb +42 -42
- data/logstash-filter-grok.gemspec +1 -1
- metadata +19 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0c29b365e4050c830edadd3f1787b202851ab318
|
4
|
+
data.tar.gz: ab4929767f5244347793b383bb33a4ae8f86741d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 41c8bb7ec629ab346b44013059e05ddc13bffaa2ca9eb25f1c8d1c8d730fb6272d6ae8d22e2651a99bd3d524d9fe870a51132558a0cda01c3a0cf3af05e4ae4b
|
7
|
+
data.tar.gz: 2825968293377f7ec3f0e6a327e6863e0bbd8c78f24100ab46f630f14f5e361006c2ff8e262efb3293544a769617dcc4644ad448a3eb972d40fdbd917973369d
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
|
+
## 2.0.3
|
2
|
+
- fix fieldref assignment to avoid assumption on mutable object
|
1
3
|
## 2.0.0
|
2
|
-
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
4
|
+
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
3
5
|
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
4
6
|
- Dependency on logstash-core update to 2.0
|
5
7
|
|
data/README.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
+
[![Build
|
4
|
+
Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-grok-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-grok-unit/)
|
5
|
+
|
3
6
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
4
7
|
|
5
8
|
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
@@ -4,7 +4,7 @@
|
|
4
4
|
require "logstash/environment"
|
5
5
|
require "logstash/patterns/core"
|
6
6
|
require "set"
|
7
|
-
|
7
|
+
|
8
8
|
# Parse arbitrary text and structure it.
|
9
9
|
#
|
10
10
|
# Grok is currently the best way in logstash to parse crappy unstructured log
|
@@ -72,7 +72,7 @@
|
|
72
72
|
# }
|
73
73
|
#
|
74
74
|
# After the grok filter, the event will have a few extra fields in it:
|
75
|
-
#
|
75
|
+
#
|
76
76
|
# * `client: 55.3.244.1`
|
77
77
|
# * `method: GET`
|
78
78
|
# * `request: /index.html`
|
@@ -138,13 +138,13 @@
|
|
138
138
|
# `SYSLOGBASE` pattern which itself is defined by other patterns.
|
139
139
|
class LogStash::Filters::Grok < LogStash::Filters::Base
|
140
140
|
config_name "grok"
|
141
|
-
|
141
|
+
|
142
142
|
# Specify a pattern to parse with. This will match the `message` field.
|
143
143
|
#
|
144
144
|
# If you want to match other fields than message, use the `match` setting.
|
145
145
|
# Multiple patterns is fine.
|
146
146
|
config :pattern, :validate => :array, :deprecated => "You should use this instead: match => { \"message\" => \"your pattern here\" }"
|
147
|
-
|
147
|
+
|
148
148
|
# A hash of matches of field => value
|
149
149
|
#
|
150
150
|
# For example:
|
@@ -159,9 +159,9 @@
|
|
159
159
|
# grok { match => { "message" => [ "Duration: %{NUMBER:duration}", "Speed: %{NUMBER:speed}" ] } }
|
160
160
|
# }
|
161
161
|
|
162
|
-
#
|
162
|
+
#
|
163
163
|
config :match, :validate => :hash, :default => {}
|
164
|
-
|
164
|
+
|
165
165
|
#
|
166
166
|
# Logstash ships by default with a bunch of patterns, so you don't
|
167
167
|
# necessarily need to define this yourself unless you are adding additional
|
@@ -169,7 +169,7 @@
|
|
169
169
|
# Note that Grok will read all files in the directory and assume its a pattern
|
170
170
|
# file (including any tilde backup files)
|
171
171
|
# [source,ruby]
|
172
|
-
# patterns_dir => ["/opt/logstash/patterns", "/opt/logstash/extra_patterns"]
|
172
|
+
# patterns_dir => ["/opt/logstash/patterns", "/opt/logstash/extra_patterns"]
|
173
173
|
#
|
174
174
|
# Pattern files are plain text with format:
|
175
175
|
# [source,ruby]
|
@@ -179,26 +179,26 @@
|
|
179
179
|
# [source,ruby]
|
180
180
|
# NUMBER \d+
|
181
181
|
config :patterns_dir, :validate => :array, :default => []
|
182
|
-
|
182
|
+
|
183
183
|
# Break on first match. The first successful match by grok will result in the
|
184
184
|
# filter being finished. If you want grok to try all patterns (maybe you are
|
185
185
|
# parsing different things), then set this to false.
|
186
186
|
config :break_on_match, :validate => :boolean, :default => true
|
187
|
-
|
187
|
+
|
188
188
|
# If `true`, only store named captures from grok.
|
189
189
|
config :named_captures_only, :validate => :boolean, :default => true
|
190
|
-
|
190
|
+
|
191
191
|
# If `true`, keep empty captures as event fields.
|
192
192
|
config :keep_empty_captures, :validate => :boolean, :default => false
|
193
|
-
|
193
|
+
|
194
194
|
# If `true`, make single-value fields simply that value, not an array
|
195
195
|
# containing that one value.
|
196
196
|
config :singles, :validate => :boolean, :default => true, :deprecated => "This behavior is the default now, you don't need to set it."
|
197
|
-
|
197
|
+
|
198
198
|
# Append values to the `tags` field when there has been no
|
199
199
|
# successful match
|
200
200
|
config :tag_on_failure, :validate => :array, :default => ["_grokparsefailure"]
|
201
|
-
|
201
|
+
|
202
202
|
# The fields to overwrite.
|
203
203
|
#
|
204
204
|
# This allows you to overwrite a value in a field that already exists.
|
@@ -216,14 +216,14 @@
|
|
216
216
|
# In this case, a line like `May 29 16:37:11 sadness logger: hello world`
|
217
217
|
# will be parsed and `hello world` will overwrite the original message.
|
218
218
|
config :overwrite, :validate => :array, :default => []
|
219
|
-
|
219
|
+
|
220
220
|
# Register default pattern paths
|
221
221
|
@@patterns_path ||= Set.new
|
222
222
|
@@patterns_path += [
|
223
223
|
LogStash::Patterns::Core.path,
|
224
224
|
LogStash::Environment.pattern_path("*")
|
225
225
|
]
|
226
|
-
|
226
|
+
|
227
227
|
public
|
228
228
|
def initialize(params)
|
229
229
|
super(params)
|
@@ -232,13 +232,13 @@
|
|
232
232
|
# a cache of capture name handler methods.
|
233
233
|
@handlers = {}
|
234
234
|
end
|
235
|
-
|
235
|
+
|
236
236
|
public
|
237
237
|
def register
|
238
238
|
require "grok-pure" # rubygem 'jls-grok'
|
239
|
-
|
239
|
+
|
240
240
|
@patternfiles = []
|
241
|
-
|
241
|
+
|
242
242
|
# Have @@patterns_path show first. Last-in pattern definitions win; this
|
243
243
|
# will let folks redefine built-in patterns at runtime.
|
244
244
|
@patterns_dir = @@patterns_path.to_a + @patterns_dir
|
@@ -247,20 +247,20 @@
|
|
247
247
|
if File.directory?(path)
|
248
248
|
path = File.join(path, "*")
|
249
249
|
end
|
250
|
-
|
250
|
+
|
251
251
|
Dir.glob(path).each do |file|
|
252
252
|
@logger.info? and @logger.info("Grok loading patterns from file", :path => file)
|
253
253
|
@patternfiles << file
|
254
254
|
end
|
255
255
|
end
|
256
|
-
|
256
|
+
|
257
257
|
@patterns = Hash.new { |h,k| h[k] = [] }
|
258
|
-
|
258
|
+
|
259
259
|
@logger.info? and @logger.info("Match data", :match => @match)
|
260
|
-
|
260
|
+
|
261
261
|
@match.each do |field, patterns|
|
262
262
|
patterns = [patterns] if patterns.is_a?(String)
|
263
|
-
|
263
|
+
|
264
264
|
@logger.info? and @logger.info("Grok compile", :field => field, :patterns => patterns)
|
265
265
|
patterns.each do |pattern|
|
266
266
|
@logger.debug? and @logger.debug("regexp: #{@type}/#{field}", :pattern => pattern)
|
@@ -272,14 +272,14 @@
|
|
272
272
|
end
|
273
273
|
end # @match.each
|
274
274
|
end # def register
|
275
|
-
|
275
|
+
|
276
276
|
public
|
277
277
|
def filter(event)
|
278
|
-
|
279
|
-
|
278
|
+
|
279
|
+
|
280
280
|
matched = false
|
281
281
|
done = false
|
282
|
-
|
282
|
+
|
283
283
|
@logger.debug? and @logger.debug("Running grok filter", :event => event);
|
284
284
|
@patterns.each do |field, groks|
|
285
285
|
if match(groks, field, event)
|
@@ -288,21 +288,16 @@
|
|
288
288
|
end
|
289
289
|
#break if done
|
290
290
|
end # @patterns.each
|
291
|
-
|
291
|
+
|
292
292
|
if matched
|
293
293
|
filter_matched(event)
|
294
294
|
else
|
295
|
-
|
296
|
-
# reparse+reindex logs if we improve the patterns given.
|
297
|
-
@tag_on_failure.each do |tag|
|
298
|
-
event["tags"] ||= []
|
299
|
-
event["tags"] << tag unless event["tags"].include?(tag)
|
300
|
-
end
|
295
|
+
@tag_on_failure.each{|tag| event.tag(tag)}
|
301
296
|
end
|
302
|
-
|
297
|
+
|
303
298
|
@logger.debug? and @logger.debug("Event now: ", :event => event)
|
304
299
|
end # def filter
|
305
|
-
|
300
|
+
|
306
301
|
private
|
307
302
|
def match(groks, field, event)
|
308
303
|
input = event[field]
|
@@ -318,7 +313,7 @@
|
|
318
313
|
rescue StandardError => e
|
319
314
|
@logger.warn("Grok regexp threw exception", :exception => e.message)
|
320
315
|
end
|
321
|
-
|
316
|
+
|
322
317
|
private
|
323
318
|
def match_against_groks(groks, input, event)
|
324
319
|
matched = false
|
@@ -332,11 +327,11 @@
|
|
332
327
|
end
|
333
328
|
return matched
|
334
329
|
end
|
335
|
-
|
330
|
+
|
336
331
|
private
|
337
332
|
def handle(field, value, event)
|
338
333
|
return if (value.nil? || (value.is_a?(String) && value.empty?)) unless @keep_empty_captures
|
339
|
-
|
334
|
+
|
340
335
|
if @overwrite.include?(field)
|
341
336
|
event[field] = value
|
342
337
|
else
|
@@ -344,14 +339,19 @@
|
|
344
339
|
if v.nil?
|
345
340
|
event[field] = value
|
346
341
|
elsif v.is_a?(Array)
|
347
|
-
|
342
|
+
# do not replace the code below with:
|
343
|
+
# event[field] << value
|
344
|
+
# this assumes implementation specific feature of returning a mutable object
|
345
|
+
# from a field ref which should not be assumed and will change in the future.
|
346
|
+
v << value
|
347
|
+
event[field] = v
|
348
348
|
elsif v.is_a?(String)
|
349
349
|
# Promote to array since we aren't overwriting.
|
350
350
|
event[field] = [v, value]
|
351
351
|
end
|
352
352
|
end
|
353
353
|
end
|
354
|
-
|
354
|
+
|
355
355
|
private
|
356
356
|
def add_patterns_from_files(paths, grok)
|
357
357
|
paths.each do |path|
|
@@ -361,5 +361,5 @@
|
|
361
361
|
grok.add_patterns_from_file(path)
|
362
362
|
end
|
363
363
|
end # def add_patterns_from_files
|
364
|
-
|
364
|
+
|
365
365
|
end # class LogStash::Filters::Grok
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-grok'
|
4
|
-
s.version = '2.0.
|
4
|
+
s.version = '2.0.3'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Parse arbitrary text and structure it."
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
metadata
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-grok
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-12-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash-core
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
@@ -19,10 +20,7 @@ dependencies:
|
|
19
20
|
- - <
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: 3.0.0
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
25
|
- - '>='
|
28
26
|
- !ruby/object:Gem::Version
|
@@ -30,48 +28,50 @@ dependencies:
|
|
30
28
|
- - <
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: 3.0.0
|
31
|
+
prerelease: false
|
32
|
+
type: :runtime
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: jls-grok
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - ~>
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: 0.11.1
|
34
40
|
requirement: !ruby/object:Gem::Requirement
|
35
41
|
requirements:
|
36
42
|
- - ~>
|
37
43
|
- !ruby/object:Gem::Version
|
38
44
|
version: 0.11.1
|
39
|
-
name: jls-grok
|
40
45
|
prerelease: false
|
41
46
|
type: :runtime
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: logstash-patterns-core
|
42
49
|
version_requirements: !ruby/object:Gem::Requirement
|
43
50
|
requirements:
|
44
|
-
- -
|
51
|
+
- - '>='
|
45
52
|
- !ruby/object:Gem::Version
|
46
|
-
version: 0
|
47
|
-
- !ruby/object:Gem::Dependency
|
53
|
+
version: '0'
|
48
54
|
requirement: !ruby/object:Gem::Requirement
|
49
55
|
requirements:
|
50
56
|
- - '>='
|
51
57
|
- !ruby/object:Gem::Version
|
52
58
|
version: '0'
|
53
|
-
name: logstash-patterns-core
|
54
59
|
prerelease: false
|
55
60
|
type: :runtime
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: logstash-devutils
|
56
63
|
version_requirements: !ruby/object:Gem::Requirement
|
57
64
|
requirements:
|
58
65
|
- - '>='
|
59
66
|
- !ruby/object:Gem::Version
|
60
67
|
version: '0'
|
61
|
-
- !ruby/object:Gem::Dependency
|
62
68
|
requirement: !ruby/object:Gem::Requirement
|
63
69
|
requirements:
|
64
70
|
- - '>='
|
65
71
|
- !ruby/object:Gem::Version
|
66
72
|
version: '0'
|
67
|
-
name: logstash-devutils
|
68
73
|
prerelease: false
|
69
74
|
type: :development
|
70
|
-
version_requirements: !ruby/object:Gem::Requirement
|
71
|
-
requirements:
|
72
|
-
- - '>='
|
73
|
-
- !ruby/object:Gem::Version
|
74
|
-
version: '0'
|
75
75
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
76
76
|
email: info@elastic.co
|
77
77
|
executables: []
|