logstash-filter-grok 2.0.2 → 2.0.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0db663241d42d604907726c6780eb3c61a091e55
4
- data.tar.gz: 9b44dbb0243e36e4f205248153b9d78a84755c85
3
+ metadata.gz: 0c29b365e4050c830edadd3f1787b202851ab318
4
+ data.tar.gz: ab4929767f5244347793b383bb33a4ae8f86741d
5
5
  SHA512:
6
- metadata.gz: 4b8430c616c8d2bf1890f645f94d791e9a776bb16a53dc6f36ce208b63b9e7611d0d4ecb6f0c75d2e893f71a140625531b6d079ff33e99ac36c624eeb5e38611
7
- data.tar.gz: e7356c65e3a21192cf0a6524b554c17274fecba4ab18143f1dcacc72d7bfe0e1be323dfd63924b71ecdea98792174a99ef2e34fbe9e22ed0fb692aa6d5d07169
6
+ metadata.gz: 41c8bb7ec629ab346b44013059e05ddc13bffaa2ca9eb25f1c8d1c8d730fb6272d6ae8d22e2651a99bd3d524d9fe870a51132558a0cda01c3a0cf3af05e4ae4b
7
+ data.tar.gz: 2825968293377f7ec3f0e6a327e6863e0bbd8c78f24100ab46f630f14f5e361006c2ff8e262efb3293544a769617dcc4644ad448a3eb972d40fdbd917973369d
data/CHANGELOG.md CHANGED
@@ -1,5 +1,7 @@
1
+ ## 2.0.3
2
+ - fix fieldref assignment to avoid assumption on mutable object
1
3
  ## 2.0.0
2
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
4
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
3
5
  instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
4
6
  - Dependency on logstash-core update to 2.0
5
7
 
data/README.md CHANGED
@@ -1,5 +1,8 @@
1
1
  # Logstash Plugin
2
2
 
3
+ [![Build
4
+ Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-grok-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-grok-unit/)
5
+
3
6
  This is a plugin for [Logstash](https://github.com/elastic/logstash).
4
7
 
5
8
  It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
@@ -4,7 +4,7 @@
4
4
  require "logstash/environment"
5
5
  require "logstash/patterns/core"
6
6
  require "set"
7
-
7
+
8
8
  # Parse arbitrary text and structure it.
9
9
  #
10
10
  # Grok is currently the best way in logstash to parse crappy unstructured log
@@ -72,7 +72,7 @@
72
72
  # }
73
73
  #
74
74
  # After the grok filter, the event will have a few extra fields in it:
75
- #
75
+ #
76
76
  # * `client: 55.3.244.1`
77
77
  # * `method: GET`
78
78
  # * `request: /index.html`
@@ -138,13 +138,13 @@
138
138
  # `SYSLOGBASE` pattern which itself is defined by other patterns.
139
139
  class LogStash::Filters::Grok < LogStash::Filters::Base
140
140
  config_name "grok"
141
-
141
+
142
142
  # Specify a pattern to parse with. This will match the `message` field.
143
143
  #
144
144
  # If you want to match other fields than message, use the `match` setting.
145
145
  # Multiple patterns is fine.
146
146
  config :pattern, :validate => :array, :deprecated => "You should use this instead: match => { \"message\" => \"your pattern here\" }"
147
-
147
+
148
148
  # A hash of matches of field => value
149
149
  #
150
150
  # For example:
@@ -159,9 +159,9 @@
159
159
  # grok { match => { "message" => [ "Duration: %{NUMBER:duration}", "Speed: %{NUMBER:speed}" ] } }
160
160
  # }
161
161
 
162
- #
162
+ #
163
163
  config :match, :validate => :hash, :default => {}
164
-
164
+
165
165
  #
166
166
  # Logstash ships by default with a bunch of patterns, so you don't
167
167
  # necessarily need to define this yourself unless you are adding additional
@@ -169,7 +169,7 @@
169
169
  # Note that Grok will read all files in the directory and assume its a pattern
170
170
  # file (including any tilde backup files)
171
171
  # [source,ruby]
172
- # patterns_dir => ["/opt/logstash/patterns", "/opt/logstash/extra_patterns"]
172
+ # patterns_dir => ["/opt/logstash/patterns", "/opt/logstash/extra_patterns"]
173
173
  #
174
174
  # Pattern files are plain text with format:
175
175
  # [source,ruby]
@@ -179,26 +179,26 @@
179
179
  # [source,ruby]
180
180
  # NUMBER \d+
181
181
  config :patterns_dir, :validate => :array, :default => []
182
-
182
+
183
183
  # Break on first match. The first successful match by grok will result in the
184
184
  # filter being finished. If you want grok to try all patterns (maybe you are
185
185
  # parsing different things), then set this to false.
186
186
  config :break_on_match, :validate => :boolean, :default => true
187
-
187
+
188
188
  # If `true`, only store named captures from grok.
189
189
  config :named_captures_only, :validate => :boolean, :default => true
190
-
190
+
191
191
  # If `true`, keep empty captures as event fields.
192
192
  config :keep_empty_captures, :validate => :boolean, :default => false
193
-
193
+
194
194
  # If `true`, make single-value fields simply that value, not an array
195
195
  # containing that one value.
196
196
  config :singles, :validate => :boolean, :default => true, :deprecated => "This behavior is the default now, you don't need to set it."
197
-
197
+
198
198
  # Append values to the `tags` field when there has been no
199
199
  # successful match
200
200
  config :tag_on_failure, :validate => :array, :default => ["_grokparsefailure"]
201
-
201
+
202
202
  # The fields to overwrite.
203
203
  #
204
204
  # This allows you to overwrite a value in a field that already exists.
@@ -216,14 +216,14 @@
216
216
  # In this case, a line like `May 29 16:37:11 sadness logger: hello world`
217
217
  # will be parsed and `hello world` will overwrite the original message.
218
218
  config :overwrite, :validate => :array, :default => []
219
-
219
+
220
220
  # Register default pattern paths
221
221
  @@patterns_path ||= Set.new
222
222
  @@patterns_path += [
223
223
  LogStash::Patterns::Core.path,
224
224
  LogStash::Environment.pattern_path("*")
225
225
  ]
226
-
226
+
227
227
  public
228
228
  def initialize(params)
229
229
  super(params)
@@ -232,13 +232,13 @@
232
232
  # a cache of capture name handler methods.
233
233
  @handlers = {}
234
234
  end
235
-
235
+
236
236
  public
237
237
  def register
238
238
  require "grok-pure" # rubygem 'jls-grok'
239
-
239
+
240
240
  @patternfiles = []
241
-
241
+
242
242
  # Have @@patterns_path show first. Last-in pattern definitions win; this
243
243
  # will let folks redefine built-in patterns at runtime.
244
244
  @patterns_dir = @@patterns_path.to_a + @patterns_dir
@@ -247,20 +247,20 @@
247
247
  if File.directory?(path)
248
248
  path = File.join(path, "*")
249
249
  end
250
-
250
+
251
251
  Dir.glob(path).each do |file|
252
252
  @logger.info? and @logger.info("Grok loading patterns from file", :path => file)
253
253
  @patternfiles << file
254
254
  end
255
255
  end
256
-
256
+
257
257
  @patterns = Hash.new { |h,k| h[k] = [] }
258
-
258
+
259
259
  @logger.info? and @logger.info("Match data", :match => @match)
260
-
260
+
261
261
  @match.each do |field, patterns|
262
262
  patterns = [patterns] if patterns.is_a?(String)
263
-
263
+
264
264
  @logger.info? and @logger.info("Grok compile", :field => field, :patterns => patterns)
265
265
  patterns.each do |pattern|
266
266
  @logger.debug? and @logger.debug("regexp: #{@type}/#{field}", :pattern => pattern)
@@ -272,14 +272,14 @@
272
272
  end
273
273
  end # @match.each
274
274
  end # def register
275
-
275
+
276
276
  public
277
277
  def filter(event)
278
-
279
-
278
+
279
+
280
280
  matched = false
281
281
  done = false
282
-
282
+
283
283
  @logger.debug? and @logger.debug("Running grok filter", :event => event);
284
284
  @patterns.each do |field, groks|
285
285
  if match(groks, field, event)
@@ -288,21 +288,16 @@
288
288
  end
289
289
  #break if done
290
290
  end # @patterns.each
291
-
291
+
292
292
  if matched
293
293
  filter_matched(event)
294
294
  else
295
- # Tag this event if we can't parse it. We can use this later to
296
- # reparse+reindex logs if we improve the patterns given.
297
- @tag_on_failure.each do |tag|
298
- event["tags"] ||= []
299
- event["tags"] << tag unless event["tags"].include?(tag)
300
- end
295
+ @tag_on_failure.each{|tag| event.tag(tag)}
301
296
  end
302
-
297
+
303
298
  @logger.debug? and @logger.debug("Event now: ", :event => event)
304
299
  end # def filter
305
-
300
+
306
301
  private
307
302
  def match(groks, field, event)
308
303
  input = event[field]
@@ -318,7 +313,7 @@
318
313
  rescue StandardError => e
319
314
  @logger.warn("Grok regexp threw exception", :exception => e.message)
320
315
  end
321
-
316
+
322
317
  private
323
318
  def match_against_groks(groks, input, event)
324
319
  matched = false
@@ -332,11 +327,11 @@
332
327
  end
333
328
  return matched
334
329
  end
335
-
330
+
336
331
  private
337
332
  def handle(field, value, event)
338
333
  return if (value.nil? || (value.is_a?(String) && value.empty?)) unless @keep_empty_captures
339
-
334
+
340
335
  if @overwrite.include?(field)
341
336
  event[field] = value
342
337
  else
@@ -344,14 +339,19 @@
344
339
  if v.nil?
345
340
  event[field] = value
346
341
  elsif v.is_a?(Array)
347
- event[field] << value
342
+ # do not replace the code below with:
343
+ # event[field] << value
344
+ # this assumes implementation specific feature of returning a mutable object
345
+ # from a field ref which should not be assumed and will change in the future.
346
+ v << value
347
+ event[field] = v
348
348
  elsif v.is_a?(String)
349
349
  # Promote to array since we aren't overwriting.
350
350
  event[field] = [v, value]
351
351
  end
352
352
  end
353
353
  end
354
-
354
+
355
355
  private
356
356
  def add_patterns_from_files(paths, grok)
357
357
  paths.each do |path|
@@ -361,5 +361,5 @@
361
361
  grok.add_patterns_from_file(path)
362
362
  end
363
363
  end # def add_patterns_from_files
364
-
364
+
365
365
  end # class LogStash::Filters::Grok
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-filter-grok'
4
- s.version = '2.0.2'
4
+ s.version = '2.0.3'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Parse arbitrary text and structure it."
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
metadata CHANGED
@@ -1,17 +1,18 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-grok
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.2
4
+ version: 2.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-10-14 00:00:00.000000000 Z
11
+ date: 2015-12-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- requirement: !ruby/object:Gem::Requirement
14
+ name: logstash-core
15
+ version_requirements: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - '>='
17
18
  - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
19
20
  - - <
20
21
  - !ruby/object:Gem::Version
21
22
  version: 3.0.0
22
- name: logstash-core
23
- prerelease: false
24
- type: :runtime
25
- version_requirements: !ruby/object:Gem::Requirement
23
+ requirement: !ruby/object:Gem::Requirement
26
24
  requirements:
27
25
  - - '>='
28
26
  - !ruby/object:Gem::Version
@@ -30,48 +28,50 @@ dependencies:
30
28
  - - <
31
29
  - !ruby/object:Gem::Version
32
30
  version: 3.0.0
31
+ prerelease: false
32
+ type: :runtime
33
33
  - !ruby/object:Gem::Dependency
34
+ name: jls-grok
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ~>
38
+ - !ruby/object:Gem::Version
39
+ version: 0.11.1
34
40
  requirement: !ruby/object:Gem::Requirement
35
41
  requirements:
36
42
  - - ~>
37
43
  - !ruby/object:Gem::Version
38
44
  version: 0.11.1
39
- name: jls-grok
40
45
  prerelease: false
41
46
  type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: logstash-patterns-core
42
49
  version_requirements: !ruby/object:Gem::Requirement
43
50
  requirements:
44
- - - ~>
51
+ - - '>='
45
52
  - !ruby/object:Gem::Version
46
- version: 0.11.1
47
- - !ruby/object:Gem::Dependency
53
+ version: '0'
48
54
  requirement: !ruby/object:Gem::Requirement
49
55
  requirements:
50
56
  - - '>='
51
57
  - !ruby/object:Gem::Version
52
58
  version: '0'
53
- name: logstash-patterns-core
54
59
  prerelease: false
55
60
  type: :runtime
61
+ - !ruby/object:Gem::Dependency
62
+ name: logstash-devutils
56
63
  version_requirements: !ruby/object:Gem::Requirement
57
64
  requirements:
58
65
  - - '>='
59
66
  - !ruby/object:Gem::Version
60
67
  version: '0'
61
- - !ruby/object:Gem::Dependency
62
68
  requirement: !ruby/object:Gem::Requirement
63
69
  requirements:
64
70
  - - '>='
65
71
  - !ruby/object:Gem::Version
66
72
  version: '0'
67
- name: logstash-devutils
68
73
  prerelease: false
69
74
  type: :development
70
- version_requirements: !ruby/object:Gem::Requirement
71
- requirements:
72
- - - '>='
73
- - !ruby/object:Gem::Version
74
- version: '0'
75
75
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
76
76
  email: info@elastic.co
77
77
  executables: []