logstash-filter-date 0.1.6 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 914517782eea0d83b0472895700d694161e28216
4
- data.tar.gz: e3095af8f12f909189936456897a37e504cec253
3
+ metadata.gz: 6a7783bc9a63927c80f641cbf1a6881424970d7f
4
+ data.tar.gz: d4225b591a3ec9ad1ee87fd10990a7baf8e6c79e
5
5
  SHA512:
6
- metadata.gz: e060eb3ae0e61704dcf39e90c8a562bff24bcd31ee5a94ede455874f136486ac7d3c1404f7bf62eed2b0f4762ef3aeb092f263e5879a9ca2de47419a8417bba6
7
- data.tar.gz: f156593255ddb4df75666df538a8ce0c48e4f023b44bc8c3ab70ff49072c1ebde99faf4faf9d86de8f41898d44765eb4be23ad76087d75ba5ee0efe4ccaa2d44
6
+ metadata.gz: a9a6f7fb0fce3faa06e9fcd541823d680ad0471abf009b99f66778d78d9b1b28cd07f47d6b2e7396fe870604f2ed28a3114818b68d673ca61c192c444113f554
7
+ data.tar.gz: 7a340f155599ede6dd2c50743fa98bd13afb6c6e5023afb891bbb61f8e2e4ee7156ff045bcc72ef76c3f1a458edd30f74a20d8fa0ee34c7bfc4a6c73fa35801d
data/CHANGELOG.md ADDED
File without changes
data/NOTICE.TXT ADDED
@@ -0,0 +1,5 @@
1
+ Elasticsearch
2
+ Copyright 2012-2015 Elasticsearch
3
+
4
+ This product includes software developed by The Apache Software
5
+ Foundation (http://www.apache.org/).
data/README.md CHANGED
@@ -13,7 +13,7 @@ Logstash provides infrastructure to automatically generate documentation for thi
13
13
 
14
14
  ## Need Help?
15
15
 
16
- Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
16
+ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
17
17
 
18
18
  ## Developing
19
19
 
@@ -39,11 +39,12 @@ class LogStash::Filters::Date < LogStash::Filters::Base
39
39
 
40
40
  # Specify a locale to be used for date parsing using either IETF-BCP47 or POSIX language tag.
41
41
  # Simple examples are `en`,`en-US` for BCP47 or `en_US` for POSIX.
42
- # If not specified, the platform default will be used.
43
42
  #
44
43
  # The locale is mostly necessary to be set for parsing month names (pattern with `MMM`) and
45
44
  # weekday names (pattern with `EEE`).
46
45
  #
46
+ # If not specified, the platform default will be used but for non-english platform default
47
+ # an english parser will also be used as a fallback mechanism.
47
48
  config :locale, :validate => :string
48
49
 
49
50
  # The date formats allowed are anything allowed by Joda-Time (java time
@@ -88,6 +89,10 @@ class LogStash::Filters::Date < LogStash::Filters::Base
88
89
  # default to updating the `@timestamp` field of the event.
89
90
  config :target, :validate => :string, :default => "@timestamp"
90
91
 
92
+ # Append values to the `tags` field when there has been no
93
+ # successful match
94
+ config :tag_on_failure, :validate => :array, :default => ["_dateparsefailure"]
95
+
91
96
  # LOGSTASH-34
92
97
  DATEPATTERNS = %w{ y d H m s S }
93
98
 
@@ -159,16 +164,30 @@ class LogStash::Filters::Date < LogStash::Filters::Base
159
164
  return (date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000)
160
165
  end
161
166
  else
162
- joda_parser = org.joda.time.format.DateTimeFormat.forPattern(format).withDefaultYear(Time.new.year)
163
- if @timezone
164
- joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
165
- else
166
- joda_parser = joda_parser.withOffsetParsed
167
- end
168
- if (locale != nil)
169
- joda_parser = joda_parser.withLocale(locale)
167
+ begin
168
+ joda_parser = org.joda.time.format.DateTimeFormat.forPattern(format).withDefaultYear(Time.new.year)
169
+ if @timezone
170
+ joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
171
+ else
172
+ joda_parser = joda_parser.withOffsetParsed
173
+ end
174
+ if locale
175
+ joda_parser = joda_parser.withLocale(locale)
176
+ end
177
+ parsers << lambda { |date| joda_parser.parseMillis(date) }
178
+
179
+ #Include a fallback parser to english when default locale is non-english
180
+ if !locale &&
181
+ "en" != java.util.Locale.getDefault().getLanguage() &&
182
+ (format.include?("MMM") || format.include?("E"))
183
+ en_joda_parser = joda_parser.withLocale(java.util.Locale.forLanguageTag('en-US'))
184
+ parsers << lambda { |date| en_joda_parser.parseMillis(date) }
185
+ end
186
+ rescue JavaException => e
187
+ raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",
188
+ :plugin => "filter", :type => "date",
189
+ :error => "#{e.message} for pattern '#{format}'")
170
190
  end
171
- parsers << lambda { |date| joda_parser.parseMillis(date) }
172
191
  end
173
192
 
174
193
  @logger.debug("Adding type with date config", :type => @type,
@@ -221,11 +240,16 @@ class LogStash::Filters::Date < LogStash::Filters::Base
221
240
  filter_matched(event)
222
241
  rescue StandardError, JavaException => e
223
242
  @logger.warn("Failed parsing date from field", :field => field,
224
- :value => value, :exception => e)
225
- # Raising here will bubble all the way up and cause an exit.
226
- # TODO(sissel): Maybe we shouldn't raise?
227
- # TODO(sissel): What do we do on a failure? Tag it like grok does?
228
- #raise e
243
+ :value => value, :exception => e.message,
244
+ :config_parsers => fieldparsers.collect {|x| x[:format]}.join(','),
245
+ :config_locale => @locale ? @locale : "default="+java.util.Locale.getDefault().toString()
246
+ )
247
+ # Tag this event if we can't parse it. We can use this later to
248
+ # reparse+reindex logs if we improve the patterns given.
249
+ @tag_on_failure.each do |tag|
250
+ event["tags"] ||= []
251
+ event["tags"] << tag unless event["tags"].include?(tag)
252
+ end
229
253
  end # begin
230
254
  end # fieldvalue.each
231
255
  end # @parsers.each
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-filter-date'
4
- s.version = '0.1.6'
4
+ s.version = '1.0.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event."
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -313,7 +313,23 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
313
313
 
314
314
  sample("thedate" => "2013/Apr/21") do
315
315
  insist { subject["@timestamp"] } != "2013-04-21T00:00:00.000Z"
316
- insist { subject["tags"] } == nil
316
+ reject { subject["tags"] }.include? "tagged"
317
+ end
318
+ end
319
+
320
+ describe "failing to parse should apply tag_on_failure" do
321
+ config <<-CONFIG
322
+ filter {
323
+ date {
324
+ match => [ "thedate", "yyyy/MM/dd" ]
325
+ tag_on_failure => ["date_failed"]
326
+ }
327
+ }
328
+ CONFIG
329
+
330
+ sample("thedate" => "2013/Apr/21") do
331
+ insist { subject["@timestamp"] } != "2013-04-21T00:00:00.000Z"
332
+ insist { subject["tags"] }.include? "date_failed"
317
333
  end
318
334
  end
319
335
 
@@ -419,4 +435,24 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
419
435
  insist { subject["@timestamp"].time } == Time.iso8601("2013-03-25T20:33:56.000Z")
420
436
  end
421
437
  end
438
+
439
+ describe "Support fallback to english for non-english default locale" do
440
+ default_locale = java.util.Locale.getDefault()
441
+ #Override default locale with non-english
442
+ java.util.Locale.setDefault(java.util.Locale.forLanguageTag('fr-FR'))
443
+ config <<-CONFIG
444
+ filter {
445
+ date {
446
+ match => [ "message", "dd MMMM yyyy" ]
447
+ timezone => "UTC"
448
+ }
449
+ }
450
+ CONFIG
451
+
452
+ sample "01 September 2014" do
453
+ insist { subject["@timestamp"].time } == Time.iso8601("2014-09-01T00:00:00.000Z").utc
454
+ end
455
+ #Restore default locale
456
+ java.util.Locale.setDefault(default_locale)
457
+ end
422
458
  end
metadata CHANGED
@@ -1,17 +1,18 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-date
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.6
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-20 00:00:00.000000000 Z
11
+ date: 2015-06-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- requirement: !ruby/object:Gem::Requirement
14
+ name: logstash-core
15
+ version_requirements: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - '>='
17
18
  - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
19
20
  - - <
20
21
  - !ruby/object:Gem::Version
21
22
  version: 2.0.0
22
- name: logstash-core
23
- prerelease: false
24
- type: :runtime
25
- version_requirements: !ruby/object:Gem::Requirement
23
+ requirement: !ruby/object:Gem::Requirement
26
24
  requirements:
27
25
  - - '>='
28
26
  - !ruby/object:Gem::Version
@@ -30,62 +28,64 @@ dependencies:
30
28
  - - <
31
29
  - !ruby/object:Gem::Version
32
30
  version: 2.0.0
31
+ prerelease: false
32
+ type: :runtime
33
33
  - !ruby/object:Gem::Dependency
34
+ name: logstash-input-generator
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
34
40
  requirement: !ruby/object:Gem::Requirement
35
41
  requirements:
36
42
  - - '>='
37
43
  - !ruby/object:Gem::Version
38
44
  version: '0'
39
- name: logstash-input-generator
40
45
  prerelease: false
41
46
  type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: logstash-codec-json
42
49
  version_requirements: !ruby/object:Gem::Requirement
43
50
  requirements:
44
51
  - - '>='
45
52
  - !ruby/object:Gem::Version
46
53
  version: '0'
47
- - !ruby/object:Gem::Dependency
48
54
  requirement: !ruby/object:Gem::Requirement
49
55
  requirements:
50
56
  - - '>='
51
57
  - !ruby/object:Gem::Version
52
58
  version: '0'
53
- name: logstash-codec-json
54
59
  prerelease: false
55
60
  type: :runtime
61
+ - !ruby/object:Gem::Dependency
62
+ name: logstash-output-null
56
63
  version_requirements: !ruby/object:Gem::Requirement
57
64
  requirements:
58
65
  - - '>='
59
66
  - !ruby/object:Gem::Version
60
67
  version: '0'
61
- - !ruby/object:Gem::Dependency
62
68
  requirement: !ruby/object:Gem::Requirement
63
69
  requirements:
64
70
  - - '>='
65
71
  - !ruby/object:Gem::Version
66
72
  version: '0'
67
- name: logstash-output-null
68
73
  prerelease: false
69
74
  type: :runtime
75
+ - !ruby/object:Gem::Dependency
76
+ name: logstash-devutils
70
77
  version_requirements: !ruby/object:Gem::Requirement
71
78
  requirements:
72
79
  - - '>='
73
80
  - !ruby/object:Gem::Version
74
81
  version: '0'
75
- - !ruby/object:Gem::Dependency
76
82
  requirement: !ruby/object:Gem::Requirement
77
83
  requirements:
78
84
  - - '>='
79
85
  - !ruby/object:Gem::Version
80
86
  version: '0'
81
- name: logstash-devutils
82
87
  prerelease: false
83
88
  type: :development
84
- version_requirements: !ruby/object:Gem::Requirement
85
- requirements:
86
- - - '>='
87
- - !ruby/object:Gem::Version
88
- version: '0'
89
89
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
90
90
  email: info@elastic.co
91
91
  executables: []
@@ -93,9 +93,11 @@ extensions: []
93
93
  extra_rdoc_files: []
94
94
  files:
95
95
  - .gitignore
96
+ - CHANGELOG.md
96
97
  - CONTRIBUTORS
97
98
  - Gemfile
98
99
  - LICENSE
100
+ - NOTICE.TXT
99
101
  - README.md
100
102
  - Rakefile
101
103
  - lib/logstash/filters/date.rb
@@ -123,7 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
123
125
  version: '0'
124
126
  requirements: []
125
127
  rubyforge_project:
126
- rubygems_version: 2.1.9
128
+ rubygems_version: 2.2.2
127
129
  signing_key:
128
130
  specification_version: 4
129
131
  summary: The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event.