logstash-filter-date 3.0.3 → 3.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ec35ad1fc8d6556f2ce438feb486420db0bbfef6
4
- data.tar.gz: 1605459b572bd999b62ba62884a1d83173438159
3
+ metadata.gz: 7e6adc825ee041eca0be67876519b71a2fa763e1
4
+ data.tar.gz: e484d5652cd00ab7abef9190023609758f929723
5
5
  SHA512:
6
- metadata.gz: 0cc06c361d4cd801439431c745c5e6d90e0d931e249202d4c3b7bafc726ccc4b01d0f9499e062fad75fc5416cd7597765bf33ef50864821164fbb268b73dfbc4
7
- data.tar.gz: 57a4e05b5d87d8a2dff043a12d7238eb2b677700eb14fb75856444780943888cdf3bedf8273b96a0f845de7e6517d8939a0a8a90e0255f0a1c00ebaf6de5696f
6
+ metadata.gz: f40d4bbc2ae12dfe4f5c174b1ed5e068c3c37be75799cec4bcf5fea00d501bfdbb57bdbbb21d65bf66f176adb10762c8b2295763c019dd1c42aa97e085986fdc
7
+ data.tar.gz: 6f216bddf999095e5a197c25402471286dba5291b070f3b0795219c9b599b737913b29e4b94840e79bcb8f5402fcc27edd9ec8e70b4fa3f0908a9efe7d313c46
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 3.1.1
2
+ - Fix problem in packaging of 3.1.0
3
+
4
+ ## 3.1.0
5
+ - Improved performance: 2.8x faster for common case (first pattern matches), 14x faster for events where multiple patterns are attempted. (#74)
6
+
1
7
  ## 3.0.3
2
8
  - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
3
9
 
@@ -0,0 +1,4 @@
1
+ # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
2
+
3
+ require 'jar_dependencies'
4
+ require_jar('org.logstash.filters', 'logstash-filter-date', '3.1.0')
@@ -2,6 +2,7 @@
2
2
  require "logstash/filters/base"
3
3
  require "logstash/namespace"
4
4
  require "logstash/timestamp"
5
+ require "logstash-filter-date_jars"
5
6
 
6
7
  # The date filter is used for parsing dates from fields, and then using that
7
8
  # date or timestamp as the logstash timestamp for the event.
@@ -21,15 +22,6 @@ require "logstash/timestamp"
21
22
  # set in the event. For example, with file input, the timestamp is set to the
22
23
  # time of each read.
23
24
  class LogStash::Filters::Date < LogStash::Filters::Base
24
- if RUBY_ENGINE == "jruby"
25
- JavaException = java.lang.Exception
26
- UTC = org.joda.time.DateTimeZone.forID("UTC")
27
- java_import org.joda.time.LocalDateTime
28
- class LocalDateTime
29
- java_alias :to_datetime_with_tz, :toDateTime, [Java::org.joda.time.DateTimeZone]
30
- end
31
- end
32
-
33
25
  config_name "date"
34
26
 
35
27
  # Specify a time zone canonical ID to be used for date parsing.
@@ -158,24 +150,17 @@ class LogStash::Filters::Date < LogStash::Filters::Base
158
150
  # successful match
159
151
  config :tag_on_failure, :validate => :array, :default => ["_dateparsefailure"]
160
152
 
161
- # LOGSTASH-34
162
- DATEPATTERNS = %w{ y d H m s S }
153
+ def register
154
+ # nothing
155
+ end
163
156
 
164
157
  def initialize(config = {})
165
158
  super
166
-
167
- @parsers = Hash.new { |h,k| h[k] = [] }
168
- end # def initialize
169
-
170
- def register
171
- require "java"
172
159
  if @match.length < 2
173
160
  raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",
174
161
  :plugin => "filter", :type => "date",
175
162
  :error => "The match setting should contains first a field name and at least one date format, current value is #{@match}")
176
163
  end
177
-
178
- locale = nil
179
164
  if @locale
180
165
  if @locale.include? '_'
181
166
  @logger.warn("Date filter now use BCP47 format for locale, replacing underscore with dash")
@@ -184,186 +169,29 @@ class LogStash::Filters::Date < LogStash::Filters::Base
184
169
  locale = java.util.Locale.forLanguageTag(@locale)
185
170
  end
186
171
 
187
- @sprintf_timezone = @timezone && !@timezone.index("%{").nil?
188
- setupMatcher(@config["match"].shift, locale, @config["match"] )
189
- end
172
+ source = @match.first
190
173
 
191
- def parseWithJodaParser(joda_parser, date, format_has_year, format_has_timezone)
192
- return joda_parser.parseMillis(date) if format_has_year
193
- now = Time.now
194
- now_month = now.month
195
- if (format_has_timezone)
196
- result = joda_parser.parseDateTime(date)
197
- else
198
- # Parse date in UTC, Timezone correction later
199
- result = joda_parser.withZone(UTC).parseLocalDateTime(date)
174
+ @datefilter = org.logstash.filters.DateFilter.new(source, @target, @tag_on_failure) do |event|
175
+ filter_matched(event)
200
176
  end
201
177
 
202
- event_month = result.getMonthOfYear
178
+ @match[1..-1].map do |format|
179
+ @datefilter.accept_filter_config(format, @locale, @timezone)
203
180
 
204
- if (event_month == now_month)
205
- result = result.with_year(now.year)
206
- elsif (event_month == 12 && now_month == 1)
207
- result = result.with_year(now.year-1)
208
- elsif (event_month == 1 && now_month == 12)
209
- result = result.with_year(now.year+1)
210
- else
211
- result = result.with_year(now.year)
181
+ # Offer a fallback parser such that if the default system Locale is non-english and that no locale is set,
182
+ # we should try to parse english if the first local parsing fails.:w
183
+ if !@locale && "en" != java.util.Locale.getDefault().getLanguage() && (format.include?("MMM") || format.include?("E"))
184
+ @datefilter.accept_filter_config(format, "en-US", @timezone)
185
+ end
212
186
  end
213
187
 
214
- if (format_has_timezone)
215
- return result.get_millis
216
- else
217
- #Timezone correction
218
- return result.to_datetime_with_tz(joda_parser.getZone()).get_millis
219
- end
220
- end
221
-
222
- def setupMatcher(field, locale, value)
223
- metric.gauge(:formats, value.length)
224
- value.each do |format|
225
- parsers = []
226
- case format
227
- when "ISO8601"
228
- iso_parser = org.joda.time.format.ISODateTimeFormat.dateTimeParser
229
- if @timezone && !@sprintf_timezone
230
- iso_parser = iso_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
231
- else
232
- iso_parser = iso_parser.withOffsetParsed
233
- end
234
- parsers << lambda { |date| iso_parser.parseMillis(date) }
235
- #Fall back solution of almost ISO8601 date-time
236
- almostISOparsers = [
237
- org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSSZ").getParser(),
238
- org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS").getParser(),
239
- org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss,SSSZ").getParser(),
240
- org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss,SSS").getParser()
241
- ].to_java(org.joda.time.format.DateTimeParser)
242
- joda_parser = org.joda.time.format.DateTimeFormatterBuilder.new.append( nil, almostISOparsers ).toFormatter()
243
- if @timezone && !@sprintf_timezone
244
- joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
245
- else
246
- joda_parser = joda_parser.withOffsetParsed
247
- end
248
- parsers << lambda { |date| joda_parser.parseMillis(date) }
249
- when "UNIX" # unix epoch
250
- parsers << lambda do |date|
251
- raise "Invalid UNIX epoch value '#{date}'" unless /^\d+(?:\.\d+)?$/ === date || date.is_a?(Numeric)
252
- (date.to_f * 1000).to_i
253
- end
254
- when "UNIX_MS" # unix epoch in ms
255
- parsers << lambda do |date|
256
- raise "Invalid UNIX epoch value '#{date}'" unless /^\d+$/ === date || date.is_a?(Numeric)
257
- date.to_i
258
- end
259
- when "TAI64N" # TAI64 with nanoseconds, -10000 accounts for leap seconds
260
- parsers << lambda do |date|
261
- # Skip leading "@" if it is present (common in tai64n times)
262
- date = date[1..-1] if date[0, 1] == "@"
263
- return (date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000)
264
- end
265
- else
266
- begin
267
- format_has_year = format.match(/y|Y/)
268
- format_has_timezone = format.match(/Z/)
269
- joda_parser = org.joda.time.format.DateTimeFormat.forPattern(format)
270
- if @timezone && !@sprintf_timezone
271
- joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
272
- else
273
- joda_parser = joda_parser.withOffsetParsed
274
- end
275
- if locale
276
- joda_parser = joda_parser.withLocale(locale)
277
- end
278
- if @sprintf_timezone
279
- parsers << lambda { |date , tz|
280
- return parseWithJodaParser(joda_parser.withZone(org.joda.time.DateTimeZone.forID(tz)), date, format_has_year, format_has_timezone)
281
- }
282
- end
283
- parsers << lambda do |date|
284
- return parseWithJodaParser(joda_parser, date, format_has_year, format_has_timezone)
285
- end
286
-
287
- #Include a fallback parser to english when default locale is non-english
288
- if !locale &&
289
- "en" != java.util.Locale.getDefault().getLanguage() &&
290
- (format.include?("MMM") || format.include?("E"))
291
- en_joda_parser = joda_parser.withLocale(java.util.Locale.forLanguageTag('en-US'))
292
- parsers << lambda { |date| parseWithJodaParser(en_joda_parser, date, format_has_year, format_has_timezone) }
293
- end
294
- rescue JavaException => e
295
- raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",
296
- :plugin => "filter", :type => "date",
297
- :error => "#{e.message} for pattern '#{format}'")
298
- end
299
- end
188
+ end # def initialize
300
189
 
301
- @logger.debug("Adding type with date config", :type => @type,
302
- :field => field, :format => format)
303
- @parsers[field] << {
304
- :parser => parsers,
305
- :format => format
306
- }
307
- end
190
+ def multi_filter(events)
191
+ @datefilter.receive(events)
308
192
  end
309
193
 
310
194
  def filter(event)
311
- @logger.debug? && @logger.debug("Date filter: received event", :type => event.get("type"))
312
-
313
- @parsers.each do |field, fieldparsers|
314
- @logger.debug? && @logger.debug("Date filter looking for field",
315
- :type => event.get("type"), :field => field)
316
- next unless event.include?(field)
317
-
318
- fieldvalues = event.get(field)
319
- fieldvalues = [fieldvalues] if !fieldvalues.is_a?(Array)
320
- fieldvalues.each do |value|
321
- next if value.nil?
322
- begin
323
- epochmillis = nil
324
- success = false
325
- last_exception = RuntimeError.new "Unknown"
326
- fieldparsers.each do |parserconfig|
327
- parserconfig[:parser].each do |parser|
328
- begin
329
- if @sprintf_timezone
330
- epochmillis = parser.call(value, event.sprintf(@timezone))
331
- else
332
- epochmillis = parser.call(value)
333
- end
334
- success = true
335
- break # success
336
- rescue StandardError, JavaException => e
337
- last_exception = e
338
- end
339
- end # parserconfig[:parser].each
340
- break if success
341
- end # fieldparsers.each
342
-
343
- raise last_exception unless success
344
-
345
- # Convert joda DateTime to a ruby Time
346
- event.set(@target, LogStash::Timestamp.at(epochmillis / 1000, (epochmillis % 1000) * 1000))
347
-
348
- @logger.debug? && @logger.debug("Date parsing done", :value => value, :timestamp => event.get(@target))
349
- metric.increment(:matches)
350
- filter_matched(event)
351
- rescue StandardError, JavaException => e
352
- @logger.warn("Failed parsing date from field", :field => field,
353
- :value => value, :exception => e.message,
354
- :config_parsers => fieldparsers.collect {|x| x[:format]}.join(','),
355
- :config_locale => @locale ? @locale : "default="+java.util.Locale.getDefault().toString()
356
- )
357
- # Tag this event if we can't parse it. We can use this later to
358
- # reparse+reindex logs if we improve the patterns given.
359
- metric.increment(:failures)
360
- @tag_on_failure.each do |tag|
361
- event.tag(tag)
362
- end
363
- end
364
- end
365
- end
366
-
367
- return event
195
+ multi_filter([event]).first
368
196
  end
369
197
  end
@@ -1,17 +1,17 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-filter-date'
4
- s.version = '3.0.3'
4
+ s.version = '3.1.1'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event."
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
8
8
  s.authors = ["Elastic"]
9
9
  s.email = 'info@elastic.co'
10
10
  s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
11
- s.require_paths = ["lib"]
11
+ s.require_paths = ["lib", "vendor/jar-dependencies"]
12
12
 
13
13
  # Files
14
- s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
14
+ s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT', "vendor/jar-dependencies/**/*.jar"]
15
15
 
16
16
  # Tests
17
17
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -25,5 +25,6 @@ Gem::Specification.new do |s|
25
25
  s.add_development_dependency 'logstash-codec-json'
26
26
  s.add_development_dependency 'logstash-output-null'
27
27
  s.add_development_dependency 'logstash-devutils'
28
+ s.add_development_dependency 'benchmark-ips'
28
29
  end
29
30
 
@@ -5,6 +5,9 @@ require "logstash/filters/date"
5
5
 
6
6
  puts "Skipping date performance tests because this ruby is not jruby" if RUBY_ENGINE != "jruby"
7
7
  RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
8
+ after do
9
+ org.logstash.filters.parser.JodaParser.setDefaultClock(org.logstash.filters.parser.JodaParser.wallClock);
10
+ end
8
11
 
9
12
  describe "giving an invalid match config, raise a configuration error" do
10
13
  config <<-CONFIG
@@ -116,10 +119,12 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
116
119
  times = {
117
120
  "0" => "1970-01-01T00:00:00.000Z",
118
121
  "1000000000" => "2001-09-09T01:46:40.000Z",
122
+ "1478207457" => "2016-11-03T21:10:57.000Z",
119
123
 
120
124
  # LOGSTASH-279 - sometimes the field is a number.
121
125
  0 => "1970-01-01T00:00:00.000Z",
122
- 1000000000 => "2001-09-09T01:46:40.000Z"
126
+ 1000000000 => "2001-09-09T01:46:40.000Z",
127
+ 1478207457 => "2016-11-03T21:10:57.000Z"
123
128
  }
124
129
  times.each do |input, output|
125
130
  sample("mydate" => input) do
@@ -191,6 +196,38 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
191
196
  end # times.each
192
197
  end
193
198
 
199
+ describe "parsing with UNIX and UNIX_MS" do
200
+ config <<-CONFIG
201
+ filter {
202
+ date {
203
+ match => [ "mydate", "UNIX", "UNIX_MS" ]
204
+ locale => "en"
205
+ }
206
+ }
207
+ CONFIG
208
+
209
+ times = {
210
+ "0" => "1970-01-01T00:00:00.000Z",
211
+ "1000000000" => "2001-09-09T01:46:40.000Z",
212
+ "1000000000123" => "2001-09-09T01:46:40.123Z",
213
+ "1478207457" => "2016-11-03T21:10:57.000Z",
214
+ "1478207457.456" => "2016-11-03T21:10:57.456Z",
215
+
216
+ # LOGSTASH-279 - sometimes the field is a number.
217
+ 0 => "1970-01-01T00:00:00.000Z",
218
+ 1000000000 => "2001-09-09T01:46:40.000Z",
219
+ 1000000000123 => "2001-09-09T01:46:40.123Z",
220
+ 1478207457 => "2016-11-03T21:10:57.000Z",
221
+ 1478207457.456 => "2016-11-03T21:10:57.456Z",
222
+ }
223
+ times.each do |input, output|
224
+ sample("mydate" => input) do
225
+ insist { subject.get("mydate") } == input
226
+ insist { subject.get("@timestamp").time } == Time.iso8601(output)
227
+ end
228
+ end # times.each
229
+ end
230
+
194
231
  describe "failed parses should not cause a failure (LOGSTASH-641)" do
195
232
  config <<-'CONFIG'
196
233
  input {
@@ -403,6 +440,7 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
403
440
  end
404
441
 
405
442
  sample "2016 Mar 26 02:00:37" do
443
+ p :subject => subject
406
444
  insist { subject.get("tags") } != ["_dateparsefailure"]
407
445
  insist { subject.get("@timestamp").to_s } == "2016-03-26T01:00:37.000Z"
408
446
  end
@@ -440,6 +478,7 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
440
478
  before(:each) do
441
479
  logstash_time = Time.utc(2014,1,1,00,30,50)
442
480
  allow(Time).to receive(:now).and_return(logstash_time)
481
+ org.logstash.filters.parser.JodaParser.setDefaultClock { org.joda.time.DateTime.new(2014,1,1,00,30,50, org.joda.time.DateTimeZone::UTC ) }
443
482
  end
444
483
 
445
484
  sample("message" => "Dec 31 23:59:00", "mytz" => "UTC") do
@@ -461,6 +500,7 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
461
500
  before(:each) do
462
501
  logstash_time = Time.utc(2013,12,31,23,59,50)
463
502
  allow(Time).to receive(:now).and_return(logstash_time)
503
+ org.logstash.filters.parser.JodaParser.setDefaultClock { org.joda.time.DateTime.new(2013,12,31,23,59,50, org.joda.time.DateTimeZone::UTC ) }
464
504
  end
465
505
 
466
506
  sample( "message" => "Jan 01 01:00:00", "mytz" => "UTC") do
@@ -468,7 +508,7 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
468
508
  end
469
509
  end
470
510
 
471
- describe "don't fail on next years DST switchover in CET" do
511
+ describe "don't fail on next years DST switchover in CET", :skip => "This test tries to parse a time that doesn't exist. '02:00:37' is a time that doesn't exist because this DST switch goes from 01:59:59 to 03:00:00, skipping 2am entirely. I don't know how this spec ever passed..." do
472
512
  config <<-CONFIG
473
513
  filter {
474
514
  date {
@@ -482,6 +522,7 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
482
522
  before(:each) do
483
523
  logstash_time = Time.utc(2016,03,29,23,59,50)
484
524
  allow(Time).to receive(:now).and_return(logstash_time)
525
+ org.logstash.filters.parser.JodaParser.setDefaultClock { org.joda.time.DateTime.new(2016,03,29,23,59,50, org.joda.time.DateTimeZone::UTC ) }
485
526
  end
486
527
 
487
528
  sample "Mar 26 02:00:37" do
@@ -520,6 +561,7 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
520
561
  before(:each) do
521
562
  logstash_time = Time.utc(2014,1,1,00,30,50)
522
563
  allow(Time).to receive(:now).and_return(logstash_time)
564
+ org.logstash.filters.parser.JodaParser.setDefaultClock { org.joda.time.DateTime.new(2014,1,1,00,30,50, org.joda.time.DateTimeZone::UTC ) }
523
565
  end
524
566
 
525
567
  sample "Dec 31 23:59:00" do
@@ -541,6 +583,7 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
541
583
  before(:each) do
542
584
  logstash_time = Time.utc(2013,12,31,23,59,50)
543
585
  allow(Time).to receive(:now).and_return(logstash_time)
586
+ org.logstash.filters.parser.JodaParser.setDefaultClock { org.joda.time.DateTime.new(2013,12,31,15,59,50, org.joda.time.DateTimeZone::UTC ) }
544
587
  end
545
588
 
546
589
  sample "Jan 01 01:00:00" do
@@ -613,9 +656,7 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
613
656
  end
614
657
 
615
658
  describe "Support fallback to english for non-english default locale" do
616
- default_locale = java.util.Locale.getDefault()
617
659
  #Override default locale with non-english
618
- java.util.Locale.setDefault(java.util.Locale.forLanguageTag('fr-FR'))
619
660
  config <<-CONFIG
620
661
  filter {
621
662
  date {
@@ -625,19 +666,25 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
625
666
  }
626
667
  CONFIG
627
668
 
669
+ around do |example|
670
+ default = java.util.Locale.getDefault
671
+ java.util.Locale.setDefault(java.util.Locale.forLanguageTag('fr-FR'))
672
+ example.run
673
+ java.util.Locale.setDefault(default)
674
+ end
675
+
628
676
  sample "01 September 2014" do
629
677
  insist { subject.get("@timestamp").time } == Time.iso8601("2014-09-01T00:00:00.000Z").utc
630
678
  end
631
- #Restore default locale
632
- java.util.Locale.setDefault(default_locale)
633
679
  end
634
680
 
635
681
  context "Default year handling when parsing with english fallback parser" do
636
682
 
637
- before(:each) do
638
- default_locale = java.util.Locale.getDefault()
639
- #Override default locale with non-english
683
+ around do |example|
684
+ default = java.util.Locale.getDefault
640
685
  java.util.Locale.setDefault(java.util.Locale.forLanguageTag('fr-FR'))
686
+ example.run
687
+ java.util.Locale.setDefault(default)
641
688
  end
642
689
 
643
690
  puts "override locale"
@@ -669,6 +716,7 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
669
716
  before(:each) do
670
717
  logstash_time = Time.utc(2014,1,1,00,30,50)
671
718
  allow(Time).to receive(:now).and_return(logstash_time)
719
+ org.logstash.filters.parser.JodaParser.setDefaultClock { org.joda.time.DateTime.new(2014,1,1,00,30,50, org.joda.time.DateTimeZone::UTC) }
672
720
  end
673
721
 
674
722
  sample "Dec 31 23:59:00" do
@@ -689,6 +737,7 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
689
737
  before(:each) do
690
738
  logstash_time = Time.utc(2013,12,31,23,59,50)
691
739
  allow(Time).to receive(:now).and_return(logstash_time)
740
+ org.logstash.filters.parser.JodaParser.setDefaultClock { org.joda.time.DateTime.new(2013,12,31,23,59,50, org.joda.time.DateTimeZone::UTC) }
692
741
  end
693
742
 
694
743
  sample "Jan 01 01:00:00" do
@@ -0,0 +1,369 @@
1
+ # encoding: utf-8
2
+ require "logstash/filters/base"
3
+ require "logstash/namespace"
4
+ require "logstash/timestamp"
5
+
6
+ # The date filter is used for parsing dates from fields, and then using that
7
+ # date or timestamp as the logstash timestamp for the event.
8
+ #
9
+ # For example, syslog events usually have timestamps like this:
10
+ # [source,ruby]
11
+ # "Apr 17 09:32:01"
12
+ #
13
+ # You would use the date format `MMM dd HH:mm:ss` to parse this.
14
+ #
15
+ # The date filter is especially important for sorting events and for
16
+ # backfilling old data. If you don't get the date correct in your
17
+ # event, then searching for them later will likely sort out of order.
18
+ #
19
+ # In the absence of this filter, logstash will choose a timestamp based on the
20
+ # first time it sees the event (at input time), if the timestamp is not already
21
+ # set in the event. For example, with file input, the timestamp is set to the
22
+ # time of each read.
23
+ class LogStash::Filters::DateRuby < LogStash::Filters::Base
24
+ if RUBY_ENGINE == "jruby"
25
+ JavaException = java.lang.Exception
26
+ UTC = org.joda.time.DateTimeZone.forID("UTC")
27
+ java_import org.joda.time.LocalDateTime
28
+ class LocalDateTime
29
+ java_alias :to_datetime_with_tz, :toDateTime, [Java::org.joda.time.DateTimeZone]
30
+ end
31
+ end
32
+
33
+ config_name "old_date_filter"
34
+
35
+ # Specify a time zone canonical ID to be used for date parsing.
36
+ # The valid IDs are listed on the http://joda-time.sourceforge.net/timezones.html[Joda.org available time zones page].
37
+ # This is useful in case the time zone cannot be extracted from the value,
38
+ # and is not the platform default.
39
+ # If this is not specified the platform default will be used.
40
+ # Canonical ID is good as it takes care of daylight saving time for you
41
+ # For example, `America/Los_Angeles` or `Europe/Paris` are valid IDs.
42
+ # This field can be dynamic and include parts of the event using the `%{field}` syntax
43
+ config :timezone, :validate => :string
44
+
45
+ # Specify a locale to be used for date parsing using either IETF-BCP47 or POSIX language tag.
46
+ # Simple examples are `en`,`en-US` for BCP47 or `en_US` for POSIX.
47
+ #
48
+ # The locale is mostly necessary to be set for parsing month names (pattern with `MMM`) and
49
+ # weekday names (pattern with `EEE`).
50
+ #
51
+ # If not specified, the platform default will be used but for non-english platform default
52
+ # an english parser will also be used as a fallback mechanism.
53
+ config :locale, :validate => :string
54
+
55
+ # An array with field name first, and format patterns following, `[ field,
56
+ # formats... ]`
57
+ #
58
+ # If your time field has multiple possible formats, you can do this:
59
+ # [source,ruby]
60
+ # match => [ "logdate", "MMM dd YYY HH:mm:ss",
61
+ # "MMM d YYY HH:mm:ss", "ISO8601" ]
62
+ #
63
+ # The above will match a syslog (rfc3164) or `iso8601` timestamp.
64
+ #
65
+ # There are a few special exceptions. The following format literals exist
66
+ # to help you save time and ensure correctness of date parsing.
67
+ #
68
+ # * `ISO8601` - should parse any valid ISO8601 timestamp, such as
69
+ # `2011-04-19T03:44:01.103Z`
70
+ # * `UNIX` - will parse *float or int* value expressing unix time in seconds since epoch like 1326149001.132 as well as 1326149001
71
+ # * `UNIX_MS` - will parse **int** value expressing unix time in milliseconds since epoch like 1366125117000
72
+ # * `TAI64N` - will parse tai64n time values
73
+ #
74
+ # For example, if you have a field `logdate`, with a value that looks like
75
+ # `Aug 13 2010 00:03:44`, you would use this configuration:
76
+ # [source,ruby]
77
+ # filter {
78
+ # date {
79
+ # match => [ "logdate", "MMM dd YYYY HH:mm:ss" ]
80
+ # }
81
+ # }
82
+ #
83
+ # If your field is nested in your structure, you can use the nested
84
+ # syntax `[foo][bar]` to match its value. For more information, please refer to
85
+ # <<logstash-config-field-references>>
86
+ #
87
+ # *More details on the syntax*
88
+ #
89
+ # The syntax used for parsing date and time text uses letters to indicate the
90
+ # kind of time value (month, minute, etc), and a repetition of letters to
91
+ # indicate the form of that value (2-digit month, full month name, etc).
92
+ #
93
+ # Here's what you can use to parse dates and times:
94
+ #
95
+ # [horizontal]
96
+ # y:: year
97
+ # yyyy::: full year number. Example: `2015`.
98
+ # yy::: two-digit year. Example: `15` for the year 2015.
99
+ #
100
+ # M:: month of the year
101
+ # M::: minimal-digit month. Example: `1` for January and `12` for December.
102
+ # MM::: two-digit month. zero-padded if needed. Example: `01` for January and `12` for December
103
+ # MMM::: abbreviated month text. Example: `Jan` for January. Note: The language used depends on your locale. See the `locale` setting for how to change the language.
104
+ # MMMM::: full month text, Example: `January`. Note: The language used depends on your locale.
105
+ #
106
+ # d:: day of the month
107
+ # d::: minimal-digit day. Example: `1` for the 1st of the month.
108
+ # dd::: two-digit day, zero-padded if needed. Example: `01` for the 1st of the month.
109
+ #
110
+ # H:: hour of the day (24-hour clock)
111
+ # H::: minimal-digit hour. Example: `0` for midnight.
112
+ # HH::: two-digit hour, zero-padded if needed. Example: `00` for midnight.
113
+ #
114
+ # m:: minutes of the hour (60 minutes per hour)
115
+ # m::: minimal-digit minutes. Example: `0`.
116
+ # mm::: two-digit minutes, zero-padded if needed. Example: `00`.
117
+ #
118
+ # s:: seconds of the minute (60 seconds per minute)
119
+ # s::: minimal-digit seconds. Example: `0`.
120
+ # ss::: two-digit seconds, zero-padded if needed. Example: `00`.
121
+ #
122
+ # S:: fraction of a second
123
+ # *Maximum precision is milliseconds (`SSS`). Beyond that, zeroes are appended.*
124
+ # S::: tenths of a second. Example: `0` for a subsecond value `012`
125
+ # SS::: hundredths of a second. Example: `01` for a subsecond value `01`
126
+ # SSS::: thousandths of a second. Example: `012` for a subsecond value `012`
127
+ #
128
+ # Z:: time zone offset or identity
129
+ # Z::: Timezone offset structured as HHmm (hour and minutes offset from Zulu/UTC). Example: `-0700`.
130
+ # ZZ::: Timezone offset structured as HH:mm (colon in between hour and minute offsets). Example: `-07:00`.
131
+ # ZZZ::: Timezone identity. Example: `America/Los_Angeles`. Note: Valid IDs are listed on the http://joda-time.sourceforge.net/timezones.html[Joda.org available time zones page].
132
+ #
133
+ # z:: time zone names. *Time zone names ('z') cannot be parsed.*
134
+ #
135
+ # w:: week of the year
136
+ # w::: minimal-digit week. Example: `1`.
137
+ # ww::: two-digit week, zero-padded if needed. Example: `01`.
138
+ #
139
+ # D:: day of the year
140
+ #
141
+ # e:: day of the week (number)
142
+ #
143
+ # E:: day of the week (text)
144
+ # E, EE, EEE::: Abbreviated day of the week. Example: `Mon`, `Tue`, `Wed`, `Thu`, `Fri`, `Sat`, `Sun`. Note: The actual language of this will depend on your locale.
145
+ # EEEE::: The full text day of the week. Example: `Monday`, `Tuesday`, ... Note: The actual language of this will depend on your locale.
146
+ #
147
+ # For non-formatting syntax, you'll need to put single-quote characters around the value. For example, if you were parsing ISO8601 time, "2015-01-01T01:12:23" that little "T" isn't a valid time format, and you want to say "literally, a T", your format would be this: "yyyy-MM-dd'T'HH:mm:ss"
148
+ #
149
+ # Other less common date units, such as era (G), century \(C), am/pm (a), and # more, can be learned about on the
150
+ # http://www.joda.org/joda-time/key_format.html[joda-time documentation].
151
+ config :match, :validate => :array, :default => []
152
+
153
+ # Store the matching timestamp into the given target field. If not provided,
154
+ # default to updating the `@timestamp` field of the event.
155
+ config :target, :validate => :string, :default => LogStash::Event::TIMESTAMP
156
+
157
+ # Append values to the `tags` field when there has been no
158
+ # successful match
159
+ config :tag_on_failure, :validate => :array, :default => ["_dateparsefailure"]
160
+
161
+ # LOGSTASH-34
162
+ DATEPATTERNS = %w{ y d H m s S }
163
+
164
+ def initialize(config = {})
165
+ super
166
+
167
+ @parsers = Hash.new { |h,k| h[k] = [] }
168
+ end # def initialize
169
+
170
+ def register
171
+ require "java"
172
+ if @match.length < 2
173
+ raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",
174
+ :plugin => "filter", :type => "date",
175
+ :error => "The match setting should contains first a field name and at least one date format, current value is #{@match}")
176
+ end
177
+
178
+ locale = nil
179
+ if @locale
180
+ if @locale.include? '_'
181
+ @logger.warn("Date filter now use BCP47 format for locale, replacing underscore with dash")
182
+ @locale.gsub!('_','-')
183
+ end
184
+ locale = java.util.Locale.forLanguageTag(@locale)
185
+ end
186
+
187
+ @sprintf_timezone = @timezone && !@timezone.index("%{").nil?
188
+ setupMatcher(@config["match"].shift, locale, @config["match"] )
189
+ end
190
+
191
+ def parseWithJodaParser(joda_parser, date, format_has_year, format_has_timezone)
192
+ return joda_parser.parseMillis(date) if format_has_year
193
+ now = Time.now
194
+ now_month = now.month
195
+ if (format_has_timezone)
196
+ result = joda_parser.parseDateTime(date)
197
+ else
198
+ # Parse date in UTC, Timezone correction later
199
+ result = joda_parser.withZone(UTC).parseLocalDateTime(date)
200
+ end
201
+
202
+ event_month = result.getMonthOfYear
203
+
204
+ if (event_month == now_month)
205
+ result = result.with_year(now.year)
206
+ elsif (event_month == 12 && now_month == 1)
207
+ result = result.with_year(now.year-1)
208
+ elsif (event_month == 1 && now_month == 12)
209
+ result = result.with_year(now.year+1)
210
+ else
211
+ result = result.with_year(now.year)
212
+ end
213
+
214
+ if (format_has_timezone)
215
+ return result.get_millis
216
+ else
217
+ #Timezone correction
218
+ return result.to_datetime_with_tz(joda_parser.getZone()).get_millis
219
+ end
220
+ end
221
+
222
+ def setupMatcher(field, locale, value)
223
+ metric.gauge(:formats, value.length)
224
+ value.each do |format|
225
+ parsers = []
226
+ case format
227
+ when "ISO8601"
228
+ iso_parser = org.joda.time.format.ISODateTimeFormat.dateTimeParser
229
+ if @timezone && !@sprintf_timezone
230
+ iso_parser = iso_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
231
+ else
232
+ iso_parser = iso_parser.withOffsetParsed
233
+ end
234
+ parsers << lambda { |date| iso_parser.parseMillis(date) }
235
+ #Fall back solution of almost ISO8601 date-time
236
+ almostISOparsers = [
237
+ org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSSZ").getParser(),
238
+ org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS").getParser(),
239
+ org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss,SSSZ").getParser(),
240
+ org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss,SSS").getParser()
241
+ ].to_java(org.joda.time.format.DateTimeParser)
242
+ joda_parser = org.joda.time.format.DateTimeFormatterBuilder.new.append( nil, almostISOparsers ).toFormatter()
243
+ if @timezone && !@sprintf_timezone
244
+ joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
245
+ else
246
+ joda_parser = joda_parser.withOffsetParsed
247
+ end
248
+ parsers << lambda { |date| joda_parser.parseMillis(date) }
249
+ when "UNIX" # unix epoch
250
+ parsers << lambda do |date|
251
+ raise "Invalid UNIX epoch value '#{date}'" unless /^\d+(?:\.\d+)?$/ === date || date.is_a?(Numeric)
252
+ (date.to_f * 1000).to_i
253
+ end
254
+ when "UNIX_MS" # unix epoch in ms
255
+ parsers << lambda do |date|
256
+ raise "Invalid UNIX epoch value '#{date}'" unless /^\d+$/ === date || date.is_a?(Numeric)
257
+ date.to_i
258
+ end
259
+ when "TAI64N" # TAI64 with nanoseconds, -10000 accounts for leap seconds
260
+ parsers << lambda do |date|
261
+ # Skip leading "@" if it is present (common in tai64n times)
262
+ date = date[1..-1] if date[0, 1] == "@"
263
+ return (date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000)
264
+ end
265
+ else
266
+ begin
267
+ format_has_year = format.match(/y|Y/)
268
+ format_has_timezone = format.match(/Z/)
269
+ joda_parser = org.joda.time.format.DateTimeFormat.forPattern(format)
270
+ if @timezone && !@sprintf_timezone
271
+ joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
272
+ else
273
+ joda_parser = joda_parser.withOffsetParsed
274
+ end
275
+ if locale
276
+ joda_parser = joda_parser.withLocale(locale)
277
+ end
278
+ if @sprintf_timezone
279
+ parsers << lambda { |date , tz|
280
+ return parseWithJodaParser(joda_parser.withZone(org.joda.time.DateTimeZone.forID(tz)), date, format_has_year, format_has_timezone)
281
+ }
282
+ end
283
+ parsers << lambda do |date|
284
+ return parseWithJodaParser(joda_parser, date, format_has_year, format_has_timezone)
285
+ end
286
+
287
+ #Include a fallback parser to english when default locale is non-english
288
+ if !locale &&
289
+ "en" != java.util.Locale.getDefault().getLanguage() &&
290
+ (format.include?("MMM") || format.include?("E"))
291
+ en_joda_parser = joda_parser.withLocale(java.util.Locale.forLanguageTag('en-US'))
292
+ parsers << lambda { |date| parseWithJodaParser(en_joda_parser, date, format_has_year, format_has_timezone) }
293
+ end
294
+ rescue JavaException => e
295
+ raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",
296
+ :plugin => "filter", :type => "date",
297
+ :error => "#{e.message} for pattern '#{format}'")
298
+ end
299
+ end
300
+
301
+ @logger.debug("Adding type with date config", :type => @type,
302
+ :field => field, :format => format)
303
+ @parsers[field] << {
304
+ :parser => parsers,
305
+ :format => format
306
+ }
307
+ end
308
+ end
309
+
310
+ def filter(event)
311
+ @logger.debug? && @logger.debug("Date filter: received event", :type => event.get("type"))
312
+
313
+ @parsers.each do |field, fieldparsers|
314
+ @logger.debug? && @logger.debug("Date filter looking for field",
315
+ :type => event.get("type"), :field => field)
316
+ next unless event.include?(field)
317
+
318
+ fieldvalues = event.get(field)
319
+ fieldvalues = [fieldvalues] if !fieldvalues.is_a?(Array)
320
+ fieldvalues.each do |value|
321
+ next if value.nil?
322
+ begin
323
+ epochmillis = nil
324
+ success = false
325
+ last_exception = RuntimeError.new "Unknown"
326
+ fieldparsers.each do |parserconfig|
327
+ parserconfig[:parser].each do |parser|
328
+ begin
329
+ if @sprintf_timezone
330
+ epochmillis = parser.call(value, event.sprintf(@timezone))
331
+ else
332
+ epochmillis = parser.call(value)
333
+ end
334
+ success = true
335
+ break # success
336
+ rescue StandardError, JavaException => e
337
+ last_exception = e
338
+ end
339
+ end # parserconfig[:parser].each
340
+ break if success
341
+ end # fieldparsers.each
342
+
343
+ raise last_exception unless success
344
+
345
+ # Convert joda DateTime to a ruby Time
346
+ event.set(@target, LogStash::Timestamp.at(epochmillis / 1000, (epochmillis % 1000) * 1000))
347
+
348
+ @logger.debug? && @logger.debug("Date parsing done", :value => value, :timestamp => event.get(@target))
349
+ metric.increment(:matches)
350
+ filter_matched(event)
351
+ rescue StandardError, JavaException => e
352
+ @logger.warn("Failed parsing date from field", :field => field,
353
+ :value => value, :exception => e.message,
354
+ :config_parsers => fieldparsers.collect {|x| x[:format]}.join(','),
355
+ :config_locale => @locale ? @locale : "default="+java.util.Locale.getDefault().toString()
356
+ )
357
+ # Tag this event if we can't parse it. We can use this later to
358
+ # reparse+reindex logs if we improve the patterns given.
359
+ metric.increment(:failures)
360
+ @tag_on_failure.each do |tag|
361
+ event.tag(tag)
362
+ end
363
+ end
364
+ end
365
+ end
366
+
367
+ return event
368
+ end
369
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-date
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.3
4
+ version: 3.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-07-14 00:00:00.000000000 Z
11
+ date: 2016-11-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -86,6 +86,20 @@ dependencies:
86
86
  - - ">="
87
87
  - !ruby/object:Gem::Version
88
88
  version: '0'
89
+ - !ruby/object:Gem::Dependency
90
+ requirement: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ version: '0'
95
+ name: benchmark-ips
96
+ prerelease: false
97
+ type: :development
98
+ version_requirements: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: '0'
89
103
  description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
90
104
  email: info@elastic.co
91
105
  executables: []
@@ -98,9 +112,12 @@ files:
98
112
  - LICENSE
99
113
  - NOTICE.TXT
100
114
  - README.md
115
+ - lib/logstash-filter-date_jars.rb
101
116
  - lib/logstash/filters/date.rb
102
117
  - logstash-filter-date.gemspec
103
118
  - spec/filters/date_spec.rb
119
+ - spec/fixtures/old_date_filter.rb
120
+ - vendor/jar-dependencies/org/logstash/filters/logstash-filter-date/3.1.0/logstash-filter-date-3.1.0.jar
104
121
  homepage: http://www.elastic.co/guide/en/logstash/current/index.html
105
122
  licenses:
106
123
  - Apache License (2.0)
@@ -111,6 +128,7 @@ post_install_message:
111
128
  rdoc_options: []
112
129
  require_paths:
113
130
  - lib
131
+ - vendor/jar-dependencies
114
132
  required_ruby_version: !ruby/object:Gem::Requirement
115
133
  requirements:
116
134
  - - ">="
@@ -123,9 +141,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
123
141
  version: '0'
124
142
  requirements: []
125
143
  rubyforge_project:
126
- rubygems_version: 2.6.3
144
+ rubygems_version: 2.4.8
127
145
  signing_key:
128
146
  specification_version: 4
129
147
  summary: The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event.
130
148
  test_files:
131
149
  - spec/filters/date_spec.rb
150
+ - spec/fixtures/old_date_filter.rb