logstash-filter-date 2.0.2 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0779b4e5cffb47a4e1fa70185deceb766726f0f4
-  data.tar.gz: 971764c7d79a63e3b5360b7350613e7de980e555
+  metadata.gz: 90514b78e723a6637fbf1f1225cd43d043ecdcb5
+  data.tar.gz: 7129c219a98c1486b73cc715471619cf1393728f
 SHA512:
-  metadata.gz: 182c23c6cfe4c53aa61d371951fb08687654fbfcd899d4771b1f561aa929ce4e91de63f4c9dbc07f1d1b81e8a5abf8fa1addf7bb529f07c6e8175682c70b0f2f
-  data.tar.gz: 11064858b17452ce30490d50e6a3e6b067d0705db782d529e62949219899cf0f6cd540206ba14108f42d490c267c2058d6d6eb00b58a21440d9e8319cacbe413
+  metadata.gz: b3fc8119ad011c783c2d13bbbe30b463db520477d752a2a0268c7a6baef22872fd18628109044121d0da2292216706c40260da90c2eda2b51479ce047ad25833
+  data.tar.gz: b2340da5d6ec579266aa4b76db2398490bcb3e19bc5bc152d0a0a5cd58dd58cc662b9cb8bf66232fd6fa2aca8d28fd278c78f11d78411399d2a2d4f4d32cc6a1

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 2.1.0
+ - New year rollover should be handled better now when a year is not present in
+   the time format. If local time is December, and event time is January, the
+   year will be set to next year. Similar for if local time is January and
+   Event time is December, the year will be set to the previous year. This
+   should help keep times correct in the upcoming year rollover. (#33, #4)
+ - The `timezone` setting now supports sprintf format (#31)
+ - use Event#tag, relax specs for Java Event, code cleanups
+
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-date-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-date-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/filters/date.rb

@@ -35,6 +35,7 @@ class LogStash::Filters::Date < LogStash::Filters::Base
   # If this is not specified the platform default will be used.
   # Canonical ID is good as it takes care of daylight saving time for you
   # For example, `America/Los_Angeles` or `Europe/Paris` are valid IDs.
+  # This field can be dynamic and include parts of the event using the `%{field}` syntax
   config :timezone, :validate => :string
 
   # Specify a locale to be used for date parsing using either IETF-BCP47 or POSIX language tag.
@@ -87,7 +88,7 @@ class LogStash::Filters::Date < LogStash::Filters::Base
 
   # Store the matching timestamp into the given target field.  If not provided,
   # default to updating the `@timestamp` field of the event.
-  config :target, :validate => :string, :default => "@timestamp"
+  config :target, :validate => :string, :default => LogStash::Event::TIMESTAMP
 
   # Append values to the `tags` field when there has been no
   # successful match
@@ -96,14 +97,12 @@ class LogStash::Filters::Date < LogStash::Filters::Base
   # LOGSTASH-34
   DATEPATTERNS = %w{ y d H m s S }
 
-  public
   def initialize(config = {})
     super
 
     @parsers = Hash.new { |h,k| h[k] = [] }
   end # def initialize
 
-  public
   def register
     require "java"
     if @match.length < 2
@@ -120,6 +119,9 @@ class LogStash::Filters::Date < LogStash::Filters::Base
       end
       locale = java.util.Locale.forLanguageTag(@locale)
     end
+
+    @sprintf_timezone = @timezone && !@timezone.index("%{").nil?
+
     setupMatcher(@config["match"].shift, locale, @config["match"] )
   end
 
@@ -129,7 +131,7 @@ class LogStash::Filters::Date < LogStash::Filters::Base
       case format
         when "ISO8601"
           iso_parser = org.joda.time.format.ISODateTimeFormat.dateTimeParser
-          if @timezone
+          if @timezone && !@sprintf_timezone
             iso_parser = iso_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
           else
             iso_parser = iso_parser.withOffsetParsed
@@ -138,10 +140,12 @@ class LogStash::Filters::Date < LogStash::Filters::Base
           #Fall back solution of almost ISO8601 date-time
           almostISOparsers = [
             org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSSZ").getParser(),
-            org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS").getParser()
+            org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS").getParser(),
+            org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss,SSSZ").getParser(),
+            org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss,SSS").getParser()
           ].to_java(org.joda.time.format.DateTimeParser)
           joda_parser = org.joda.time.format.DateTimeFormatterBuilder.new.append( nil, almostISOparsers ).toFormatter()
-          if @timezone
+          if @timezone && !@sprintf_timezone
             joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
           else
             joda_parser = joda_parser.withOffsetParsed
@@ -158,15 +162,16 @@ class LogStash::Filters::Date < LogStash::Filters::Base
             date.to_i
           end
         when "TAI64N" # TAI64 with nanoseconds, -10000 accounts for leap seconds
-          parsers << lambda do |date| 
+          parsers << lambda do |date|
             # Skip leading "@" if it is present (common in tai64n times)
             date = date[1..-1] if date[0, 1] == "@"
             return (date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000)
           end
         else
           begin
-            joda_parser = org.joda.time.format.DateTimeFormat.forPattern(format).withDefaultYear(Time.new.year)
-            if @timezone
+            format_has_year = format.match(/y|Y/)
+            joda_parser = org.joda.time.format.DateTimeFormat.forPattern(format)
+            if @timezone && !@sprintf_timezone
               joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
             else
               joda_parser = joda_parser.withOffsetParsed
@@ -174,7 +179,28 @@ class LogStash::Filters::Date < LogStash::Filters::Base
             if locale
               joda_parser = joda_parser.withLocale(locale)
             end
-            parsers << lambda { |date| joda_parser.parseMillis(date) }
+            if @sprintf_timezone
+              parsers << lambda { |date , tz|
+                joda_parser.withZone(org.joda.time.DateTimeZone.forID(tz)).parseMillis(date)
+              }
+            end
+            parsers << lambda do |date|
+              return joda_parser.parseMillis(date) if format_has_year
+              now = Time.now
+              now_month = now.month
+              result = joda_parser.parseDateTime(date)
+              event_month = result.getMonthOfYear
+
+              if (event_month == now_month)
+                result.with_year(now.year)
+              elsif (event_month == 12 && now_month == 1)
+                result.with_year(now.year-1)
+              elsif (event_month == 1 && now_month == 12)
+                result.with_year(now.year+1)
+              else
+                result.with_year(now.year)
+              end.get_millis
+            end
 
             #Include a fallback parser to english when default locale is non-english
             if !locale &&
@@ -199,12 +225,9 @@ class LogStash::Filters::Date < LogStash::Filters::Base
     end
   end
 
-  # def register
-
-  public
   def filter(event)
     @logger.debug? && @logger.debug("Date filter: received event", :type => event["type"])
-    
+
     @parsers.each do |field, fieldparsers|
       @logger.debug? && @logger.debug("Date filter looking for field",
                                       :type => event["type"], :field => field)
@@ -221,7 +244,11 @@ class LogStash::Filters::Date < LogStash::Filters::Base
           fieldparsers.each do |parserconfig|
             parserconfig[:parser].each do |parser|
               begin
-                epochmillis = parser.call(value)
+                if @sprintf_timezone
+                  epochmillis = parser.call(value, event.sprintf(@timezone))
+                else
+                  epochmillis = parser.call(value)
+                end
                 success = true
                 break # success
               rescue StandardError, JavaException => e
@@ -247,13 +274,12 @@ class LogStash::Filters::Date < LogStash::Filters::Base
           # Tag this event if we can't parse it. We can use this later to
           # reparse+reindex logs if we improve the patterns given.
           @tag_on_failure.each do |tag|
-            event["tags"] ||= []
-            event["tags"] << tag unless event["tags"].include?(tag)
+            event.tag(tag)
           end
-        end # begin
-      end # fieldvalue.each
-    end # @parsers.each
+        end
+      end
+    end
 
     return event
-  end # def filter
-end # class LogStash::Filters::Date
+  end
+end

data/logstash-filter-date.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-date'
-  s.version         = '2.0.2'
+  s.version         = '2.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/date_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/filters/date"
 
@@ -48,15 +50,18 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
       "2001-09-05T16:36:36.123+0700"     => "2001-09-05T09:36:36.123Z",
       "2001-11-06T20:45:45.123-0000"     => "2001-11-06T20:45:45.123Z",
       "2001-12-07T23:54:54.123Z"         => "2001-12-07T23:54:54.123Z",
+      "2001-12-07T23:54:54,123Z"         => "2001-12-07T23:54:54.123Z",
 
       #Almost ISO8601 support, with timezone
 
       "2001-11-06 20:45:45.123-0000"     => "2001-11-06T20:45:45.123Z",
       "2001-12-07 23:54:54.123Z"         => "2001-12-07T23:54:54.123Z",
+      "2001-12-07 23:54:54,123Z"         => "2001-12-07T23:54:54.123Z",
 
       #Almost ISO8601 support, without timezone
 
       "2001-11-06 20:45:45.123"     => "2001-11-06T20:45:45.123Z",
+      "2001-11-06 20:45:45,123"     => "2001-11-06T20:45:45.123Z",
 
     }
 
@@ -357,6 +362,30 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
     end # times.each
   end
 
+  describe "parsing with timezone from event" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => ["mydate", "yyyy MMM dd HH:mm:ss"]
+          locale => "en"
+          timezone => "%{mytz}"
+        }
+      }
+    CONFIG
+
+    require 'java'
+    times = {
+      "2013 Nov 24 01:29:01" => "2013-11-24T09:29:01.000Z",
+      "2013 Jun 24 01:29:01" => "2013-06-24T08:29:01.000Z",
+    }
+    times.each do |input, output|
+      sample("mydate" => input, "mytz" => "America/Los_Angeles") do
+        insist { subject["mydate"] } == input
+        insist { subject["@timestamp"].time } == Time.iso8601(output).utc
+      end
+    end # times.each
+  end
+
   describe "LOGSTASH-34 - Default year should be this year" do
     config <<-CONFIG
       filter {
@@ -372,6 +401,42 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
     end
   end
 
+  describe "fill last year if december events arrive in january" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "message", "MMM dd HH:mm:ss" ]
+          locale => "en"
+          timezone => "UTC"
+        }
+      }
+    CONFIG
+
+    sample "Dec 31 23:59:00" do
+      logstash_time = Time.utc(2014,1,1,00,30,50)
+      expect(Time).to receive(:now).at_most(:twice).and_return(logstash_time)
+      insist { subject["@timestamp"].year } == 2013
+    end
+  end
+
+  describe "fill next year if january events arrive in december" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "message", "MMM dd HH:mm:ss" ]
+          locale => "en"
+          timezone => "UTC"
+        }
+      }
+    CONFIG
+
+    sample "Jan 01 01:00:00" do
+      logstash_time = Time.utc(2013,12,31,23,59,50)
+      expect(Time).to receive(:now).at_most(:twice).and_return(logstash_time)
+      insist { subject["@timestamp"].year } == 2014
+    end
+  end
+
   describe "Supporting locale only" do
     config <<-CONFIG
       filter {

metadata

@@ -1,22 +1,22 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-date
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2015-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0.beta2
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
   name: logstash-core
@@ -24,16 +24,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0.beta2
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-input-generator
@@ -41,13 +41,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-json
@@ -55,13 +55,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-output-null
@@ -69,13 +69,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -83,7 +83,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
@@ -113,12 +113,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []