logstash-filter-date 2.1.2 → 2.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eb61244507b109cd3259147af0cfd04a15747f7c
-  data.tar.gz: 2a79943e40e56365ad2a82b40f3b30c978312072
+  metadata.gz: ea6c51877b6a28e814bd00e623932612e5b9f8f1
+  data.tar.gz: ebe269a2eb0210e9b4eedcb796ecedc06bb830e4
 SHA512:
-  metadata.gz: 095c0061ca96d90605ce2c9ca8c6a19a62858ab0149ea2fa21adfc1d1bbbf4e36f5a9b3ca8a1d84449e1f7b04cda1afe140226c420a364d4fd7dd506071936ae
-  data.tar.gz: 6faabecb214c5f18290cb6ae7ad1141c57bb7a5229cb193771d0f028980cc4c18099c338cc85b6d27f356b6eaa9c3f0a1016df915819cec4104af702cd009633
+  metadata.gz: 8d905fb89de9f982b6b2125bab4adfc68861d17dcfd4d0bb92b3497382681964f29c82e7e26ee4930fb83aaa13929dcc8a54daac7c5063f430eca182337138ad
+  data.tar.gz: 75b71d3e79ac533ab6ad51d4b2ee749847d841f0601a9f8db44577b70941e4df30c534df9996e7ec6b220d5c556e94e11eb3457c001017ec4edef7cc32c232db

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 2.1.4
+  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
+# 2.1.3
+  - New dependency requirements for logstash-core for the 5.0 release
 ## 2.1.2
   - Make tests less reliant on implementation details of LogStash::Event
 ## 2.1.1

data/lib/logstash/filters/date.rb

@@ -48,11 +48,6 @@ class LogStash::Filters::Date < LogStash::Filters::Base
   # an english parser will also be used as a fallback mechanism.
   config :locale, :validate => :string
 
-  # The date formats allowed are anything allowed by Joda-Time (java time
-  # library). You can see the docs for this format here:
-  #
-  # http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html[joda.time.format.DateTimeFormat]
-  #
   # An array with field name first, and format patterns following, `[ field,
   # formats... ]`
   #
@@ -84,6 +79,71 @@ class LogStash::Filters::Date < LogStash::Filters::Base
   # If your field is nested in your structure, you can use the nested
   # syntax `[foo][bar]` to match its value. For more information, please refer to
   # <<logstash-config-field-references>>
+  #
+  # *More details on the syntax*
+  #
+  # The syntax used for parsing date and time text uses letters to indicate the
+  # kind of time value (month, minute, etc), and a repetition of letters to
+  # indicate the form of that value (2-digit month, full month name, etc).
+  #
+  # Here's what you can use to parse dates and times:
+  #
+  # [horizontal]
+  # y:: year
+  #   yyyy::: full year number. Example: `2015`.
+  #   yy::: two-digit year. Example: `15` for the year 2015.
+  #
+  # M:: month of the year
+  #   M::: minimal-digit month. Example: `1` for January and `12` for December.
+  #   MM::: two-digit month. zero-padded if needed. Example: `01` for January  and `12` for December
+  #   MMM::: abbreviated month text. Example: `Jan` for January. Note: The language used depends on your locale. See the `locale` setting for how to change the language.
+  #   MMMM::: full month text, Example: `January`. Note: The language used depends on your locale.
+  #
+  # d:: day of the month
+  #   d::: minimal-digit day. Example: `1` for the 1st of the month.
+  #   dd::: two-digit day, zero-padded if needed. Example: `01` for the 1st of the month.
+  #
+  # H:: hour of the day (24-hour clock)
+  #   H::: minimal-digit hour. Example: `0` for midnight.
+  #   HH::: two-digit hour, zero-padded if needed. Example: `00` for midnight.
+  #
+  # m:: minutes of the hour (60 minutes per hour)
+  #   m::: minimal-digit minutes. Example: `0`.
+  #   mm::: two-digit minutes, zero-padded if needed. Example: `00`.
+  #
+  # s:: seconds of the minute (60 seconds per minute)
+  #   s::: minimal-digit seconds. Example: `0`.
+  #   ss::: two-digit seconds, zero-padded if needed. Example: `00`.
+  #
+  # S:: fraction of a second
+  #   *Maximum precision is milliseconds (`SSS`). Beyond that, zeroes are appended.*
+  #   S::: tenths of a second. Example:  `0` for a subsecond value `012`
+  #   SS::: hundredths of a second. Example:  `01` for a subsecond value `01`
+  #   SSS::: thousandths of a second. Example:  `012` for a subsecond value `012`
+  #
+  # Z:: time zone offset or identity
+  #   Z::: Timezone offset structured as HHmm (hour and minutes offset from Zulu/UTC). Example: `-0700`.
+  #   ZZ::: Timezone offset structured as HH:mm (colon in between hour and minute offsets). Example: `-07:00`.
+  #   ZZZ::: Timezone identity. Example: `America/Los_Angeles`. Note: Valid IDs are listed on the http://joda-time.sourceforge.net/timezones.html[Joda.org available time zones page].
+  #
+  # z:: time zone names. *Time zone names ('z') cannot be parsed.*
+  #
+  # w:: week of the year
+  #   w::: minimal-digit week. Example: `1`.
+  #   ww::: two-digit week, zero-padded if needed. Example: `01`.
+  #
+  # D:: day of the year
+  #
+  # e:: day of the week (number)
+  #
+  # E:: day of the week (text)
+  #   E, EE, EEE::: Abbreviated day of the week. Example:  `Mon`, `Tue`, `Wed`, `Thu`, `Fri`, `Sat`, `Sun`. Note: The actual language of this will depend on your locale.
+  #   EEEE::: The full text day of the week. Example: `Monday`, `Tuesday`, ... Note: The actual language of this will depend on your locale.
+  #
+  # For non-formatting syntax, you'll need to put single-quote characters around the value. For example, if you were parsing ISO8601 time, "2015-01-01T01:12:23" that little "T" isn't a valid time format, and you want to say "literally, a T", your format would be this: "yyyy-MM-dd'T'HH:mm:ss"
+  #
+  # Other less common date units, such as era (G), century \(C), am/pm (a), and # more, can be learned about on the
+  # http://www.joda.org/joda-time/key_format.html[joda-time documentation].
   config :match, :validate => :array, :default => []
 
   # Store the matching timestamp into the given target field.  If not provided,
@@ -125,6 +185,24 @@ class LogStash::Filters::Date < LogStash::Filters::Base
     setupMatcher(@config["match"].shift, locale, @config["match"] )
   end
 
+  def parseWithJodaParser(joda_parser, date, format_has_year)
+    return joda_parser.parseMillis(date) if format_has_year
+    now = Time.now
+    now_month = now.month
+    result = joda_parser.parseDateTime(date)
+    event_month = result.getMonthOfYear
+
+    if (event_month == now_month)
+      result.with_year(now.year)
+    elsif (event_month == 12 && now_month == 1)
+      result.with_year(now.year-1)
+    elsif (event_month == 1 && now_month == 12)
+      result.with_year(now.year+1)
+    else
+      result.with_year(now.year)
+    end.get_millis
+  end
+
   def setupMatcher(field, locale, value)
     value.each do |format|
       parsers = []
@@ -181,25 +259,11 @@ class LogStash::Filters::Date < LogStash::Filters::Base
             end
             if @sprintf_timezone
               parsers << lambda { |date , tz|
-                joda_parser.withZone(org.joda.time.DateTimeZone.forID(tz)).parseMillis(date)
+                return parseWithJodaParser(joda_parser.withZone(org.joda.time.DateTimeZone.forID(tz)), date, format_has_year)
               }
             end
             parsers << lambda do |date|
-              return joda_parser.parseMillis(date) if format_has_year
-              now = Time.now
-              now_month = now.month
-              result = joda_parser.parseDateTime(date)
-              event_month = result.getMonthOfYear
-
-              if (event_month == now_month)
-                result.with_year(now.year)
-              elsif (event_month == 12 && now_month == 1)
-                result.with_year(now.year-1)
-              elsif (event_month == 1 && now_month == 12)
-                result.with_year(now.year+1)
-              else
-                result.with_year(now.year)
-              end.get_millis
+              return parseWithJodaParser(joda_parser, date, format_has_year)
             end
 
             #Include a fallback parser to english when default locale is non-english
@@ -207,7 +271,7 @@ class LogStash::Filters::Date < LogStash::Filters::Base
               "en" != java.util.Locale.getDefault().getLanguage() &&
               (format.include?("MMM") || format.include?("E"))
               en_joda_parser = joda_parser.withLocale(java.util.Locale.forLanguageTag('en-US'))
-              parsers << lambda { |date| en_joda_parser.parseMillis(date) }
+              parsers << lambda { |date| parseWithJodaParser(en_joda_parser, date, format_has_year) }
             end
           rescue JavaException => e
             raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",

data/logstash-filter-date.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-date'
-  s.version         = '2.1.2'
+  s.version         = '2.1.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
   s.add_runtime_dependency 'logstash-input-generator'
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'logstash-output-null'

data/spec/filters/date_spec.rb

@@ -386,6 +386,67 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
     end # times.each
   end
 
+  context "Default year handling when parsing with timezone from event" do
+
+    describe "LOGSTASH-34 - Default year should be this year" do
+      config <<-CONFIG
+        filter {
+          date {
+            match => [ "message", "EEE MMM dd HH:mm:ss" ]
+            locale => "en"
+            timezone => "%{mytz}"
+          }
+        }
+      CONFIG
+
+      sample("message" => "Sun Jun 02 20:38:03", "mytz" => "UTC") do
+        insist { subject["@timestamp"].year } == Time.now.year
+      end
+    end
+
+    describe "fill last year if december events arrive in january" do
+      config <<-CONFIG
+        filter {
+          date {
+            match => [ "message", "MMM dd HH:mm:ss" ]
+            locale => "en"
+            timezone => "%{mytz}"
+          }
+        }
+      CONFIG
+
+      before(:each) do
+        logstash_time = Time.utc(2014,1,1,00,30,50)
+        allow(Time).to receive(:now).and_return(logstash_time)
+      end
+
+      sample("message" => "Dec 31 23:59:00", "mytz" => "UTC") do
+        insist { subject["@timestamp"].year } == 2013
+      end
+    end
+
+    describe "fill next year if january events arrive in december" do
+      config <<-CONFIG
+        filter {
+          date {
+            match => [ "message", "MMM dd HH:mm:ss" ]
+            locale => "en"
+            timezone => "%{mytz}"
+          }
+        }
+      CONFIG
+
+      before(:each) do
+        logstash_time = Time.utc(2013,12,31,23,59,50)
+        allow(Time).to receive(:now).and_return(logstash_time)
+      end
+
+      sample( "message" => "Jan 01 01:00:00", "mytz" => "UTC") do
+        insist { subject["@timestamp"].year } == 2014
+      end
+    end
+  end
+
   describe "LOGSTASH-34 - Default year should be this year" do
     config <<-CONFIG
       filter {
@@ -526,4 +587,69 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
     #Restore default locale
     java.util.Locale.setDefault(default_locale)
   end
+
+  context "Default year handling when parsing with english fallback parser" do
+
+    before(:each) do
+      default_locale = java.util.Locale.getDefault()
+      #Override default locale with non-english
+      java.util.Locale.setDefault(java.util.Locale.forLanguageTag('fr-FR'))
+    end
+
+    puts "override locale"
+    describe "LOGSTASH-34 - Default year should be this year" do
+      config <<-CONFIG
+        filter {
+          date {
+            match => [ "message", "EEE MMM dd HH:mm:ss" ]
+            timezone => "UTC"
+          }
+        }
+      CONFIG
+
+      sample "Sun Jun 02 20:38:03" do
+        insist { subject["@timestamp"].year } == Time.now.year
+      end
+    end
+
+    describe "fill last year if december events arrive in january" do
+      config <<-CONFIG
+        filter {
+          date {
+            match => [ "message", "MMM dd HH:mm:ss" ]
+            timezone => "UTC"
+          }
+        }
+      CONFIG
+
+      before(:each) do
+        logstash_time = Time.utc(2014,1,1,00,30,50)
+        allow(Time).to receive(:now).and_return(logstash_time)
+      end
+
+      sample "Dec 31 23:59:00" do
+        insist { subject["@timestamp"].year } == 2013
+      end
+    end
+
+    describe "fill next year if january events arrive in december" do
+      config <<-CONFIG
+        filter {
+          date {
+            match => [ "message", "MMM dd HH:mm:ss" ]
+            timezone => "UTC"
+          }
+        }
+      CONFIG
+
+      before(:each) do
+        logstash_time = Time.utc(2013,12,31,23,59,50)
+        allow(Time).to receive(:now).and_return(logstash_time)
+      end
+
+      sample "Jan 01 01:00:00" do
+        insist { subject["@timestamp"].year } == 2014
+      end
+    end
+  end
 end

metadata

@@ -1,91 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-date
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 2.1.4
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-09 00:00:00.000000000 Z
+date: 2016-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
-      - !ruby/object:Gem::Version
-        version: 3.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
-      - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '1.0'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-input-generator
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-input-generator
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-codec-json
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-output-null
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-output-null
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -113,12 +107,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []