logstash-filter-date 0.1.6 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 914517782eea0d83b0472895700d694161e28216
-  data.tar.gz: e3095af8f12f909189936456897a37e504cec253
+  metadata.gz: 6a7783bc9a63927c80f641cbf1a6881424970d7f
+  data.tar.gz: d4225b591a3ec9ad1ee87fd10990a7baf8e6c79e
 SHA512:
-  metadata.gz: e060eb3ae0e61704dcf39e90c8a562bff24bcd31ee5a94ede455874f136486ac7d3c1404f7bf62eed2b0f4762ef3aeb092f263e5879a9ca2de47419a8417bba6
-  data.tar.gz: f156593255ddb4df75666df538a8ce0c48e4f023b44bc8c3ab70ff49072c1ebde99faf4faf9d86de8f41898d44765eb4be23ad76087d75ba5ee0efe4ccaa2d44
+  metadata.gz: a9a6f7fb0fce3faa06e9fcd541823d680ad0471abf009b99f66778d78d9b1b28cd07f47d6b2e7396fe870604f2ed28a3114818b68d673ca61c192c444113f554
+  data.tar.gz: 7a340f155599ede6dd2c50743fa98bd13afb6c6e5023afb891bbb61f8e2e4ee7156ff045bcc72ef76c3f1a458edd30f74a20d8fa0ee34c7bfc4a6c73fa35801d

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -13,7 +13,7 @@ Logstash provides infrastructure to automatically generate documentation for thi
 
 ## Need Help?
 
-Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
 
 ## Developing
 

data/lib/logstash/filters/date.rb

@@ -39,11 +39,12 @@ class LogStash::Filters::Date < LogStash::Filters::Base
 
   # Specify a locale to be used for date parsing using either IETF-BCP47 or POSIX language tag.
   # Simple examples are `en`,`en-US` for BCP47 or `en_US` for POSIX.
-  # If not specified, the platform default will be used.
   #
   # The locale is mostly necessary to be set for parsing month names (pattern with `MMM`) and
   # weekday names (pattern with `EEE`).
   #
+  # If not specified, the platform default will be used but for non-english platform default
+  # an english parser will also be used as a fallback mechanism.
   config :locale, :validate => :string
 
   # The date formats allowed are anything allowed by Joda-Time (java time
@@ -88,6 +89,10 @@ class LogStash::Filters::Date < LogStash::Filters::Base
   # default to updating the `@timestamp` field of the event.
   config :target, :validate => :string, :default => "@timestamp"
 
+  # Append values to the `tags` field when there has been no
+  # successful match
+  config :tag_on_failure, :validate => :array, :default => ["_dateparsefailure"]
+
   # LOGSTASH-34
   DATEPATTERNS = %w{ y d H m s S }
 
@@ -159,16 +164,30 @@ class LogStash::Filters::Date < LogStash::Filters::Base
             return (date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000)
           end
         else
-          joda_parser = org.joda.time.format.DateTimeFormat.forPattern(format).withDefaultYear(Time.new.year)
-          if @timezone
-            joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
-          else
-            joda_parser = joda_parser.withOffsetParsed
-          end
-          if (locale != nil)
-            joda_parser = joda_parser.withLocale(locale)
+          begin
+            joda_parser = org.joda.time.format.DateTimeFormat.forPattern(format).withDefaultYear(Time.new.year)
+            if @timezone
+              joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
+            else
+              joda_parser = joda_parser.withOffsetParsed
+            end
+            if locale
+              joda_parser = joda_parser.withLocale(locale)
+            end
+            parsers << lambda { |date| joda_parser.parseMillis(date) }
+
+            #Include a fallback parser to english when default locale is non-english
+            if !locale &&
+              "en" != java.util.Locale.getDefault().getLanguage() &&
+              (format.include?("MMM") || format.include?("E"))
+              en_joda_parser = joda_parser.withLocale(java.util.Locale.forLanguageTag('en-US'))
+              parsers << lambda { |date| en_joda_parser.parseMillis(date) }
+            end
+          rescue JavaException => e
+            raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",
+              :plugin => "filter", :type => "date",
+              :error => "#{e.message} for pattern '#{format}'")
           end
-          parsers << lambda { |date| joda_parser.parseMillis(date) }
       end
 
       @logger.debug("Adding type with date config", :type => @type,
@@ -221,11 +240,16 @@ class LogStash::Filters::Date < LogStash::Filters::Base
           filter_matched(event)
         rescue StandardError, JavaException => e
           @logger.warn("Failed parsing date from field", :field => field,
-                       :value => value, :exception => e)
-          # Raising here will bubble all the way up and cause an exit.
-          # TODO(sissel): Maybe we shouldn't raise?
-          # TODO(sissel): What do we do on a failure? Tag it like grok does?
-          #raise e
+                       :value => value, :exception => e.message,
+                       :config_parsers => fieldparsers.collect {|x| x[:format]}.join(','),
+                       :config_locale => @locale ? @locale : "default="+java.util.Locale.getDefault().toString()
+                       )
+          # Tag this event if we can't parse it. We can use this later to
+          # reparse+reindex logs if we improve the patterns given.
+          @tag_on_failure.each do |tag|
+            event["tags"] ||= []
+            event["tags"] << tag unless event["tags"].include?(tag)
+          end
         end # begin
       end # fieldvalue.each
     end # @parsers.each

data/logstash-filter-date.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-date'
-  s.version         = '0.1.6'
+  s.version         = '1.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/date_spec.rb

@@ -313,7 +313,23 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
 
     sample("thedate" => "2013/Apr/21") do
       insist { subject["@timestamp"] } != "2013-04-21T00:00:00.000Z"
-      insist { subject["tags"] } == nil
+      reject { subject["tags"] }.include? "tagged"
+    end
+  end
+
+  describe "failing to parse should apply tag_on_failure" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "thedate", "yyyy/MM/dd" ]
+          tag_on_failure => ["date_failed"]
+        }
+      }
+    CONFIG
+
+    sample("thedate" => "2013/Apr/21") do
+      insist { subject["@timestamp"] } != "2013-04-21T00:00:00.000Z"
+      insist { subject["tags"] }.include? "date_failed"
     end
   end
 
@@ -419,4 +435,24 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
       insist { subject["@timestamp"].time } == Time.iso8601("2013-03-25T20:33:56.000Z")
     end
   end
+
+  describe "Support fallback to english for non-english default locale" do
+    default_locale = java.util.Locale.getDefault()
+    #Override default locale with non-english
+    java.util.Locale.setDefault(java.util.Locale.forLanguageTag('fr-FR'))
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "message", "dd MMMM yyyy" ]
+          timezone => "UTC"
+        }
+      }
+    CONFIG
+
+    sample "01 September 2014" do
+      insist { subject["@timestamp"].time } == Time.iso8601("2014-09-01T00:00:00.000Z").utc
+    end
+    #Restore default locale
+    java.util.Locale.setDefault(default_locale)
+  end
 end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-date
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 1.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-20 00:00:00.000000000 Z
+date: 2015-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,62 +28,64 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: logstash-input-generator
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-input-generator
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-json
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-output-null
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-output-null
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -93,9 +93,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
+- NOTICE.TXT
 - README.md
 - Rakefile
 - lib/logstash/filters/date.rb
@@ -123,7 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.2.2
 signing_key:
 specification_version: 4
 summary: The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event.