logstash-filter-urldecode 3.0.2 → 3.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 347c512c13e0c2147de77917102bb96b71dff378
-  data.tar.gz: 7c7e5ada002341e5cf6bfff7dda4fbc41d4409a3
+  metadata.gz: 65a6db3f0aa1e22c3bbea648cbcbdeb8f3d4aa0e
+  data.tar.gz: 91323d88b743f22fbf15d0c29143b1a01b451752
 SHA512:
-  metadata.gz: 6fcd3e4668a93aeb479870a8e18704eb5ab95d701eb6926369309d62b54a64c972708549a84c19720f6eae567c306e7d4d2ba4475e09f22ece8dffe7a67df3e4
-  data.tar.gz: f1834c60803e9643bbea97fb242709f287dd4e504743c533a5d9cb63cf862af8b0630521facf658397fcbce1c666f7737b4b3daaff7056cea7430cbaee65405b
+  metadata.gz: 05c0ef24d921d36df65d7955aefac8eae78f68c5f8214e4fe0c3c5de6112513d21bb3aec2eaf1f93f1839c644c8b96bd2c08ab6636e91505b956c7182bd67611
+  data.tar.gz: 25e8b811a4f03967ac4c488eef8d64f6b5b5ef24cb74d272ff9685de7686cea40473a645adea126a612c26a425296d4508de964f9f35363937da7b38267c7c43

data/CHANGELOG.md

@@ -1,16 +1,23 @@
+## 3.0.3
+  - Fix 6 "incompatible encodings: UTF-8 and ASCII-8BIT"
+
 ## 3.0.2
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 
 ## 3.0.1
   - Republish all the gems under jruby.
+
 ## 3.0.0
   - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
-# 2.0.4
+
+## 2.0.4
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
-# 2.0.3
+
+## 2.0.3
   - New dependency requirements for logstash-core for the 5.0 release
+
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 

data/CONTRIBUTORS

@@ -7,6 +7,7 @@ Contributors:
 * Jordan Sissel (jordansissel)
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
+* Guy Boertje (guyboertje)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/lib/logstash/filters/urldecode.rb

@@ -14,6 +14,9 @@ class LogStash::Filters::Urldecode < LogStash::Filters::Base
   # Urldecode all fields
   config :all_fields, :validate => :boolean, :default => false
 
+  # Append values to the `tags` field when an exception is thrown
+  config :tag_on_failure, :validate => :array, :default => ["_urldecodefailure"]
+
   # Thel character encoding used in this filter. Examples include `UTF-8`
   # and `cp1252`
   #
@@ -21,6 +24,10 @@ class LogStash::Filters::Urldecode < LogStash::Filters::Base
   # or in another character set other than `UTF-8`.
   config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
 
+  UnescapeRe = /((?:%[0-9a-fA-F]{2})+)/
+  UnescapePercent = '%'.freeze
+  UnescapePack = 'H*'.freeze
+
   public
   def register
     @converter = LogStash::Util::Charset.new(@charset)
@@ -29,23 +36,40 @@ class LogStash::Filters::Urldecode < LogStash::Filters::Base
 
   public
   def filter(event)
-    # If all_fields is true then try to decode them all
-    if @all_fields
-      event.to_hash.each { |name, value| event.set(name, urldecode(value)) }
-    # Else decode the specified field
-    else
-      event.set(@field, urldecode(event.get(@field)))
+    begin
+      # If all_fields is true then try to decode them all
+      if @all_fields
+        event.to_hash.each do |name, value|
+          event.set(name, urldecode(value))
+        end
+      # else decode the specified field
+      else
+        event.set(field, urldecode(event.get(@field)))
+      end
+    rescue => e
+      @tag_on_failure.each{|tag| event.tag(tag)}
     end
+
     filter_matched(event)
   end # def filter
 
+  private
+
+  def unescape(value)
+    # There is a bug in the Ruby stdlib URI.unescape
+    # this: URI.unescape("europ%C3%A9enneeuropéenne") throws the following error
+    # Encoding::CompatibilityError: incompatible encodings: ASCII-8BIT and UTF-8
+    # because Array#pack returns ASCII-8BIT encoded strings
+    # this a hybrid of URI.unescape and CGI.unescape
+    value.gsub(UnescapeRe){|s|[s.delete(UnescapePercent)].pack(UnescapePack).force_encoding(value.encoding) }
+  end
+
   # Attempt to handle string, array, and hash values for fields.
   # For all other datatypes, just return, URI.unescape doesn't support them.
-  private
   def urldecode(value)
     case value
     when String
-      escaped = URI.unescape(value)
+      escaped = unescape(value)
       return @converter.convert(escaped)
     when Array
       ret_values = []

data/logstash-filter-urldecode.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-urldecode'
-  s.version         = '3.0.2'
+  s.version         = '3.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The urldecode filter is for decoding fields that are urlencoded."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/urldecode_spec.rb

@@ -15,7 +15,8 @@ describe LogStash::Filters::Urldecode do
     CONFIG
 
     sample("message" => "http%3A%2F%2Flogstash.net%2Fdocs%2F1.3.2%2Ffilters%2Furldecode") do
-      insist { subject.get("message") } == "http://logstash.net/docs/1.3.2/filters/urldecode"
+      expect(subject.get("message")).to eq "http://logstash.net/docs/1.3.2/filters/urldecode"
+      expect(subject.get("tags")).to be_nil
     end
   end
 
@@ -28,7 +29,8 @@ describe LogStash::Filters::Urldecode do
     CONFIG
 
     sample("message" => "http://logstash.net/docs/1.3.2/filters/urldecode") do
-      insist { subject.get("message") } == "http://logstash.net/docs/1.3.2/filters/urldecode"
+      expect(subject.get("message")).to eq "http://logstash.net/docs/1.3.2/filters/urldecode"
+      expect(subject.get("tags")).to be_nil
     end
   end
 
@@ -44,8 +46,9 @@ describe LogStash::Filters::Urldecode do
     CONFIG
 
     sample("message" => "http%3A%2F%2Flogstash.net%2Fdocs%2F1.3.2%2Ffilters%2Furldecode", "nonencoded" => "http://logstash.net/docs/1.3.2/filters/urldecode") do
-      insist { subject.get("message") } == "http://logstash.net/docs/1.3.2/filters/urldecode"
-      insist { subject.get("nonencoded") } == "http://logstash.net/docs/1.3.2/filters/urldecode"
+      expect(subject.get("message")).to eq "http://logstash.net/docs/1.3.2/filters/urldecode"
+      expect(subject.get("nonencoded")).to eq "http://logstash.net/docs/1.3.2/filters/urldecode"
+      expect(subject.get("tags")).to be_nil
     end
   end
 
@@ -56,7 +59,60 @@ describe LogStash::Filters::Urldecode do
       }
      CONFIG
      sample("message" => "/a/sa/search?rgu=0;+%C3%BB%D3%D0%D5%D2%B5%BD=;+%B7%A2%CB%CD=") do
-       insist { subject.get("message") } == "/a/sa/search?rgu=0;+û\\xD3\\xD0\\xD5ҵ\\xBD=;+\\xB7\\xA2\\xCB\\xCD="
+       expect(subject.get("message")).to eq "/a/sa/search?rgu=0;+û\\xD3\\xD0\\xD5ҵ\\xBD=;+\\xB7\\xA2\\xCB\\xCD="
+       expect(subject.get("tags")).to be_nil
+     end
+   end
+
+   describe "urldecode should handle non RFC 3986 compliant strings with encoding in params portion" do
+     config <<-CONFIG
+      filter {
+        urldecode {
+          field => "url"
+        }
+      }
+     CONFIG
+     sample("url" => "/fr/search-results?queryText=Organigramme%20de%20la%20Commission%20europ%C3%A9enne%201998&additionalTextParam=organigramme%20de%20la%20commission%20européenne%201998") do
+       expect(subject.get("url")).to eq "/fr/search-results?queryText=Organigramme de la Commission européenne 1998&additionalTextParam=organigramme de la commission européenne 1998"
+       expect(subject.get("tags")).to be_nil
+     end
+   end
+
+   describe "urldecode should handle non RFC 3986 compliant strings with encoding in url portion" do
+     config <<-CONFIG
+      filter {
+        urldecode {
+          field => "url"
+        }
+      }
+     CONFIG
+     sample("url" => "http%3A%2F%2Fl%C3%B8gstash.net%2Fd%C3%B8cs%2F1.3.2%2Ffilters%2Furldecøde?name=frødø%20båggins") do
+       expect(subject.get("url")).to eq "http://løgstash.net/døcs/1.3.2/filters/urldecøde?name=frødø båggins"
+       expect(subject.get("tags")).to be_nil
+     end
+   end
+
+   describe "urldecode should handle hashes" do
+     config <<-CONFIG
+      filter {
+        urldecode {}
+      }
+     CONFIG
+     sample("message" => {"url" => "http%3A%2F%2Flogstash.net%2Fdocs%2F1.3.2%2Ffilters%2Furldecode"}) do
+       expect(subject.get("[message][url]")).to eq "http://logstash.net/docs/1.3.2/filters/urldecode"
+       expect(subject.get("tags")).to be_nil
+     end
+   end
+
+   describe "urldecode should handle arrays" do
+     config <<-CONFIG
+      filter {
+        urldecode {}
+      }
+     CONFIG
+     sample("message" => ["http%3A%2F%2Flogstash.net%2Fdocs%2F1.3.2%2Ffilters%2Furldecode"]) do
+       expect(subject.get("[message][0]")).to eq "http://logstash.net/docs/1.3.2/filters/urldecode"
+       expect(subject.get("tags")).to be_nil
      end
    end
 end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-urldecode
 version: !ruby/object:Gem::Version
-  version: 3.0.2
+  version: 3.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2017-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core-plugin-api
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
@@ -30,20 +28,22 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -81,7 +81,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: The urldecode filter is for decoding fields that are urlencoded.