logstash-filter-xml 2.0.2 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5eeac68d6edb559c401be45588545472d0f7251c
-  data.tar.gz: 31b715339f8138873c3d9d40680fb163c8747589
+  metadata.gz: 0af48f46dcce1e66c87380592f9b490d0c377aae
+  data.tar.gz: 41ebeb840603ae50e82be8ef300d1de47c681a2f
 SHA512:
-  metadata.gz: 76065a921e5de7e614e934849a8e1264b7086c2a64a5d2ff75c233c050b576e5def73ef0b860c25f4731effb83f9ad02e89319f9f1307e60be1a8c88d214a8ae
-  data.tar.gz: 371c85df1aa6ae51f00f172b6056a58e190d7a9b93fcc461c808193faf762578bcc3d1791996267a374145c5a1b9de591b3240378e3460fec6ac6bfaf2ac18bd
+  metadata.gz: 64f80c71aa98dd82c1b846799b23aa5402518c7886059c0d01dde2789fe982076bcebce470dbeb2d7fd869e14dfa530a7185114b5a7532de4715614ef64e50ea
+  data.tar.gz: 03005e58d2645fbfdab2826a962a6ba7fbd9a4f2a8f48bebe822ae4a5faf3c06c57c67e8135a77178ca36fff44f395ff5a0932c013d3cc79fa425b12ab89a716

data/CHANGELOG.md

@@ -1,5 +1,10 @@
+## 2.1.1
+ - Refactored field references, code cleanups
+
+## 2.1.0
+ - Support for namespace declarations to use parsing the XML document
+
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
-

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-xml-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-xml-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/filters/xml.rb

@@ -62,49 +62,73 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
   # field as described above. Setting this to false will prevent that.
   config :store_xml, :validate => :boolean, :default => true
 
+  # By default only namespaces declarations on the root element are considered.
+  # This allows to configure all namespace declarations to parse the XML document.
+  #
+  # Example:
+  #
+  # [source,ruby]
+  # filter {
+  #   xml {
+  #     namespaces => {
+  #       "xsl" => "http://www.w3.org/1999/XSL/Transform"
+  #       "xhtml" => http://www.w3.org/1999/xhtml"
+  #     }
+  #   }
+  # }
+  #
+  config :namespaces, :validate => :hash, :default => {}
+
   # Remove all namespaces from all nodes in the document.
   # Of course, if the document had nodes with the same names but different namespaces, they will now be ambiguous.
   config :remove_namespaces, :validate => :boolean, :default => false
 
-  public
+  XMLPARSEFAILURE_TAG = "_xmlparsefailure"
+
   def register
     require "nokogiri"
     require "xmlsimple"
+  end
 
-  end # def register
-
-  public
   def filter(event)
-    
     matched = false
 
-    @logger.debug("Running xml filter", :event => event)
-
-    return unless event.include?(@source)
+    @logger.debug? && @logger.debug("Running xml filter", :event => event)
 
     value = event[@source]
+    return unless value
 
-    if value.is_a?(Array) && value.length > 1
-      @logger.warn("XML filter only works on fields of length 1",
-                   :source => @source, :value => value)
+    if value.is_a?(Array)
+      if value.length != 1
+        event.tag(XMLPARSEFAILURE_TAG)
+        @logger.warn("XML filter expects single item array", :source => @source, :value => value)
+        return
+      end
+
+      value = value.first
+    end
+
+    unless value.is_a?(String)
+      event.tag(XMLPARSEFAILURE_TAG)
+      @logger.warn("XML filter expects a string but received a #{value.class}", :source => @source, :value => value)
       return
     end
 
     # Do nothing with an empty string.
-    return if value.strip.length == 0
+    return if value.strip.empty?
 
     if @xpath
       begin
         doc = Nokogiri::XML(value, nil, value.encoding.to_s)
       rescue => e
-        event.tag("_xmlparsefailure")
-        @logger.warn("Trouble parsing xml", :source => @source, :value => value,
-                     :exception => e, :backtrace => e.backtrace)
+        event.tag(XMLPARSEFAILURE_TAG)
+        @logger.warn("Error parsing xml", :source => @source, :value => value, :exception => e, :backtrace => e.backtrace)
         return
       end
       doc.remove_namespaces! if @remove_namespaces
+
       @xpath.each do |xpath_src, xpath_dest|
-        nodeset = doc.xpath(xpath_src)
+        nodeset = @namespaces.empty? ? doc.xpath(xpath_src) : doc.xpath(xpath_src, @namespaces)
 
         # If asking xpath for a String, like "name(/*)", we get back a
         # String instead of a NodeSet.  We normalize that here.
@@ -112,32 +136,39 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
 
         normalized_nodeset.each do |value|
           # some XPath functions return empty arrays as string
-          if value.is_a?(Array)
-            return if value.length == 0
-          end
+          # TODO: (colin) the return statement here feels like a bug and should probably be a next ?
+          return if value.is_a?(Array) && value.length == 0
 
-          unless value.nil?
+          if value
             matched = true
-            event[xpath_dest] ||= []
-            event[xpath_dest] << value.to_s
+            # TODO: (colin) this can probably be optimized to avoid the Event get/set at every loop iteration anf
+            # the array should probably be created once, filled in the loop and set at after the loop but the return
+            # statement above screws this strategy and is likely a bug anyway so I will not touch this until I can
+            # deep a big deeper and verify there is a sufficient test harness to refactor this.
+            data = event[xpath_dest] || []
+            data << value.to_s
+            event[xpath_dest] = data
+
+            # do not use the following construct to set the event, we cannot assume anymore that the field values are in-place mutable
+            # event[xpath_dest] ||= []
+            # event[xpath_dest] << value.to_s
           end
-        end # XPath.each
-      end # @xpath.each
-    end # if @xpath
+        end
+      end
+    end
 
     if @store_xml
       begin
         event[@target] = XmlSimple.xml_in(value)
         matched = true
       rescue => e
-        event.tag("_xmlparsefailure")
-        @logger.warn("Trouble parsing xml with XmlSimple", :source => @source,
-                     :value => value, :exception => e, :backtrace => e.backtrace)
+        event.tag(XMLPARSEFAILURE_TAG)
+        @logger.warn("Error parsing xml with XmlSimple", :source => @source, :value => value, :exception => e, :backtrace => e.backtrace)
         return
       end
-    end # if @store_xml
+    end
 
     filter_matched(event) if matched
-    @logger.debug("Event after xml filter", :event => event)
-  end # def filter
-end # class LogStash::Filters::Xml
+    @logger.debug? && @logger.debug("Event after xml filter", :event => event)
+  end
+end

data/logstash-filter-xml.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-xml'
-  s.version         = '2.0.2'
+  s.version         = '2.1.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Takes a field that contains XML and expands it into an actual datastructure."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/xml_spec.rb

@@ -31,6 +31,36 @@ describe LogStash::Filters::Xml do
       insist { subject["data"]} == {"key1" => [{"key2" => ["value"]}]}
     end
 
+    # parse xml in single item array
+    sample("raw" => ["<foo bar=\"baz\"/>"]) do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == {"bar" => "baz"}
+    end
+
+    # fail in multi items array
+    sample("raw" => ["<foo bar=\"baz\"/>", "jojoba"]) do
+      insist { subject["tags"] }.include?("_xmlparsefailure")
+      insist { subject["data"]} == nil
+    end
+
+    # fail in empty array
+    sample("raw" => []) do
+      insist { subject["tags"] }.include?("_xmlparsefailure")
+      insist { subject["data"]} == nil
+    end
+
+    # fail for non string field
+    sample("raw" => {"foo" => "bar"}) do
+      insist { subject["tags"] }.include?("_xmlparsefailure")
+      insist { subject["data"]} == nil
+    end
+
+    # fail for non string single item array
+    sample("raw" => [{"foo" => "bar"}]) do
+      insist { subject["tags"] }.include?("_xmlparsefailure")
+      insist { subject["data"]} == nil
+    end
+
     #From bad xml
     sample("raw" => '<foo /') do
       insist { subject["tags"] }.include?("_xmlparsefailure")
@@ -189,6 +219,42 @@ describe LogStash::Filters::Xml do
     end
   end
 
+  describe "parse including namespaces declarations on root" do
+      config <<-CONFIG
+      filter {
+        xml {
+          source => "xmldata"
+          xpath => [ "/foo/h:div", "xpath_field" ]
+          namespaces => {"h" => "http://www.w3.org/TR/html4/"}
+          remove_namespaces => false
+        }
+      }
+      CONFIG
+
+      # Single value
+      sample("xmldata" => '<foo xmlns:h="http://www.w3.org/TR/html4/"><h:div>Content</h:div></foo>') do
+        insist { subject["xpath_field"] } == ["<h:div>Content</h:div>"]
+      end
+  end
+
+  describe "parse including namespaces declarations on child" do
+      config <<-CONFIG
+      filter {
+        xml {
+          source => "xmldata"
+          xpath => [ "/foo/h:div", "xpath_field" ]
+          namespaces => {"h" => "http://www.w3.org/TR/html4/"}
+          remove_namespaces => false
+        }
+      }
+      CONFIG
+
+      # Single value
+      sample("xmldata" => '<foo><h:div xmlns:h="http://www.w3.org/TR/html4/">Content</h:div></foo>') do
+        insist { subject["xpath_field"] } == ["<h:div xmlns:h=\"http://www.w3.org/TR/html4/\">Content</h:div>"]
+      end
+  end
+
   describe "parse removing namespaces" do
     config <<-CONFIG
     filter {

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-xml
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.1.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2015-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,48 +28,50 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: nokogiri
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: nokogiri
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: xml-simple
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: xml-simple
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []