logstash-filter-xml 3.0.1 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ffb5fe018cd5b200542dbf05659717098a28a774
-  data.tar.gz: 08e30920d683357ce04971b45baf654ffaaeaeac
+  metadata.gz: 8b31248ae74ec510edd487e3fa3027fd74aa42b5
+  data.tar.gz: a18009abaf99dd29a9f671db974604d8496373ae
 SHA512:
-  metadata.gz: 3503a6edfc5b325008d21993b36a5545004836c258387f756cc33ee75f1c0212726e79b03097c136f333208722611d6d44d76276de13abed094214a077a81b00
-  data.tar.gz: e09bcb380e9a1d04a8d04932a0c268f139a7450a676d26e7cf1839b48afc0a70e1023299ed2c08b5abbf454fefe392ee036e7e01e8fe5fe49f77b06df2c2d26f
+  metadata.gz: 7eef1c44fbfa9b7d8ea09fe1fb077c92a4ebd4dc0614a2e19eb9fd71f14410bc3369385ad78ec51d60242436dc8b5ff8e3c07f9fb34da347077590bb2951c7f2
+  data.tar.gz: b1231cacd1b1502de53383b8c059828925b826f206c66eb2e2d2383d39bb0c235e16e3d0c65ad9623a818d76bf5bd45a01a4f59fe29bae3d2a0728c81e93f1dc

data/CHANGELOG.md

@@ -1,13 +1,24 @@
+## 4.0.0
+  - breaking,config: New configuration `suppress_empty`. Default to true change default behaviour of the plugin in favor of avoiding mapping conflicts when reaching elasticsearch
+  - config: New configuration `force_content`. By default the filter expands attributes differently from content in xml elements.
+    This option allows you to force text content and attributes to always parse to a hash value.
+  - config: Ensure that `target` is set when storing xml content in the event (`store_xml => true`)
+
 ## 3.0.1
   - Republish all the gems under jruby.
+
 ## 3.0.0
   - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
-# 2.1.4
+
+## 2.1.4
   - Added setting to disable forcing single values to be added in arrays. Ref: https://github.com/logstash-plugins/logstash-filter-xml/pull/28.
-# 2.1.3
+
+## 2.1.3
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
-# 2.1.2
+
+## 2.1.2
   - New dependency requirements for logstash-core for the 5.0 release
+
 ## 2.1.1
  - Refactored field references, code cleanups
 

data/lib/logstash/filters/xml.rb

@@ -51,10 +51,10 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
   # destination field as an array. As such, multiple matches across
   # multiple source fields will produce duplicate entries in the field.
   #
-  # More on XPath: http://www.w3schools.com/xpath/
+  # More on XPath: http://www.w3schools.com/xml/xml_xpath.asp
   #
   # The XPath functions are particularly powerful:
-  # http://www.w3schools.com/xpath/xpath_functions.asp
+  # http://www.w3schools.com/xsl/xsl_functions.asp
   #
   config :xpath, :validate => :hash, :default => {}
 
@@ -66,6 +66,11 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
   # false will prevent storing single elements in arrays.
   config :force_array, :validate => :boolean, :default => true
 
+  # By default the filter will expand attributes differently from content inside
+  # of tags. This option allows you to force text content and attributes to always
+  # parse to a hash value.
+  config :force_content, :validate => :boolean, :default => false
+
   # By default only namespaces declarations on the root element are considered.
   # This allows to configure all namespace declarations to parse the XML document.
   #
@@ -87,11 +92,24 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
   # Of course, if the document had nodes with the same names but different namespaces, they will now be ambiguous.
   config :remove_namespaces, :validate => :boolean, :default => false
 
+  # By default, output nothing if the element is empty.
+  # If set to `false`, empty element will result in an empty hash object.
+  config :suppress_empty, :validate => :boolean, :default => true
+
   XMLPARSEFAILURE_TAG = "_xmlparsefailure"
 
   def register
     require "nokogiri"
     require "xmlsimple"
+
+    if @store_xml && (!@target || @target.empty?)
+      raise LogStash::ConfigurationError, I18n.t(
+        "logstash.agent.configuration.invalid_plugin_register",
+        :plugin => "filter",
+        :type => "xml",
+        :error => "When the 'store_xml' configuration option is true, 'target' must also be set"
+      )
+    end
   end
 
   def filter(event)
@@ -159,7 +177,9 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
 
     if @store_xml
       begin
-        event.set(@target, XmlSimple.xml_in(value, "ForceArray" => @force_array))
+        xml_options = {"ForceArray" => @force_array, "ForceContent" => @force_content}
+        xml_options["SuppressEmpty"] = true if @suppress_empty
+        event.set(@target, XmlSimple.xml_in(value, xml_options))
         matched = true
       rescue => e
         event.tag(XMLPARSEFAILURE_TAG)

data/logstash-filter-xml.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-xml'
-  s.version         = '3.0.1'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Takes a field that contains XML and expands it into an actual datastructure."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/xml_spec.rb

@@ -209,6 +209,7 @@ describe LogStash::Filters::Xml do
         source => "xmldata"
         xpath => [ "/foo/h:div", "xpath_field" ]
         remove_namespaces => false
+        store_xml => false
       }
     }
     CONFIG
@@ -227,6 +228,7 @@ describe LogStash::Filters::Xml do
           xpath => [ "/foo/h:div", "xpath_field" ]
           namespaces => {"h" => "http://www.w3.org/TR/html4/"}
           remove_namespaces => false
+          store_xml => false
         }
       }
       CONFIG
@@ -245,6 +247,7 @@ describe LogStash::Filters::Xml do
           xpath => [ "/foo/h:div", "xpath_field" ]
           namespaces => {"h" => "http://www.w3.org/TR/html4/"}
           remove_namespaces => false
+          store_xml => false
         }
       }
       CONFIG
@@ -262,6 +265,7 @@ describe LogStash::Filters::Xml do
         source => "xmldata"
         xpath => [ "/foo/div", "xpath_field" ]
         remove_namespaces => true
+        store_xml => false
       }
     }
     CONFIG
@@ -305,4 +309,92 @@ describe LogStash::Filters::Xml do
       insist { subject.get("parseddata") } == { "bar" => "Content" }
     end
   end
+
+  context "Using suppress_empty option" do
+    describe "suppress_empty => false" do
+      config <<-CONFIG
+      filter {
+        xml {
+          source => "xmldata"
+          target => "data"
+          suppress_empty => false
+        }
+      }
+      CONFIG
+
+      sample("xmldata" => '<foo><key>value1</key><key></key></foo>') do
+        insist { subject.get("tags") }.nil?
+        insist { subject.get("data") } == {"key" => ["value1", {}]}
+      end
+    end
+
+    describe "suppress_empty => true" do
+      config <<-CONFIG
+      filter {
+        xml {
+          source => "xmldata"
+          target => "data"
+          suppress_empty => true
+        }
+      }
+      CONFIG
+
+      sample("xmldata" => '<foo><key>value1</key><key></key></foo>') do
+        insist { subject.get("tags") }.nil?
+        insist { subject.get("data") } == {"key" => ["value1"]}
+      end
+    end
+  end
+
+  context "Using force content option" do
+    describe "force_content => false" do
+      config <<-CONFIG
+      filter {
+        xml {
+          source => "xmldata"
+          target => "data"
+          force_array => false
+          force_content => false
+        }
+      }
+      CONFIG
+
+      sample("xmldata" => '<opt><x>text1</x><y a="2">text2</y></opt>') do
+        insist { subject.get("tags") }.nil?
+        insist { subject.get("data") } ==  { 'x' => 'text1', 'y' => { 'a' => '2', 'content' => 'text2' } }
+      end
+    end
+    describe "force_content => true" do
+      config <<-CONFIG
+      filter {
+        xml {
+          source => "xmldata"
+          target => "data"
+          force_array => false
+          force_content => true
+        }
+      }
+      CONFIG
+
+      sample("xmldata" => '<opt><x>text1</x><y a="2">text2</y></opt>') do
+        insist { subject.get("tags") }.nil?
+        insist { subject.get("data") } ==  { 'x' => { 'content' => 'text1' }, 'y' => { 'a' => '2', 'content' => 'text2' } }
+      end
+    end
+  end
+
+  describe "plugin registration" do
+    config <<-CONFIG
+    filter {
+      xml {
+        xmldata => "message"
+        store_xml => true
+      }
+    }
+    CONFIG
+
+    sample("xmldata" => "<foo>random message</foo>") do
+      insist { subject }.raises(LogStash::ConfigurationError)
+    end
+  end
 end

metadata

@@ -1,71 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-xml
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-09 00:00:00.000000000 Z
+date: 2016-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '2.0'
-  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: nokogiri
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '2.0'
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: nokogiri
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: xml-simple
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: xml-simple
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -93,12 +93,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []