logstash-filter-xml 2.2.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c54641de27d0a2d3d1521cf95ca13053cf79fdf
-  data.tar.gz: 45fa43b4e3e0e8d55de05be69ab68c8b2e63b8e0
+  metadata.gz: 1840e844c5ece350a424e0cebe3277defc8c5e8b
+  data.tar.gz: 61fb53f8feba4439c651c834fa64393191c2c4a6
 SHA512:
-  metadata.gz: e43cd5a04b877149db47c036303f1a74cfef0e6f3033c5994409f2d98edd957dcd9dcefff64e679bee7f134aa1e737c37c5b39adc21143223807e78f2580cec0
-  data.tar.gz: a54963d0704933285befe3b70ef4ec35fd921808cea3c631d5900cb0a6934fcc8c48e863f3d722eeccff1d82027289a4b837dd3054d0061698bbb25d1ed1af0c
+  metadata.gz: 4130e7435dc522d2c1d92d598eb6284c051b5ffad47f147281c679cdc49fff2b28568c5d02e36b84bec2e0633a06f051c62709cf849c83c3efca0cff88687d62
+  data.tar.gz: 75e3d58462c5f088c164c55b5fc2fd9fad38f6543974796b65bbd58c97b8346082213886632699cc46460f94d00fdd8ff8f13cf1e20062345c0a9b11c39966c2

data/CHANGELOG.md

@@ -1,19 +1,11 @@
-## 2.2.0
- - config: New configuration `suppress_empty`. By default the filter creates empty hash from empty xml elements (`suppress_empty => false`). 
-   This can now be configured, `supress_empty => true` will not create event fields from empty xml elements.
- - config: New configuration `force_content`. By default the filter expands attributes differently from content in xml elements.
-   This option allows you to force text content and attributes to always parse to a hash value.
- - config: Ensure that `target` is set when storing xml content in the event (`store_xml => true`)
-
-## 2.1.4
+## 3.0.0
+  - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
+# 2.1.4
   - Added setting to disable forcing single values to be added in arrays. Ref: https://github.com/logstash-plugins/logstash-filter-xml/pull/28.
-
-## 2.1.3
+# 2.1.3
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
-
-## 2.1.2
+# 2.1.2
   - New dependency requirements for logstash-core for the 5.0 release
-
 ## 2.1.1
  - Refactored field references, code cleanups
 

data/Gemfile

@@ -1,2 +1,4 @@
 source 'https://rubygems.org'
-gemspec
+
+# Specify your gem's dependencies in logstash-mass_effect.gemspec
+gemspec

data/lib/logstash/filters/xml.rb

@@ -51,10 +51,10 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
   # destination field as an array. As such, multiple matches across
   # multiple source fields will produce duplicate entries in the field.
   #
-  # More on XPath: http://www.w3schools.com/xml/xml_xpath.asp
+  # More on XPath: http://www.w3schools.com/xpath/
   #
   # The XPath functions are particularly powerful:
-  # http://www.w3schools.com/xsl/xsl_functions.asp
+  # http://www.w3schools.com/xpath/xpath_functions.asp
   #
   config :xpath, :validate => :hash, :default => {}
 
@@ -66,11 +66,6 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
   # false will prevent storing single elements in arrays.
   config :force_array, :validate => :boolean, :default => true
 
-  # By default the filter will expand attributes differently from content inside
-  # of tags. This option allows you to force text content and attributes to always
-  # parse to a hash value.
-  config :force_content, :validate => :boolean, :default => false
-
   # By default only namespaces declarations on the root element are considered.
   # This allows to configure all namespace declarations to parse the XML document.
   #
@@ -92,24 +87,11 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
   # Of course, if the document had nodes with the same names but different namespaces, they will now be ambiguous.
   config :remove_namespaces, :validate => :boolean, :default => false
 
-  # By default, empty element will result in an empty hash object
-  # If set to `true`, output nothing if the element is empty..
-  config :suppress_empty, :validate => :boolean, :default => false
-
   XMLPARSEFAILURE_TAG = "_xmlparsefailure"
 
   def register
     require "nokogiri"
     require "xmlsimple"
-
-    if @store_xml && (!@target || @target.empty?)
-      raise LogStash::ConfigurationError, I18n.t(
-        "logstash.agent.configuration.invalid_plugin_register",
-        :plugin => "filter",
-        :type => "xml",
-        :error => "When the 'store_xml' configuration option is true, 'target' must also be set"
-      )
-    end
   end
 
   def filter(event)
@@ -117,7 +99,7 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
 
     @logger.debug? && @logger.debug("Running xml filter", :event => event)
 
-    value = event[@source]
+    value = event.get(@source)
     return unless value
 
     if value.is_a?(Array)
@@ -167,13 +149,9 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
             # the array should probably be created once, filled in the loop and set at after the loop but the return
             # statement above screws this strategy and is likely a bug anyway so I will not touch this until I can
             # deep a big deeper and verify there is a sufficient test harness to refactor this.
-            data = event[xpath_dest] || []
+            data = event.get(xpath_dest) || []
             data << value.to_s
-            event[xpath_dest] = data
-
-            # do not use the following construct to set the event, we cannot assume anymore that the field values are in-place mutable
-            # event[xpath_dest] ||= []
-            # event[xpath_dest] << value.to_s
+            event.set(xpath_dest, data)
           end
         end
       end
@@ -181,9 +159,7 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
 
     if @store_xml
       begin
-        xml_options = {"ForceArray" => @force_array, "ForceContent" => @force_content}
-        xml_options["SuppressEmpty"] = true if @suppress_empty
-        event[@target] = XmlSimple.xml_in(value, xml_options)
+        event.set(@target, XmlSimple.xml_in(value, "ForceArray" => @force_array))
         matched = true
       rescue => e
         event.tag(XMLPARSEFAILURE_TAG)

data/logstash-filter-xml.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-xml'
-  s.version         = '2.2.0'
+  s.version         = '3.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Takes a field that contains XML and expands it into an actual datastructure."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
   s.add_runtime_dependency 'nokogiri'
   s.add_runtime_dependency 'xml-simple'
 

data/spec/filters/xml_spec.rb

@@ -15,55 +15,55 @@ describe LogStash::Filters::Xml do
     CONFIG
 
     sample("raw" => '<foo key="value"/>') do
-      insist { subject["tags"] }.nil?
-      insist { subject["data"]} == {"key" => "value"}
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("data")} == {"key" => "value"}
     end
 
     #From parse xml with array as a value
     sample("raw" => '<foo><key>value1</key><key>value2</key></foo>') do
-      insist { subject["tags"] }.nil?
-      insist { subject["data"]} == {"key" => ["value1", "value2"]}
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("data")} == {"key" => ["value1", "value2"]}
     end
 
     #From parse xml with hash as a value
     sample("raw" => '<foo><key1><key2>value</key2></key1></foo>') do
-      insist { subject["tags"] }.nil?
-      insist { subject["data"]} == {"key1" => [{"key2" => ["value"]}]}
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("data")} == {"key1" => [{"key2" => ["value"]}]}
     end
 
     # parse xml in single item array
     sample("raw" => ["<foo bar=\"baz\"/>"]) do
-      insist { subject["tags"] }.nil?
-      insist { subject["data"]} == {"bar" => "baz"}
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("data")} == {"bar" => "baz"}
     end
 
     # fail in multi items array
     sample("raw" => ["<foo bar=\"baz\"/>", "jojoba"]) do
-      insist { subject["tags"] }.include?("_xmlparsefailure")
-      insist { subject["data"]} == nil
+      insist { subject.get("tags") }.include?("_xmlparsefailure")
+      insist { subject.get("data")} == nil
     end
 
     # fail in empty array
     sample("raw" => []) do
-      insist { subject["tags"] }.include?("_xmlparsefailure")
-      insist { subject["data"]} == nil
+      insist { subject.get("tags") }.include?("_xmlparsefailure")
+      insist { subject.get("data")} == nil
     end
 
     # fail for non string field
     sample("raw" => {"foo" => "bar"}) do
-      insist { subject["tags"] }.include?("_xmlparsefailure")
-      insist { subject["data"]} == nil
+      insist { subject.get("tags") }.include?("_xmlparsefailure")
+      insist { subject.get("data")} == nil
     end
 
     # fail for non string single item array
     sample("raw" => [{"foo" => "bar"}]) do
-      insist { subject["tags"] }.include?("_xmlparsefailure")
-      insist { subject["data"]} == nil
+      insist { subject.get("tags") }.include?("_xmlparsefailure")
+      insist { subject.get("data")} == nil
     end
 
     #From bad xml
     sample("raw" => '<foo /') do
-      insist { subject["tags"] }.include?("_xmlparsefailure")
+      insist { subject.get("tags") }.include?("_xmlparsefailure")
     end
   end
 
@@ -79,8 +79,8 @@ describe LogStash::Filters::Xml do
     CONFIG
 
     sample("raw" => '<foo key="value"/>') do
-      insist { subject["tags"] }.nil?
-      insist { subject["data"]} == nil
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("data")} == nil
     end
   end
 
@@ -97,14 +97,14 @@ describe LogStash::Filters::Xml do
 
     # Single value
     sample("raw" => '<foo><key>value</key></foo>') do
-      insist { subject["tags"] }.nil?
-      insist { subject["xpath_field"]} == ["value"]
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("xpath_field")} == ["value"]
     end
 
     #Multiple values
     sample("raw" => '<foo><key>value1</key><key>value2</key></foo>') do
-      insist { subject["tags"] }.nil?
-      insist { subject["xpath_field"]} == ["value1","value2"]
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("xpath_field")} == ["value1","value2"]
     end
   end
 
@@ -121,25 +121,25 @@ describe LogStash::Filters::Xml do
     CONFIG
 
     sample("xmldata" => '<foo key="value"/>') do
-      insist { subject["tags"] }.nil?
-      insist { subject["data"]} == {"key" => "value"}
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("data") } == {"key" => "value"}
     end
 
     #From parse xml with array as a value
     sample("xmldata" => '<foo><key>value1</key><key>value2</key></foo>') do
-      insist { subject["tags"] }.nil?
-      insist { subject["data"]} == {"key" => ["value1", "value2"]}
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("data") } == {"key" => ["value1", "value2"]}
     end
 
     #From parse xml with hash as a value
     sample("xmldata" => '<foo><key1><key2>value</key2></key1></foo>') do
-      insist { subject["tags"] }.nil?
-      insist { subject["data"]} == {"key1" => [{"key2" => ["value"]}]}
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("data") } == {"key1" => [{"key2" => ["value"]}]}
     end
 
     #From bad xml
     sample("xmldata" => '<foo /') do
-      insist { subject["tags"] }.include?("_xmlparsefailure")
+      insist { subject.get("tags") }.include?("_xmlparsefailure")
     end
   end
 
@@ -155,8 +155,8 @@ describe LogStash::Filters::Xml do
     CONFIG
 
     sample("xmldata" => '<foo key="value"/>') do
-      insist { subject["tags"] }.nil?
-      insist { subject["data"]} == nil
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("data")} == nil
     end
   end
 
@@ -173,14 +173,14 @@ describe LogStash::Filters::Xml do
 
     # Single value
     sample("xmldata" => '<foo><key>value</key></foo>') do
-      insist { subject["tags"] }.nil?
-      insist { subject["xpath_field"]} == ["value"]
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("xpath_field") } == ["value"]
     end
 
     #Multiple values
     sample("xmldata" => '<foo><key>value1</key><key>value2</key></foo>') do
-      insist { subject["tags"] }.nil?
-      insist { subject["xpath_field"]} == ["value1","value2"]
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("xpath_field") } == ["value1","value2"]
     end
   end
 
@@ -197,8 +197,8 @@ describe LogStash::Filters::Xml do
 
     # Single value
     sample("xmldata" => '<foo><key>Français</key></foo>') do
-      insist { subject["tags"] }.nil?
-      insist { subject["xpath_field"]} == ["Français"]
+      insist { subject.get("tags") }.nil?
+      insist { subject.get("xpath_field")} == ["Français"]
     end
   end
 
@@ -209,14 +209,13 @@ describe LogStash::Filters::Xml do
         source => "xmldata"
         xpath => [ "/foo/h:div", "xpath_field" ]
         remove_namespaces => false
-        store_xml => false
       }
     }
     CONFIG
 
     # Single value
     sample("xmldata" => '<foo xmlns:h="http://www.w3.org/TR/html4/"><h:div>Content</h:div></foo>') do
-      insist { subject["xpath_field"] } == ["<h:div>Content</h:div>"]
+      insist { subject.get("xpath_field") } == ["<h:div>Content</h:div>"]
     end
   end
 
@@ -228,14 +227,13 @@ describe LogStash::Filters::Xml do
           xpath => [ "/foo/h:div", "xpath_field" ]
           namespaces => {"h" => "http://www.w3.org/TR/html4/"}
           remove_namespaces => false
-          store_xml => false
         }
       }
       CONFIG
 
       # Single value
       sample("xmldata" => '<foo xmlns:h="http://www.w3.org/TR/html4/"><h:div>Content</h:div></foo>') do
-        insist { subject["xpath_field"] } == ["<h:div>Content</h:div>"]
+        insist { subject.get("xpath_field") } == ["<h:div>Content</h:div>"]
       end
   end
 
@@ -247,14 +245,13 @@ describe LogStash::Filters::Xml do
           xpath => [ "/foo/h:div", "xpath_field" ]
           namespaces => {"h" => "http://www.w3.org/TR/html4/"}
           remove_namespaces => false
-          store_xml => false
         }
       }
       CONFIG
 
       # Single value
       sample("xmldata" => '<foo><h:div xmlns:h="http://www.w3.org/TR/html4/">Content</h:div></foo>') do
-        insist { subject["xpath_field"] } == ["<h:div xmlns:h=\"http://www.w3.org/TR/html4/\">Content</h:div>"]
+        insist { subject.get("xpath_field") } == ["<h:div xmlns:h=\"http://www.w3.org/TR/html4/\">Content</h:div>"]
       end
   end
 
@@ -265,14 +262,13 @@ describe LogStash::Filters::Xml do
         source => "xmldata"
         xpath => [ "/foo/div", "xpath_field" ]
         remove_namespaces => true
-        store_xml => false
       }
     }
     CONFIG
 
     # Single value
     sample("xmldata" => '<foo xmlns:h="http://www.w3.org/TR/html4/"><h:div>Content</h:div></foo>') do
-      insist { subject["xpath_field"] } == ["<div>Content</div>"]
+      insist { subject.get("xpath_field") } == ["<div>Content</div>"]
     end
   end
 
@@ -289,7 +285,7 @@ describe LogStash::Filters::Xml do
 
     # Single value
     sample("xmldata" => '<foo><bar>Content</bar></foo>') do
-      insist { subject["parseddata"] } == { "bar" => ["Content"] }
+      insist { subject.get("parseddata") } == { "bar" => ["Content"] }
     end
   end
 
@@ -306,96 +302,7 @@ describe LogStash::Filters::Xml do
 
     # Single value
     sample("xmldata" => '<foo><bar>Content</bar></foo>') do
-      insist { subject["parseddata"] } == { "bar" => "Content" }
-    end
-  end
-
-  context "Using suppress_empty option" do
-    describe "suppress_empty => false" do
-      config <<-CONFIG
-      filter {
-        xml {
-          source => "xmldata"
-          target => "data"
-          suppress_empty => false
-        }
-      }
-      CONFIG
-
-      sample("xmldata" => '<foo><key>value1</key><key></key></foo>') do
-        insist { subject["tags"] }.nil?
-        insist { subject["data"] } == {"key" => ["value1", {}]}
-      end
-    end
-
-    describe "suppress_empty => true" do
-      config <<-CONFIG
-      filter {
-        xml {
-          source => "xmldata"
-          target => "data"
-          suppress_empty => true
-        }
-      }
-      CONFIG
-
-      sample("xmldata" => '<foo><key>value1</key><key></key></foo>') do
-        insist { subject["tags"] }.nil?
-        insist { subject["data"] } == {"key" => ["value1"]}
-      end
-    end
-  end
-
-  context "Using force content option" do
-    describe "force_content => false" do
-      config <<-CONFIG
-      filter {
-        xml {
-          source => "xmldata"
-          target => "data"
-          force_array => false
-          force_content => false
-        }
-      }
-      CONFIG
-
-      sample("xmldata" => '<opt><x>text1</x><y a="2">text2</y></opt>') do
-        insist { subject["tags"] }.nil?
-        insist { subject["data"] } ==  { 'x' => 'text1', 'y' => { 'a' => '2', 'content' => 'text2' } }
-      end
-    end
-    describe "force_content => true" do
-      config <<-CONFIG
-      filter {
-        xml {
-          source => "xmldata"
-          target => "data"
-          force_array => false
-          force_content => true
-        }
-      }
-      CONFIG
-
-      sample("xmldata" => '<opt><x>text1</x><y a="2">text2</y></opt>') do
-        insist { subject["tags"] }.nil?
-        insist { subject["data"] } ==  { 'x' => { 'content' => 'text1' }, 'y' => { 'a' => '2', 'content' => 'text2' } }
-      end
-    end
-  end
-
-  describe "plugin registration" do
-    config <<-CONFIG
-    filter {
-      xml {
-        xmldata => "message"
-        store_xml => true
-      }
-    }
-    CONFIG
-
-    sample("xmldata" => "<foo>random message</foo>") do
-      insist { subject }.raises(LogStash::ConfigurationError)
+      insist { subject.get("parseddata") } == { "bar" => "Content" }
     end
   end
-
 end

metadata

@@ -1,72 +1,74 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-xml
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 3.0.0
 platform: ruby
 authors:
 - Elastic
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-24 00:00:00.000000000 Z
+date: 2016-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
-  prerelease: false
+        version: '2.0'
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: nokogiri
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: xml-simple
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: xml-simple
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -87,24 +89,24 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: filter
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.4.8
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
 specification_version: 4
 summary: Takes a field that contains XML and expands it into an actual datastructure.
 test_files: