logstash-filter-xml 4.0.7 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5752546d28d84f25aba5b8938f3554668455b2ba53a20bb2d9d555a32339bc21
-  data.tar.gz: 6f35d6fcf1915907399f5c3254a4f555ded7e3e8e80e3118691ea4aba1d3b922
+  metadata.gz: e533534aa762b35aa38226b37adfda6d03678657070439da36c3bd639a4947a5
+  data.tar.gz: f8010acbb8c698a6809748562301d1dc74d90396a5fc7a32cc4f23005201e8a5
 SHA512:
-  metadata.gz: b567fab011de36e7976dc9939b3d4ae06933d2c5ff56a54c95794c8d25a0f81e29d6aa2b22f459203dd9f9f3f5c30b9e5d750d96aec047f2ca990c66c5d553ea
-  data.tar.gz: 3e5b8fedd5ee99177cf1462487100572aa8a4620985ba89bdf263e0d836b74381735bb919c9892d66d7f09f4323317c85f1e69f7f7a7597770799048cd1059a2
+  metadata.gz: e5dd55d1f420614894a97eb5e4848e1ef9302cb5879edad959d96c2b79855fa22928d62539585731451bc4da6771e2c97d8ff8bec7f157a0639f0f98afba532a
+  data.tar.gz: 8af5b8843765191ddce68ee5fcd79d55d201ecfedf404612873ca462c0a4d83eac376acbec6c4a3eb52a309dfa0d1095279b7bdfe50021b5d060702192b7c76d

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.1.0
+  - Feat: added parser_options for more control over XML parsing [#68](https://github.com/logstash-plugins/logstash-filter-xml/pull/68)
+
 ## 4.0.7
   - Fixed creation of empty arrays when xpath failed [#59](https://github.com/logstash-plugins/logstash-filter-xml/pull/59)
 

data/docs/index.asciidoc

@@ -34,6 +34,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-force_array>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-force_content>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-namespaces>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-parser_options>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-remove_namespaces>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-source>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-store_xml>> |<<boolean,boolean>>|No
@@ -87,6 +88,17 @@ filter {
   }
 }
 
+[id="plugins-{type}s-{plugin}-parser_options"]
+===== `parser_options`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Setting XML parser options allows for more control of the parsing process.
+By default the parser is not strict and thus accepts some invalid content.
+Currently supported options are:
+
+ - `strict` - forces the parser to fail early instead of accumulating errors when content is not valid xml.
 
 [id="plugins-{type}s-{plugin}-remove_namespaces"]
 ===== `remove_namespaces` 

data/lib/logstash/filters/xml.rb

@@ -58,6 +58,13 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
   #
   config :xpath, :validate => :hash, :default => {}
 
+  # Supported XML parsing options are 'strict', 'no_error' and 'no_warning'.
+  # - strict mode turns on strict parsing rules (non-compliant xml will fail)
+  # - no_error and no_warning can be used to suppress errors/warnings
+  config :parse_options, :validate => :string
+  # NOTE: technically we support more but we purposefully do not document those.
+  # e.g. setting "strict|recover" will not turn on strict as they're conflicting
+
   # By default the filter will store the whole parsed XML in the destination
   # field as described above. Setting this to false will prevent that.
   config :store_xml, :validate => :boolean, :default => true
@@ -110,6 +117,7 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
         :error => "When the 'store_xml' configuration option is true, 'target' must also be set"
       )
     end
+    xml_parse_options # validates parse_options => ...
   end
 
   def filter(event)
@@ -141,11 +149,13 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
 
     if @xpath
       begin
-        doc = Nokogiri::XML(value, nil, value.encoding.to_s)
+        doc = Nokogiri::XML::Document.parse(value, nil, value.encoding.to_s, xml_parse_options)
       rescue => e
         event.tag(XMLPARSEFAILURE_TAG)
         @logger.warn("Error parsing xml", :source => @source, :value => value, :exception => e, :backtrace => e.backtrace)
         return
+      else
+        doc.errors.any? && @logger.debug? && @logger.debug("Parsed xml with #{doc.errors.size} errors")
       end
       doc.remove_namespaces! if @remove_namespaces
 
@@ -194,4 +204,26 @@ class LogStash::Filters::Xml < LogStash::Filters::Base
     filter_matched(event) if matched
     @logger.debug? && @logger.debug("Event after xml filter", :event => event)
   end
+
+  private
+
+  def xml_parse_options
+    return Nokogiri::XML::ParseOptions::DEFAULT_XML unless @parse_options # (RECOVER | NONET)
+    @xml_parse_options ||= begin
+      parse_options = @parse_options.split(/,|\|/).map do |opt|
+        name = opt.strip.tr('_', '').upcase
+        if name.empty?
+          nil
+        else
+          begin
+            Nokogiri::XML::ParseOptions.const_get(name)
+          rescue NameError
+            raise LogStash::ConfigurationError, "unsupported parse option: #{opt.inspect}"
+          end
+        end
+      end
+      parse_options.compact.inject(0, :|) # e.g. NOERROR | NOWARNING
+    end
+  end
+
 end

data/logstash-filter-xml.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-xml'
-  s.version         = '4.0.7'
+  s.version         = '4.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parses XML into fields"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/xml_spec.rb

@@ -418,4 +418,63 @@ describe LogStash::Filters::Xml do
       end
     end
   end
+
+  describe "parsing invalid xml" do
+    subject { described_class.new(options) }
+    let(:options) { ({ 'source' => 'xmldata', 'store_xml' => false }) }
+    let(:xmldata) { "<xml> <sample attr='foo' attr=\"bar\"> <invalid> </sample> </xml>" }
+    let(:event) { LogStash::Event.new(data) }
+    let(:data) { { "xmldata" => xmldata } }
+
+    before { subject.register }
+    after { subject.close }
+
+    it 'does not fail (by default)' do
+      subject.filter(event)
+      expect( event.get("tags") ).to be nil
+    end
+
+    context 'strict option' do
+      let(:options) { super.merge({ 'parse_options' => 'strict' }) }
+
+      it 'does fail parsing' do
+        subject.filter(event)
+        expect( event.get("tags") ).to_not be nil
+        expect( event.get("tags") ).to include '_xmlparsefailure'
+      end
+    end
+  end
+
+  describe "parse_options" do
+    subject { described_class.new(options) }
+    let(:options) { ({ 'source' => 'xmldata', 'store_xml' => false, 'parse_options' => parse_options }) }
+
+    context 'strict (supported option)' do
+      let(:parse_options) { 'strict' }
+
+      it 'registers filter' do
+        subject.register
+        expect( subject.send(:xml_parse_options) ).
+            to eql Nokogiri::XML::ParseOptions::STRICT
+      end
+    end
+
+    context 'valid' do
+      let(:parse_options) { 'no_error,NOWARNING' }
+
+      it 'registers filter' do
+        subject.register
+        expect( subject.send(:xml_parse_options) ).
+            to eql Nokogiri::XML::ParseOptions::NOERROR | Nokogiri::XML::ParseOptions::NOWARNING
+      end
+    end
+
+    context 'invalid' do
+      let(:parse_options) { 'strict,invalid0' }
+
+      it 'fails to register' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, 'unsupported parse option: "invalid0"')
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-xml
 version: !ruby/object:Gem::Version
-  version: 4.0.7
+  version: 4.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-02-27 00:00:00.000000000 Z
+date: 2020-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement