logstash-filter-cleverxml 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 728014a70cda22cc1fdb803c25cbdb4498c7ee9d
+  data.tar.gz: b50591f1688faf6d1aa75ef06cca34759d86998b
+SHA512:
+  metadata.gz: 68e0c28402a8bff7b126e784eb6e89fc2049c2c83f7a23e6c9878a761fa357c2473cab731a69c6a8cbfa3be48b111871397805016a3f0985250b2c2b1b3dfcab
+  data.tar.gz: 239ce573efd1f24fb5149cc79bb01109d8bf39617412adb968e9a3725dceb8f839f4e63f06849b99a67d75436e4ab5bb00dc6720514aee6f4be79e53f9af604b

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 1.0.0
+ - First release.

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,21 @@
+# Logstash Clever XML Plugin
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
+
+- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+

data/lib/logstash/filters/cleverxml.rb

@@ -0,0 +1,163 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# XML filter. Takes a field that contains XML and expands it into
+# an actual datastructure.
+class LogStash::Filters::Cleverxml < LogStash::Filters::Base
+
+  config_name "cleverxml"
+
+  # Config for xml to hash is:
+  #
+  #     source => source_field
+  #
+  # For example, if you have the whole xml document in your @message field:
+  #
+  #     filter {
+  #       xml {
+  #         source => "message"
+  #       }
+  #     }
+  #
+  # The above would parse the xml from the @message field
+  config :source, :validate => :string
+
+  # Define target for placing the data
+  #
+  # for example if you want the data to be put in the 'doc' field:
+  #
+  #     filter {
+  #       xml {
+  #         target => "doc"
+  #       }
+  #     }
+  #
+  # XML in the value of the source field will be expanded into a
+  # datastructure in the "target" field.
+  # Note: if the "target" field already exists, it will be overridden
+  # Required
+  # If target is not defined and store_xml is set to "true", the output
+  # will be written to the root of the event
+  config :target, :validate => :string
+
+  # xpath will additionally select string values (.to_s on whatever is selected)
+  # from parsed XML (using each source field defined using the method above)
+  # and place those values in the destination fields. Configuration:
+  #
+  # xpath => [ "xpath-syntax", "destination-field" ]
+  #
+  # Values returned by XPath parsring from xpath-synatx will be put in the
+  # destination field. Multiple values returned will be pushed onto the
+  # destination field as an array. As such, multiple matches across
+  # multiple source fields will produce duplicate entries in the field
+  #
+  # More on xpath: http://www.w3schools.com/xpath/
+  #
+  # The xpath functions are particularly powerful:
+  # http://www.w3schools.com/xpath/xpath_functions.asp
+  #
+  config :xpath, :validate => :hash, :default => {}
+
+  # By default the filter will store the whole parsed xml in the destination
+  # field as described above. Setting this to false will prevent that. 
+  config :store_xml, :validate => :boolean, :default => true
+  
+   
+  # By default XmlSimple will return an array for each element, even if there is
+  # only one value, if you set this option to "false" it will return an array only
+  # if there is more than one value
+  # Note: if store_xml is set to "true" and target is not defined, the elements will
+  # written as array to the root of the message, if you want the elements to be
+  # written to the root of the message as single fields set force_array to "true"
+  config :force_array, :validate => :boolean, :default => false
+  
+  public
+  def register
+    require "nokogiri"
+    require "xmlsimple"
+
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+    matched = false
+
+    @logger.debug("Running xml filter", :event => event)
+
+    return unless event.include?(@source)
+
+    value = event[@source]
+
+    if value.is_a?(Array) && value.length > 1
+      @logger.warn("XML filter only works on fields of length 1",
+                   :source => @source, :value => value)
+      return
+    end
+
+    # Do nothing with an empty string.
+    return if value.strip.length == 0
+
+    if @xpath
+      begin
+        doc = Nokogiri::XML(value)
+      rescue => e
+        event.tag("_xmlparsefailure")
+        @logger.warn("Trouble parsing xml", :source => @source, :value => value,
+                     :exception => e, :backtrace => e.backtrace)
+        return
+      end
+
+      @xpath.each do |xpath_src, xpath_dest|
+        nodeset = doc.xpath(xpath_src)
+
+        # If asking xpath for a String, like "name(/*)", we get back a
+        # String instead of a NodeSet.  We normalize that here.
+        normalized_nodeset = nodeset.kind_of?(Nokogiri::XML::NodeSet) ? nodeset : [nodeset]
+
+        normalized_nodeset.each do |value|
+          # some XPath functions return empty arrays as string
+          if value.is_a?(Array)
+            return if value.length == 0
+          end
+
+          unless value.nil?
+            matched = true
+            event[xpath_dest] ||= []
+            event[xpath_dest] << value.to_s
+          end
+        end # XPath.each
+      end # @xpath.each
+    end # if @xpath
+
+    if @target.nil?
+        # Default is to write to the root of the event.
+        dest = event.to_hash
+      else
+        dest = event[@target]
+    end
+    
+    if @store_xml
+      begin
+        if dest.nil?
+          dest = XmlSimple.xml_in(value,{'ForceArray' => @force_array, 'SuppressEmpty' => true})
+	  event[@target] = dest
+	else
+          dest.merge!(XmlSimple.xml_in(value,{'ForceArray' => @force_array, 'SuppressEmpty' => true}))
+       	end
+	
+	matched = true
+      rescue => e
+        event.tag("_xmlparsefailure")
+        @logger.warn("Trouble parsing xml with XmlSimple", :source => @source,
+                     :value => value, :exception => e, :backtrace => e.backtrace)
+        return
+      end
+    end # if @store_xml
+
+    filter_matched(event) if matched
+    @logger.debug("Event after xml filter", :event => event)
+  end # def filter
+end # class LogStash::Filters::Xml
+

data/logstash-filter-cleverxml.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-filter-cleverxml'
+  s.version         = '1.0.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Takes a field that contains XML and expands it into an actual datastructure."
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Clever Age", "William Pottier"]
+  s.email           = 'wpottier@clever-age.com'
+  s.homepage        = "http://www.clever-age.com/"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','Gemfile','LICENSE']
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", ">= 1.4.0", "< 2.0.0"
+  s.add_runtime_dependency 'nokogiri'
+  s.add_runtime_dependency 'xml-simple'
+
+  s.add_development_dependency 'logstash-devutils'
+end
+

data/spec/filters/cleverxml_spec.rb

@@ -0,0 +1,209 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/cleverxml"
+
+describe LogStash::Filters::Cleverxml do
+
+  describe "parse standard xml (Deprecated checks)" do
+    config <<-CONFIG
+    filter {
+      cleverxml {
+        source => "raw"
+        target => "data"
+      }
+    }
+    CONFIG
+
+    sample("raw" => '<foo key="value"/>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == {"key" => "value"}
+    end
+
+    #From parse xml with array as a value
+    sample("raw" => '<foo><key>value1</key><key>value2</key></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == {"key" => ["value1", "value2"]}
+    end
+
+    #From parse xml with hash as a value
+    sample("raw" => '<foo><key1><key2>value</key2></key1></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == {"key1" => [{"key2" => ["value"]}]}
+    end
+
+    #From bad xml
+    sample("raw" => '<foo /') do
+      insist { subject["tags"] }.include?("_xmlparsefailure")
+    end
+  end
+
+  describe "parse standard xml but do not store (Deprecated checks)" do
+    config <<-CONFIG
+    filter {
+      cleverxml {
+        source => "raw"
+        target => "data"
+        store_xml => false
+      }
+    }
+    CONFIG
+
+    sample("raw" => '<foo key="value"/>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == nil
+    end
+  end
+
+  describe "parse xml and store values with xpath (Deprecated checks)" do
+    config <<-CONFIG
+    filter {
+      cleverxml {
+        source => "raw"
+        target => "data"
+        xpath => [ "/foo/key/text()", "xpath_field" ]
+      }
+    }
+    CONFIG
+
+    # Single value
+    sample("raw" => '<foo><key>value</key></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["xpath_field"]} == ["value"]
+    end
+
+    #Multiple values
+    sample("raw" => '<foo><key>value1</key><key>value2</key></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["xpath_field"]} == ["value1","value2"]
+    end
+  end
+
+  ## New tests
+
+  describe "parse standard xml" do
+    config <<-CONFIG
+    filter {
+      cleverxml {
+        source => "xmldata"
+        target => "data"
+      }
+    }
+    CONFIG
+
+    sample("xmldata" => '<foo key="value"/>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == {"key" => "value"}
+    end
+
+    #From parse xml with array as a value
+    sample("xmldata" => '<foo><key>value1</key><key>value2</key></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == {"key" => ["value1", "value2"]}
+    end
+
+    #From parse xml with hash as a value
+    sample("xmldata" => '<foo><key1><key2>value</key2></key1></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == {"key1" => [{"key2" => ["value"]}]}
+    end
+
+    #From bad xml
+    sample("xmldata" => '<foo /') do
+      insist { subject["tags"] }.include?("_xmlparsefailure")
+    end
+  end
+
+  describe "parse standard xml but do not store" do
+    config <<-CONFIG
+    filter {
+      cleverxml {
+        source => "xmldata"
+        target => "data"
+        store_xml => false
+      }
+    }
+    CONFIG
+
+    sample("xmldata" => '<foo key="value"/>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == nil
+    end
+  end
+
+  describe "parse xml and store values with xpath" do
+    config <<-CONFIG
+    filter {
+      cleverxml {
+        source => "xmldata"
+        target => "data"
+        xpath => [ "/foo/key/text()", "xpath_field" ]
+      }
+    }
+    CONFIG
+
+    # Single value
+    sample("xmldata" => '<foo><key>value</key></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["xpath_field"]} == ["value"]
+    end
+
+    #Multiple values
+    sample("xmldata" => '<foo><key>value1</key><key>value2</key></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["xpath_field"]} == ["value1","value2"]
+    end
+  end
+
+  describe "parse correctly non ascii content with xpath" do
+    config <<-CONFIG
+    filter {
+      cleverxml {
+        source => "xmldata"
+        target => "data"
+        xpath => [ "/foo/key/text()", "xpath_field" ]
+      }
+    }
+    CONFIG
+
+    # Single value
+    sample("xmldata" => '<foo><key>Français</key></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["xpath_field"]} == ["Français"]
+    end
+  end
+
+  describe "parse including namespaces" do
+    config <<-CONFIG
+    filter {
+      cleverxml {
+        source => "xmldata"
+        xpath => [ "/foo/h:div", "xpath_field" ]
+        remove_namespaces => false
+      }
+    }
+    CONFIG
+
+    # Single value
+    sample("xmldata" => '<foo xmlns:h="http://www.w3.org/TR/html4/"><h:div>Content</h:div></foo>') do
+      insist { subject["xpath_field"] } == ["<h:div>Content</h:div>"]
+    end
+  end
+
+  describe "parse removing namespaces" do
+    config <<-CONFIG
+    filter {
+      cleverxml {
+        source => "xmldata"
+        xpath => [ "/foo/div", "xpath_field" ]
+        remove_namespaces => true
+      }
+    }
+    CONFIG
+
+    # Single value
+    sample("xmldata" => '<foo xmlns:h="http://www.w3.org/TR/html4/"><h:div>Content</h:div></foo>') do
+      insist { subject["xpath_field"] } == ["<div>Content</div>"]
+    end
+  end
+
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-cleverxml
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Clever Age
+- William Pottier
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-10-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: nokogiri
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: xml-simple
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: wpottier@clever-age.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/filters/cleverxml.rb
+- logstash-filter-cleverxml.gemspec
+- spec/filters/cleverxml_spec.rb
+homepage: http://www.clever-age.com/
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.5
+signing_key:
+specification_version: 4
+summary: Takes a field that contains XML and expands it into an actual datastructure.
+test_files:
+- spec/filters/cleverxml_spec.rb