fluent-plugin-xml-parser 0.0.8 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ed5cb30a4cfc986555ae39795e557a75d5df15af19697f5bc215f42febb12e58
-  data.tar.gz: b9ec96aac968f460f0800a40da1d526280cf68dc7ec66fe9a77ad6b9499d070b
+  metadata.gz: c13b08f4ac7b056b6dc68e9bed2e9ebb644826af34701f654575aafa2a836b43
+  data.tar.gz: 5a9c26f558089544bdca1f383ffa1ef5a279a3634d07d3bc9731834a789eb3f6
 SHA512:
-  metadata.gz: 4c3b6f464b7c11b775c949704b2f85006ee5490dc3bb0069b24cdb9d08fa4d9a4f40908da61f9d8992de0fedbd295ce4ab1fc6c6ef1c49f0d66ab5e35e0a812d
-  data.tar.gz: c63e6a37fec38d45a889c784a4e2dd654e2fb711aa14356b8a31f8c40b90afd6adcaa21a12784a4caaebc1c41b85bb6ebbb8cb4f10dffd4e56bc417da0ae781a
+  metadata.gz: 7e3763f8a0ce61351b12f943f27278504472fbcf136455c490eed20e467d10e9d2a7f02634a499899c74a79458dca73b3e12d7921ab947db59f92fce48057267
+  data.tar.gz: e6fe60b671b11dc54faa26b05d0af6a54eb91fdec1b65c9b7552ff154262db5292cd0b3635869bb3f9c00565ba184aa87a55b6275ad9bd360771f47e7881ed07

data/README.md

@@ -2,6 +2,13 @@
 
 Fluent plugin for Parsing XML Input
 
+## Requirements
+
+| fluent-plugin-xml-parser  | Fluentd     | Ruby   |
+|---------------------------|-------------|--------|
+| >= 1.0.0                  | >= v0.14.0  | >= 2.1 |
+| < 1.0.0                   | >= v0.12.0  | >= 1.9 |
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -25,6 +32,33 @@ specified at 'source' directive.
 
 The followings are an example configuration for reformatting Libelium SmartCity sensor data to fit ElasticSearch received via MQTT protocol([fluent-plugin-mqtt-io](https://github.com/toyokazu/fluent-plugin-mqtt-io)).
 
+### For v1.0
+
+Put configuration options in `<parse>` tag
+
+```
+
+<source>
+  type mqtt
+  bind 127.0.0.1
+  port 11883
+  topic 'Libelium/+/#'
+  @label @MQTT_OUT
+  <parse>
+    type @xml
+    time_xpath '["cap:alert/cap:info/cap:onset", "text"]'
+    time_key '@timestamp'
+    attr_xpaths '[["cap:alert/cap:info/cap:parameter/cap:valueName", "text"]]'
+    value_xpaths '[["cap:alert/cap:info/cap:parameter/cap:value", "text"]]'
+  </parse>
+</source>
+
+```
+
+### For v0.12
+
+Use `format` instead of `<parse></parse` and put configuration options in `<source>` tag
+
 ```
 
 <source>

data/fluent-plugin-xml-parser.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-xml-parser"
-  spec.version       = "0.0.8"
+  spec.version       = "1.0.0"
   spec.authors       = ["Toyokazu Akiyama", "Brendan McGrath"]
   spec.email         = ["toyokazu@gmail.com", "brendan@redmandi.com"]
   spec.summary       = %q{fluentd xml parser plugin}
@@ -17,8 +17,9 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "bundler", ">= 1.5"
   spec.add_development_dependency "rake"
-  spec.add_development_dependency "fluentd", "< 2"
+  spec.add_development_dependency "fluentd", [">= 0.14.0", "< 2"]
 
+  spec.add_runtime_dependency "fluentd", [">= 0.14.0", "< 2"]
 end

data/lib/fluent/plugin/parser_xml.rb

@@ -1,126 +1,123 @@
-require 'fluent/parser'
+require 'fluent/plugin/parser'
 require 'rexml/document'
-module Fluent
-  class TextParser
-    class XmlParser < Parser
-      # Register this parser as "xml"
-      Plugin.register_parser("xml", self)
 
-      # How to specify the target attributes and values
-      # The followings are an example description for Libelium SmartCity sensor data.
-      #
-      # time_xpath '["cap:alert/cap:info/cap:onset", "text"]'
-      # attr_xpaths '[[null, "description"], ["cap:alert/cap:info/cap:parameter/cap:valueName", "text"]]'
-      # value_xpaths '[["cap:alert/cap:info/cap:description", "text"], ["cap:alert/cap:info/cap:parameter/cap:value", "text"]]'
-      #
-      # attr_xpaths indicates attribute name of the target value. Each array with two strings
-      # means xpath of the attribute name and the attribute of the XML element (name, text etc).
-      # XPath can be omitted as 'null' and specify your own attribute name as the second
-      # parameter.
-      # 
-      # value_xpaths indicates the target value to be extracted. Each array with two strings
-      # means xpath of the target value and the attribute of the XML element (name, text etc).
-      # XPath can be omitted as 'null' and specify your own value as the second parameter.
-      #
-      # You can check your own XML data structure by using irb or pry
-      #
-      # require 'rexml/document'
-      # doc = REXML::Document.new(open("test.xml"))
-      # doc.elements['cap:alert/cap:info'].children
-      #
-      config_param :time_xpath, :string, :default => nil
-      config_param :time_key, :string, :default => nil
-      config_param :time_format, :string, :default => nil # time_format is configurable
-      config_param :attr_xpaths, :string, :default => '[]'
-      config_param :value_xpaths, :string, :default => '[]'
-      # This method is called after config_params have read configuration parameters
-      def configure(conf)
-        super
+module Fluent::Plugin
+  class XmlParser < Parser
+    # Register this parser as "xml"
+    Fluent::Plugin.register_parser("xml", self)
 
-        if conf['time_xpath'].nil?
-          @time_xpath = nil
-        else
-          @time_xpath = json_parse(conf['time_xpath'])
-        end
-        @time_key = conf['time_key']
-        @time_format = conf['time_format']
-        @time_parser = TimeParser.new(@time_format)
-        @attr_xpaths = json_parse(conf['attr_xpaths'])
-        @value_xpaths = json_parse(conf['value_xpaths'])
-        # TimeParser class is already given. It takes a single argument as the time format
-        # to parse the time string with.
+    # How to specify the target attributes and values
+    # The followings are an example description for Libelium SmartCity sensor data.
+    #
+    # time_xpath '["cap:alert/cap:info/cap:onset", "text"]'
+    # attr_xpaths '[[null, "description"], ["cap:alert/cap:info/cap:parameter/cap:valueName", "text"]]'
+    # value_xpaths '[["cap:alert/cap:info/cap:description", "text"], ["cap:alert/cap:info/cap:parameter/cap:value", "text"]]'
+    #
+    # attr_xpaths indicates attribute name of the target value. Each array with two strings
+    # means xpath of the attribute name and the attribute of the XML element (name, text etc).
+    # XPath can be omitted as 'null' and specify your own attribute name as the second
+    # parameter.
+    #
+    # value_xpaths indicates the target value to be extracted. Each array with two strings
+    # means xpath of the target value and the attribute of the XML element (name, text etc).
+    # XPath can be omitted as 'null' and specify your own value as the second parameter.
+    #
+    # You can check your own XML data structure by using irb or pry
+    #
+    # require 'rexml/document'
+    # doc = REXML::Document.new(open("test.xml"))
+    # doc.elements['cap:alert/cap:info'].children
+    #
+    config_param :time_xpath, :string, :default => nil
+    config_param :time_key, :string, :default => nil
+    config_param :time_format, :string, :default => nil # time_format is configurable
+    config_param :attr_xpaths, :string, :default => '[]'
+    config_param :value_xpaths, :string, :default => '[]'
+    # This method is called after config_params have read configuration parameters
+    def configure(conf)
+      super
+
+      if conf['time_xpath'].nil?
+        @time_xpath = nil
+      else
+        @time_xpath = json_parse(@time_xpath)
       end
+      @time_parser = Fluent::TimeParser.new(@time_format)
+      @attr_xpaths = json_parse(@attr_xpaths)
+      @value_xpaths = json_parse(@value_xpaths)
+      # TimeParser class is already given. It takes a single argument as the time format
+      # to parse the time string with.
+    end
 
-      # This is the main method. The input "text" is the unit of data to be parsed.
-      # If this is the in_tail plugin, it would be a line. If this is for in_syslog,
-      # it is a single syslog message.
-      def parse(text)
-        begin
-          doc = REXML::Document.new(text)
-          $log.debug doc
-          # parse time field
-          if @time_xpath.nil?
-            @time = Fluent::Engine.now
-          else
-            @time = @time_parser.parse(doc.elements[@time_xpath[0]].method(@time_xpath[1]).call)
-          end
-          record = {}
-          if !@time_key.nil?
-            record = {@time_key => format_time(@time)}
-          end
-          attrs = @attr_xpaths.map do |attr_xpath|
-            if attr_xpath[0].nil? # when null is specified
-              attr_xpath[1] # second parameter is used as the attribute name
-            else # otherwise, the target attribute name is extracted from XML
-              el = doc.elements[attr_xpath[0]]
-              unless el.nil? and attr_xpath.size > 2 
-                el.method(attr_xpath[1]).call
-              else # unless it's not in the XML and we have a third parameter
-                attr_xpath[2] # then the third parameter is used as the target value
-              end
+    # This is the main method. The input "text" is the unit of data to be parsed.
+    # If this is the in_tail plugin, it would be a line. If this is for in_syslog,
+    # it is a single syslog message.
+    def parse(text)
+      begin
+        doc = REXML::Document.new(text)
+        $log.debug doc
+        # parse time field
+        if @time_xpath.nil?
+          time = Fluent::Engine.now
+        else
+          time = @time_parser.parse(doc.elements[@time_xpath[0]].method(@time_xpath[1]).call)
+        end
+        record = {}
+        if !@time_key.nil?
+          record = {@time_key => format_time(@time)}
+        end
+        attrs = @attr_xpaths.map do |attr_xpath|
+          if attr_xpath[0].nil? # when null is specified
+            attr_xpath[1] # second parameter is used as the attribute name
+          else # otherwise, the target attribute name is extracted from XML
+            el = doc.elements[attr_xpath[0]]
+            unless el.nil? and attr_xpath.size > 2
+              el.method(attr_xpath[1]).call
+            else # unless it's not in the XML and we have a third parameter
+              attr_xpath[2] # then the third parameter is used as the target value
             end
           end
-          values = @value_xpaths.map do |value_xpath|
-            if value_xpath[0].nil? # when null is specified
-              value_xpath[1] # second parameter is used as the target value
-            else # otherwise, the target value is extracted from XML
-              el = doc.elements[value_xpath[0]]
-              unless el.nil? and value_xpath.size > 2 
-                el.method(value_xpath[1]).call
-              else # unless it's not in the XML and we have a third parameter
-                value_xpath[2] # then the third parameter is used as the target value
-              end
+        end
+        values = @value_xpaths.map do |value_xpath|
+          if value_xpath[0].nil? # when null is specified
+            value_xpath[1] # second parameter is used as the target value
+          else # otherwise, the target value is extracted from XML
+            el = doc.elements[value_xpath[0]]
+            unless el.nil? and value_xpath.size > 2
+              el.method(value_xpath[1]).call
+            else # unless it's not in the XML and we have a third parameter
+              value_xpath[2] # then the third parameter is used as the target value
             end
           end
-          attrs.size.times do |i|
-            record[attrs[i]] = values[i]
-          end
-          yield @time, record
-        rescue REXML::ParseException => e
-          $log.warn "Parse error", :error => e.to_s
-          $log.debug_backtrace(e.backtrace)
-        rescue Exception => e
-          $log.warn "error", :error => e.to_s
-          $log.debug_backtrace(e.backtrace)
         end
+        attrs.size.times do |i|
+          record[attrs[i]] = values[i]
+        end
+        yield time, record
+      rescue REXML::ParseException => e
+        $log.warn "Parse error", :error => e.to_s
+        $log.debug_backtrace(e.backtrace)
+      rescue Exception => e
+        $log.warn "error", :error => e.to_s
+        $log.debug_backtrace(e.backtrace)
       end
+    end
 
-      def format_time(time)
-        if @time_format.nil?
-          Time.at(time).iso8601
-        else
-          Time.at(time).strftime(@time_format)
-        end
+    def format_time(time)
+      if @time_format.nil?
+        Time.at(time).iso8601
+      else
+        Time.at(time).strftime(@time_format)
       end
+    end
 
-      def json_parse message
-        begin
-          y = Yajl::Parser.new
-          y.parse(message)
-        rescue
-          $log.error "JSON parse error", :error => $!.to_s, :error_class => $!.class.to_s
-          $log.warn_backtrace $!.backtrace         
-        end
+    def json_parse message
+      begin
+        y = Yajl::Parser.new
+        y.parse(message)
+      rescue
+        $log.error "JSON parse error", :error => $!.to_s, :error_class => $!.class.to_s
+        $log.warn_backtrace $!.backtrace
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-xml-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 1.0.0
 platform: ruby
 authors:
 - Toyokazu Akiyama
@@ -15,14 +15,14 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.5'
 - !ruby/object:Gem::Dependency
@@ -43,6 +43,9 @@ dependencies:
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -50,6 +53,29 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'