fluent-plugin-xml-parser 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: b080b8bbb4fef60f53d153d3a0c7f72a71465906
4
+ data.tar.gz: 0ade2363ce81d3d3639b7fa7977382b3dfdb4f8c
5
+ SHA512:
6
+ metadata.gz: ba3f247d30b10a7e21bfebaaff9b1ce569e7f7366c2a22e69dbc1486937b05396529ed4ae3eed15609f3b46efd88abbd68bf5cca8f51af14506002f2747de0fb
7
+ data.tar.gz: 8535026f8b26aac6d7ec9d5005419f94a65a2f3c1257de02caea45f2ee7648709a160a82b463e12b0d685adbe65f805a0b6cf650eb716b1722aadef8c53baf64
data/.gitignore ADDED
@@ -0,0 +1,18 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ *~
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gem 'test-unit'
4
+
5
+ # Specify your gem's dependencies in fluent-plugin-mqtt.gemspec
6
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2015 Toyokazu Akiyama
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
22
+
data/README.md ADDED
@@ -0,0 +1,66 @@
1
+ # Fluent::Plugin::XmlParser
2
+
3
+ Fluent plugin for Parsing XML Input
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'fluent-plugin-xml-parser'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install fluent-plugin-xml-parser
18
+
19
+ ## Usage
20
+
21
+ fluent-plugin-xml-parser provides input data conversion from XML to JSON for
22
+ simple case, like sensor data. It can be specified at source directive as
23
+ 'format' option.
24
+
25
+ The followings are an example description for Libelium SmartCity sensor data.
26
+
27
+ ```
28
+
29
+ <source>
30
+ type mqtt
31
+ bind 127.0.0.1
32
+ port 1883
33
+ format xml
34
+ attr_xpaths '[[null, "@timestamp"], ["cap:alert/cap:info/cap:parameter/cap:valueName", "text"], [null, "location"]]'
35
+ value_xpaths '[["cap:alert/cap:info/cap:onset", "text"], ["cap:alert/cap:info/cap:parameter/cap:value", "text"], [null, "Kyoto"]]'
36
+ </source>
37
+
38
+ ```
39
+
40
+ attr_xpaths indicates attribute name of the target value. Each array with two strings
41
+ means xpath of the attribute name and the attribute of the XML element (name, text etc).
42
+ XPath can be omitted as 'null' and specify your own attribute name as the second
43
+ parameter.
44
+
45
+ value_xpaths indicates the target value to be extracted. Each array with two strings
46
+ means xpath of the target value and the attribute of the XML element (name, text etc).
47
+ XPath can be omitted as 'null' and specify your own value as the second parameter.
48
+
49
+ You can check your own XML data structure by using irb or pry
50
+
51
+ ```
52
+
53
+ require 'rexml/document'
54
+ doc = REXML::Document.new(open("test.xml"))
55
+ doc.elements['cap:alert/cap:info'].children
56
+
57
+ ```
58
+
59
+ ## Contributing
60
+
61
+ 1. Fork it ( http://github.com/toyokazu/fluent-plugin-xml-parser/fork )
62
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
63
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
64
+ 4. Push to the branch (`git push origin my-new-feature`)
65
+ 5. Create new Pull Request
66
+
data/Rakefile ADDED
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+
4
+ Rake::TestTask.new(:test) do |test|
5
+ test.libs << 'lib' << 'test'
6
+ test.pattern = 'test/**/test_*.rb'
7
+ test.verbose = true
8
+ end
9
+
10
+ task :default => :test
@@ -0,0 +1,24 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "fluent-plugin-xml-parser"
7
+ spec.version = "0.0.1"
8
+ spec.authors = ["Toyokazu Akiyama"]
9
+ spec.email = ["toyokazu@gmail.com"]
10
+ spec.summary = %q{fluentd xml parser plugin}
11
+ spec.description = %q{fluentd xml parser plugin}
12
+ spec.homepage = "http://github.com/toyokazu/fluent-plugin-xml-parser"
13
+ spec.license = "MIT"
14
+
15
+ spec.files = `git ls-files`.split($/)
16
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_development_dependency "bundler", "~> 1.5"
21
+ spec.add_development_dependency "rake"
22
+ spec.add_development_dependency "fluentd"
23
+
24
+ end
@@ -0,0 +1,95 @@
1
+ require 'rexml/document'
2
+ module Fluent
3
+ class TextParser
4
+ class XmlParser < Parser
5
+ # Register this parser as "xml"
6
+ Plugin.register_parser("xml", self)
7
+
8
+ # How to specify the target attributes and values
9
+ # The followings are an example description for Libelium SmartCity sensor data.
10
+ #
11
+ # attr_xpaths '[[null, "@timestamp"], ["cap:alert/cap:info/cap:parameter/cap:valueName", "text"], [null, "location"]]'
12
+ # value_xpaths '[["cap:alert/cap:info/cap:onset", "text"], ["cap:alert/cap:info/cap:parameter/cap:value", "text"], [null, "Kyoto"]]'
13
+ #
14
+ # attr_xpaths indicates attribute name of the target value. Each array with two strings
15
+ # means xpath of the attribute name and the attribute of the XML element (name, text etc).
16
+ # XPath can be omitted as 'null' and specify your own attribute name as the second
17
+ # parameter.
18
+ #
19
+ # value_xpaths indicates the target value to be extracted. Each array with two strings
20
+ # means xpath of the target value and the attribute of the XML element (name, text etc).
21
+ # XPath can be omitted as 'null' and specify your own value as the second parameter.
22
+ #
23
+ # You can check your own XML data structure by using irb or pry
24
+ #
25
+ # require 'rexml/document'
26
+ # doc = REXML::Document.new(open("test.xml"))
27
+ # doc.elements['cap:alert/cap:info'].children
28
+ #
29
+ config_param :attr_xpaths, :string, :default => '[]'
30
+ config_param :value_xpaths, :string, :default => '[]'
31
+ config_param :time_format, :string, :default => nil # time_format is configurable
32
+ # This method is called after config_params have read configuration parameters
33
+ def configure(conf)
34
+ super
35
+
36
+ @attr_xpaths = json_parse(conf['attr_xpaths'])
37
+ @value_xpaths = json_parse(conf['value_xpaths'])
38
+ @time_format = conf['time_format']
39
+ # TimeParser class is already given. It takes a single argument as the time format
40
+ # to parse the time string with.
41
+ @time_parser = TimeParser.new(@time_format)
42
+ end
43
+
44
+ # This is the main method. The input "text" is the unit of data to be parsed.
45
+ # If this is the in_tail plugin, it would be a line. If this is for in_syslog,
46
+ # it is a single syslog message.
47
+ def parse(text)
48
+ begin
49
+ doc = REXML::Document.new(text)
50
+ $log.debug doc
51
+ record = {}
52
+ attrs = @attr_xpaths.map do |attr_xpath|
53
+ if attr_xpath[0].nil? # when null is specified
54
+ attr_xpath[1] # second parameter is used as the attribute name
55
+ else # otherwise, the target attribute name is extracted from XML
56
+ doc.elements[attr_xpath[0]].method(attr_xpath[1]).call
57
+ end
58
+ end
59
+ values = @value_xpaths.map do |value_xpath|
60
+ if value_xpath[0].nil? # when null is specified
61
+ value_xpath[1] # second parameter is used as the target value
62
+ else # otherwise, the target value is extracted from XML
63
+ doc.elements[value_xpath[0]].method(value_xpath[1]).call
64
+ end
65
+ end
66
+ attrs.size.times do |i|
67
+ if i == 0 # time value
68
+ @time = @time_parser.parse(values[i])
69
+ record[attrs[i]] = @time
70
+ else
71
+ record[attrs[i]] = values[i]
72
+ end
73
+ end
74
+ yield @time, record
75
+ rescue REXML::ParseException => e
76
+ $log.warn "Parse error", :error => e.to_s
77
+ $log.debug_backtrace(e.backtrace)
78
+ rescue Exception => e
79
+ $log.warn "error", :error => e.to_s
80
+ $log.debug_backtrace(e.backtrace)
81
+ end
82
+ end
83
+
84
+ def json_parse message
85
+ begin
86
+ y = Yajl::Parser.new
87
+ y.parse(message)
88
+ rescue
89
+ $log.error "JSON parse error", :error => $!.to_s, :error_class => $!.class.to_s
90
+ $log.warn_backtrace $!.backtrace
91
+ end
92
+ end
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,25 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <cap:alert xmlns:cap="urn:oasis:names:tc:emergency:cap:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:oasis:names:tc:emergency:cap:1.2 CAP-v1.2-os.xsd">
3
+ <cap:identifier>Libelium_2015-09-21T18:54:24+:00:00495</cap:identifier>
4
+ <cap:sender>Libelium_SmartCity_387243170</cap:sender>
5
+ <cap:sent>2015-09-21T18:54:24+:00:00</cap:sent>
6
+ <cap:status>Actual</cap:status>
7
+ <cap:msgType>Alert</cap:msgType>
8
+ <cap:scope>Public</cap:scope>
9
+ <cap:code>KPI</cap:code>
10
+ <cap:info>
11
+ <cap:category>Other</cap:category>
12
+ <cap:event>Libelium</cap:event>
13
+ <cap:urgency>Inmediate</cap:urgency>
14
+ <cap:severity>Unknown</cap:severity>
15
+ <cap:certainty>Observed</cap:certainty>
16
+ <cap:onset>2015-09-21T18:54:24+:00:00</cap:onset>
17
+ <cap:senderName>Libelium</cap:senderName>
18
+ <cap:headline>Waspmote sensors</cap:headline>
19
+ <cap:description>Sensor data from Waspmote devices: MCP</cap:description>
20
+ <cap:parameter>
21
+ <cap:valueName>MCP</cap:valueName>
22
+ <cap:value>50</cap:value>
23
+ </cap:parameter>
24
+ </cap:info>
25
+ </cap:alert>
File without changes
metadata ADDED
@@ -0,0 +1,97 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-xml-parser
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Toyokazu Akiyama
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-11-03 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.5'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.5'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: fluentd
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ description: fluentd xml parser plugin
56
+ email:
57
+ - toyokazu@gmail.com
58
+ executables: []
59
+ extensions: []
60
+ extra_rdoc_files: []
61
+ files:
62
+ - ".gitignore"
63
+ - Gemfile
64
+ - LICENSE
65
+ - README.md
66
+ - Rakefile
67
+ - fluent-plugin-xml-parser.gemspec
68
+ - lib/fluent/plugin/parser_xml.rb
69
+ - test/data/test.xml
70
+ - test/plugin/test_parser_xml.rb
71
+ homepage: http://github.com/toyokazu/fluent-plugin-xml-parser
72
+ licenses:
73
+ - MIT
74
+ metadata: {}
75
+ post_install_message:
76
+ rdoc_options: []
77
+ require_paths:
78
+ - lib
79
+ required_ruby_version: !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - ">="
82
+ - !ruby/object:Gem::Version
83
+ version: '0'
84
+ required_rubygems_version: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ version: '0'
89
+ requirements: []
90
+ rubyforge_project:
91
+ rubygems_version: 2.4.5
92
+ signing_key:
93
+ specification_version: 4
94
+ summary: fluentd xml parser plugin
95
+ test_files:
96
+ - test/data/test.xml
97
+ - test/plugin/test_parser_xml.rb