fluent-plugin-uri-parser 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a55f51dbb0e385c958549ded9a66fcf809bffbb2
4
+ data.tar.gz: fe62b1f8010f1d293c2ea4cd1bfec3456ff2c28d
5
+ SHA512:
6
+ metadata.gz: bb7ca9cf114b0dd4ef080ae891f81b17d8ce91a2d450b17295b136324dcce259933a947a184e0da0ae9ead2dcfbb6c5d56df44ed69c8712ec6708da7dff9c80d
7
+ data.tar.gz: 4e1eba24dead66bf039d056e3241dba725c682006c5c6aa8979f1f3688c06717662cba509c7657cdde0dff7d638fadd6587baf23b50f104cc6942248f67e1726
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-url-parser.gemspec
4
+ gemspec
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2016 Daichi Hirata
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
@@ -0,0 +1,43 @@
1
+ # fluent-plugin-uri-parser
2
+
3
+ [![wercker status](https://app.wercker.com/status/a735d29143f3a1a727fc65653bc81e2a/s "wercker status")](https://app.wercker.com/project/bykey/a735d29143f3a1a727fc65653bc81e2a)
4
+
5
+ This is a Fluentd plugin to parse uri and query string in log messages.
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'fluent-plugin-uri-parser'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install fluent-plugin-uri-parser
22
+
23
+ ## Usage
24
+
25
+ fluent-plugin-uri-parser includes 2 plugins.
26
+
27
+ * uri_parser filter plugin
28
+ * query_string_parser filter plugin
29
+
30
+ ## TODO
31
+
32
+ * Write configuration sample in README.md
33
+
34
+ ## Development
35
+
36
+ After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
37
+
38
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
39
+
40
+ ## Contributing
41
+
42
+ Bug reports and pull requests are welcome on GitHub at https://github.com/daichirata/fluent-plugin-uri-parser.
43
+
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ require 'rake/testtask'
4
+ Rake::TestTask.new(:test) do |test|
5
+ test.libs << 'lib' << 'test'
6
+ test.pattern = 'test/**/test_*.rb'
7
+ test.verbose = true
8
+ end
9
+
10
+ task :default => :test
@@ -0,0 +1,25 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "fluent-plugin-uri-parser"
7
+ spec.version = "0.0.1"
8
+ spec.authors = ["Daichi HIRATA"]
9
+ spec.email = ["bunny.hop.md@gmail.com"]
10
+ spec.license = "Apache-2.0"
11
+
12
+ spec.summary = "This is a Fluentd plugin to parse uri and query string in log messages."
13
+ spec.description = "This is a Fluentd plugin to parse uri and query string in log messages."
14
+ spec.homepage = "https://github.com/daichirata/fluent-plugin-uri-parser"
15
+
16
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
17
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_runtime_dependency "fluentd", "~> 0.12.0"
21
+
22
+ spec.add_development_dependency "test-unit"
23
+ spec.add_development_dependency "bundler", "~> 1.11"
24
+ spec.add_development_dependency "rake", "~> 10.0"
25
+ end
@@ -0,0 +1,44 @@
1
+ class Fluent::QueryStringParserFilter < Fluent::Filter
2
+ Fluent::Plugin.register_filter("query_string_parser", self)
3
+
4
+ config_param :key_name, :string
5
+ config_param :hash_value_field, :string, default: nil
6
+ config_param :inject_key_prefix, :string, default: nil
7
+ config_param :suppress_parse_error_log, :bool, default: false
8
+ config_param :ignore_key_not_exist, :bool, default: false
9
+
10
+ def initialize
11
+ super
12
+ require "uri"
13
+ end
14
+
15
+ def filter_stream(tag, es)
16
+ new_es = Fluent::MultiEventStream.new
17
+
18
+ es.each do |time, record|
19
+ raw_value = record[@key_name]
20
+ if raw_value.nil?
21
+ new_es.add(time, record) unless @ignore_key_not_exist
22
+ next
23
+ end
24
+
25
+ begin
26
+ values = Hash[URI.decode_www_form(raw_value)]
27
+
28
+ unless values.empty?
29
+ if @inject_key_prefix
30
+ values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
31
+ end
32
+ r = @hash_value_field ? { @hash_value_field => values } : values
33
+ record = record.merge(r)
34
+ end
35
+
36
+ new_es.add(time, record)
37
+ rescue => e
38
+ log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
39
+ end
40
+ end
41
+
42
+ new_es
43
+ end
44
+ end
@@ -0,0 +1,76 @@
1
+ class Fluent::URIParserFilter < Fluent::Filter
2
+ Fluent::Plugin.register_filter("uri_parser", self)
3
+
4
+ DEFAULT_PORT_MAP = {
5
+ "http" => 80,
6
+ "https" => 443
7
+ }
8
+
9
+ config_param :key_name, :string
10
+ config_param :hash_value_field, :string, default: nil
11
+ config_param :inject_key_prefix, :string, default: nil
12
+ config_param :suppress_parse_error_log, :bool, default: false
13
+ config_param :ignore_key_not_exist, :bool, default: false
14
+ config_param :ignore_nil, :bool, default: false
15
+
16
+ config_param :out_key_scheme, :string, default: nil
17
+ config_param :out_key_host, :string, default: nil
18
+ config_param :out_key_port, :string, default: nil
19
+ config_param :out_key_path, :string, default: nil
20
+ config_param :out_key_query, :string, default: nil
21
+ config_param :out_key_fragment, :string, default: nil
22
+
23
+ def initialize
24
+ super
25
+ require "uri"
26
+ end
27
+
28
+ def filter_stream(tag, es)
29
+ new_es = Fluent::MultiEventStream.new
30
+
31
+ es.each do |time, record|
32
+ raw_value = record[@key_name]
33
+ if raw_value.nil?
34
+ new_es.add(time, record) unless @ignore_key_not_exist
35
+ next
36
+ end
37
+
38
+ begin
39
+ scheme, host, port, path, query, fragment = parse_uri(raw_value)
40
+
41
+ values = {}
42
+ values[@out_key_scheme] = scheme if @out_key_scheme
43
+ values[@out_key_host] = host if @out_key_host
44
+ values[@out_key_port] = port if @out_key_port
45
+ values[@out_key_path] = path if @out_key_path
46
+ values[@out_key_query] = query if @out_key_query
47
+ values[@out_key_fragment] = fragment if @out_key_fragment
48
+ values.reject! {|_, v| v.nil? } if @ignore_nil
49
+
50
+ unless values.empty?
51
+ if @inject_key_prefix
52
+ values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
53
+ end
54
+ r = @hash_value_field ? { @hash_value_field => values } : values
55
+ record = record.merge(r)
56
+ end
57
+
58
+ new_es.add(time, record)
59
+ rescue => e
60
+ log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
61
+ end
62
+ end
63
+
64
+ new_es
65
+ end
66
+
67
+ private
68
+
69
+ def parse_uri(uri)
70
+ # URI.parse is useful, but it's very slow.
71
+ scheme, _, host, port, _, path, _, query, fragment = URI.split(uri)
72
+ port = port.nil? ? DEFAULT_PORT_MAP[scheme] : port.to_i
73
+
74
+ [scheme, host, port, path, query, fragment]
75
+ end
76
+ end
@@ -0,0 +1,25 @@
1
+ box: ruby:2.3.0
2
+ # Build definition
3
+ build:
4
+ steps:
5
+ - script:
6
+ name: update bundler
7
+ code: gem update bundler
8
+
9
+ # A step that executes `bundle install` command
10
+ - bundle-install:
11
+ jobs: 4
12
+
13
+ # A custom script step, name value is used in the UI
14
+ # and the code value contains the command that get executed
15
+ - script:
16
+ name: echo ruby information
17
+ code: |
18
+ echo "ruby version $(ruby --version) running"
19
+ echo "from location $(which ruby)"
20
+ echo -p "gem list: $(gem list)"
21
+
22
+ # Add more steps here:
23
+ - script:
24
+ name: test
25
+ code: bundle exec rake test
metadata ADDED
@@ -0,0 +1,109 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-uri-parser
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Daichi HIRATA
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2016-05-16 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: 0.12.0
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 0.12.0
27
+ - !ruby/object:Gem::Dependency
28
+ name: test-unit
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: bundler
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '1.11'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.11'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rake
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '10.0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '10.0'
69
+ description: This is a Fluentd plugin to parse uri and query string in log messages.
70
+ email:
71
+ - bunny.hop.md@gmail.com
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - ".gitignore"
77
+ - Gemfile
78
+ - LICENSE.txt
79
+ - README.md
80
+ - Rakefile
81
+ - fluent-plugin-uri-parser.gemspec
82
+ - lib/fluent/plugin/filter_query_string_parser.rb
83
+ - lib/fluent/plugin/filter_uri_parser.rb
84
+ - wercker.yml
85
+ homepage: https://github.com/daichirata/fluent-plugin-uri-parser
86
+ licenses:
87
+ - Apache-2.0
88
+ metadata: {}
89
+ post_install_message:
90
+ rdoc_options: []
91
+ require_paths:
92
+ - lib
93
+ required_ruby_version: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - ">="
96
+ - !ruby/object:Gem::Version
97
+ version: '0'
98
+ required_rubygems_version: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: '0'
103
+ requirements: []
104
+ rubyforge_project:
105
+ rubygems_version: 2.5.1
106
+ signing_key:
107
+ specification_version: 4
108
+ summary: This is a Fluentd plugin to parse uri and query string in log messages.
109
+ test_files: []