fluent-plugin-extract_query_params 0.0.10 → 0.0.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -1
- data/.travis.yml +3 -7
- data/Appraisals +7 -0
- data/README.md +15 -1
- data/fluent-plugin-extract_query_params.gemspec +2 -2
- data/gemfiles/fluentd_0.10.gemfile +7 -0
- data/gemfiles/fluentd_0.12.gemfile +7 -0
- data/lib/fluent/plugin/filter_extract_query_params.rb +32 -0
- data/lib/fluent/plugin/out_extract_query_params.rb +6 -80
- data/lib/fluent/plugin/query_params_extractor.rb +132 -0
- data/test/plugin/test_filter_extract_query_params.rb +332 -0
- data/test/test_helper.rb +4 -0
- metadata +23 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ac87462a28ad9c0a04464033080272f321863f1d
|
4
|
+
data.tar.gz: b4304e2e55078d5d9554b788f4ad95ee1d2d1016
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2247eb4e072f07d11bb16477de324bae075fcd8cd455401463b007b7503f354374e816ba02d7693a08052c997cee4c5fd9c647d0d88833c5358db1cf4eaf0a66
|
7
|
+
data.tar.gz: e90bcefe1aba7baaaa115a79e4d7e91d53b9d2dd2f7f93050ef147bc275f30219148070141bb5ee01d4d5ad90cfae396520a4f163e6f21b188a2c199eb3cbb37
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
@@ -3,13 +3,9 @@ language: ruby
|
|
3
3
|
rvm:
|
4
4
|
- 2.1
|
5
5
|
- 2.2
|
6
|
-
- rbx
|
7
6
|
|
8
7
|
gemfile:
|
9
|
-
|
8
|
+
- gemfiles/fluentd_0.10.gemfile
|
9
|
+
- gemfiles/fluentd_0.12.gemfile
|
10
10
|
|
11
|
-
script: bundle exec rake test
|
12
|
-
|
13
|
-
matrix:
|
14
|
-
allow_failures:
|
15
|
-
- rvm: rbx
|
11
|
+
script: bundle exec rake test
|
data/Appraisals
ADDED
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# fluent-plugin-extract_query_params, a plugin for [Fluentd](http://fluentd.org)
|
1
|
+
# fluent-plugin-extract_query_params, a plugin for [Fluentd](http://fluentd.org) [![](https://travis-ci.org/kentaro/fluent-plugin-extract_query_params.svg)](https://travis-ci.org/kentaro/fluent-plugin-extract_query_params)
|
2
2
|
|
3
3
|
## Component
|
4
4
|
|
@@ -10,6 +10,8 @@ Fluentd plugin to extract key/values from URL query parameters.
|
|
10
10
|
|
11
11
|
Imagin you have a config as below:
|
12
12
|
|
13
|
+
fluentd `< 0.12`:
|
14
|
+
|
13
15
|
```
|
14
16
|
<match test.**>
|
15
17
|
type extract_query_params
|
@@ -20,6 +22,18 @@ Imagin you have a config as below:
|
|
20
22
|
</match>
|
21
23
|
```
|
22
24
|
|
25
|
+
fluentd `>= 0.12`:
|
26
|
+
|
27
|
+
```
|
28
|
+
<filter test.**>
|
29
|
+
type extract_query_params
|
30
|
+
|
31
|
+
key url
|
32
|
+
add_tag_prefix extracted.
|
33
|
+
only foo, baz
|
34
|
+
</match>
|
35
|
+
```
|
36
|
+
|
23
37
|
And you feed such a value into fluentd:
|
24
38
|
|
25
39
|
```
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |gem|
|
2
2
|
gem.name = 'fluent-plugin-extract_query_params'
|
3
|
-
gem.version = '0.0.
|
3
|
+
gem.version = '0.0.11'
|
4
4
|
gem.authors = ['Kentaro Kuribayashi']
|
5
5
|
gem.email = ['kentarok@gmail.com']
|
6
6
|
gem.homepage = 'http://github.com/kentaro/fluent-plugin-extract_query_params'
|
@@ -19,5 +19,5 @@ Gem::Specification.new do |gem|
|
|
19
19
|
|
20
20
|
gem.add_development_dependency 'rake'
|
21
21
|
gem.add_runtime_dependency 'fluentd'
|
22
|
+
gem.add_runtime_dependency 'appraisal'
|
22
23
|
end
|
23
|
-
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Fluent
|
2
|
+
class ExtractQueryParamsFilter < Filter
|
3
|
+
|
4
|
+
Fluent::Plugin.register_filter('extract_query_params', self)
|
5
|
+
|
6
|
+
config_param :key, :string
|
7
|
+
config_param :only, :string, :default => nil
|
8
|
+
config_param :except, :string, :default => nil
|
9
|
+
config_param :discard_key, :bool, :default => false
|
10
|
+
config_param :add_field_prefix, :string, :default => nil
|
11
|
+
config_param :permit_blank_key, :bool, :default => false
|
12
|
+
|
13
|
+
config_param :add_url_scheme, :bool, :default => false
|
14
|
+
config_param :add_url_host, :bool, :default => false
|
15
|
+
config_param :add_url_port, :bool, :default => false
|
16
|
+
config_param :add_url_path, :bool, :default => false
|
17
|
+
|
18
|
+
def initialize
|
19
|
+
super
|
20
|
+
require 'fluent/plugin/query_params_extractor'
|
21
|
+
end
|
22
|
+
|
23
|
+
def configure(conf)
|
24
|
+
super
|
25
|
+
@extractor = QueryParamsExtractor.new(self, conf)
|
26
|
+
end
|
27
|
+
|
28
|
+
def filter(tag, time, record)
|
29
|
+
@extractor.add_query_params_field(record)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -1,6 +1,3 @@
|
|
1
|
-
require 'uri'
|
2
|
-
require 'cgi/util'
|
3
|
-
|
4
1
|
module Fluent
|
5
2
|
class ExtractQueryParamsOutput < Output
|
6
3
|
include Fluent::HandleTagNameMixin
|
@@ -25,30 +22,18 @@ module Fluent
|
|
25
22
|
config_param :add_url_path, :bool, :default => false
|
26
23
|
|
27
24
|
def initialize
|
25
|
+
require 'fluent/plugin/query_params_extractor'
|
28
26
|
super
|
29
|
-
require 'webrick'
|
30
27
|
end
|
31
28
|
|
32
29
|
def configure(conf)
|
33
30
|
super
|
31
|
+
@extractor = QueryParamsExtractor.new(self, conf)
|
32
|
+
end
|
34
33
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
!add_tag_prefix &&
|
39
|
-
!add_tag_suffix
|
40
|
-
)
|
41
|
-
raise ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
|
42
|
-
end
|
43
|
-
|
44
|
-
@include_keys = only && only.split(/\s*,\s*/).inject({}) do |hash, i|
|
45
|
-
hash[i] = true
|
46
|
-
hash
|
47
|
-
end
|
48
|
-
@exclude_keys = except && except.split(/\s*,\s*/).inject({}) do |hash, i|
|
49
|
-
hash[i] = true
|
50
|
-
hash
|
51
|
-
end
|
34
|
+
def filter_record(tag, time, record)
|
35
|
+
record = @extractor.add_query_params_field(record)
|
36
|
+
super(tag, time, record)
|
52
37
|
end
|
53
38
|
|
54
39
|
def emit(tag, es, chain)
|
@@ -60,64 +45,5 @@ module Fluent
|
|
60
45
|
|
61
46
|
chain.next
|
62
47
|
end
|
63
|
-
|
64
|
-
def filter_record(tag, time, record)
|
65
|
-
if record[key]
|
66
|
-
begin
|
67
|
-
url = begin
|
68
|
-
URI.parse(record[key])
|
69
|
-
rescue URI::InvalidURIError => e
|
70
|
-
URI.parse(WEBrick::HTTPUtils.escape(record[key]))
|
71
|
-
end
|
72
|
-
|
73
|
-
if @add_url_scheme
|
74
|
-
url_scheme_key = 'url_scheme'
|
75
|
-
url_scheme_key = @add_field_prefix + url_scheme_key if @add_field_prefix
|
76
|
-
record[url_scheme_key] = url.scheme || ''
|
77
|
-
end
|
78
|
-
|
79
|
-
if @add_url_host
|
80
|
-
url_host_key = 'url_host'
|
81
|
-
url_host_key = @add_field_prefix + url_host_key if @add_field_prefix
|
82
|
-
record[url_host_key] = url.host || ''
|
83
|
-
end
|
84
|
-
|
85
|
-
if @add_url_port
|
86
|
-
url_port_key = 'url_port'
|
87
|
-
url_port_key = @add_field_prefix + url_port_key if @add_field_prefix
|
88
|
-
record[url_port_key] = url.port || ''
|
89
|
-
end
|
90
|
-
|
91
|
-
if @add_url_path
|
92
|
-
url_path_key = 'url_path'
|
93
|
-
url_path_key = @add_field_prefix + url_path_key if @add_field_prefix
|
94
|
-
record[url_path_key] = url.path || ''
|
95
|
-
end
|
96
|
-
|
97
|
-
unless url.query.nil?
|
98
|
-
url.query.split('&').each do |pair|
|
99
|
-
key, value = pair.split('=', 2).map { |i| CGI.unescape(i) }
|
100
|
-
next if (key.nil? || key.empty?) && (!@permit_blank_key || value.nil? || value.empty?)
|
101
|
-
key ||= ''
|
102
|
-
value ||= ''
|
103
|
-
|
104
|
-
key = @add_field_prefix + key if @add_field_prefix
|
105
|
-
if only
|
106
|
-
record[key] = value if @include_keys.has_key?(key)
|
107
|
-
elsif except
|
108
|
-
record[key] = value if !@exclude_keys.has_key?(key)
|
109
|
-
else
|
110
|
-
record[key] = value
|
111
|
-
end
|
112
|
-
end
|
113
|
-
end
|
114
|
-
record.delete key if discard_key
|
115
|
-
rescue URI::InvalidURIError => error
|
116
|
-
$log.warn("out_extract_query_params: #{error.message}")
|
117
|
-
end
|
118
|
-
end
|
119
|
-
|
120
|
-
super(tag, time, record)
|
121
|
-
end
|
122
48
|
end
|
123
49
|
end
|
@@ -0,0 +1,132 @@
|
|
1
|
+
require 'uri'
|
2
|
+
require 'cgi/util'
|
3
|
+
require 'webrick'
|
4
|
+
|
5
|
+
module Fluent
|
6
|
+
class QueryParamsExtractor
|
7
|
+
|
8
|
+
attr_reader :log
|
9
|
+
|
10
|
+
def initialize(plugin, conf)
|
11
|
+
@log = plugin.log
|
12
|
+
|
13
|
+
if plugin.instance_of?(Fluent::ExtractQueryParamsOutput)
|
14
|
+
unless have_tag_option?(plugin)
|
15
|
+
raise ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
@key = plugin.key
|
20
|
+
@only = plugin.only
|
21
|
+
@except = plugin.except
|
22
|
+
@discard_key = plugin.discard_key
|
23
|
+
@add_field_prefix = plugin.add_field_prefix
|
24
|
+
@permit_blank_key = plugin.permit_blank_key
|
25
|
+
|
26
|
+
@add_url_scheme = plugin.add_url_scheme
|
27
|
+
@add_url_host = plugin.add_url_host
|
28
|
+
@add_url_port = plugin.add_url_port
|
29
|
+
@add_url_path = plugin.add_url_path
|
30
|
+
|
31
|
+
if @only
|
32
|
+
@include_keys = @only.split(/\s*,\s*/).inject({}) do |hash, i|
|
33
|
+
hash[i] = true
|
34
|
+
hash
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
if @except
|
39
|
+
@exclude_keys = @except.split(/\s*,\s*/).inject({}) do |hash, i|
|
40
|
+
hash[i] = true
|
41
|
+
hash
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def add_query_params_field(record)
|
47
|
+
return record unless record[@key]
|
48
|
+
url = parse_url(record[@key])
|
49
|
+
add_url_scheme(url, record)
|
50
|
+
add_url_host(url, record)
|
51
|
+
add_url_port(url, record)
|
52
|
+
add_url_path(url, record)
|
53
|
+
add_query_params(url, record)
|
54
|
+
record.delete(@key) if @discard_key
|
55
|
+
record
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
def have_tag_option?(plugin)
|
61
|
+
plugin.remove_tag_prefix ||
|
62
|
+
plugin.remove_tag_suffix ||
|
63
|
+
plugin.add_tag_prefix ||
|
64
|
+
plugin.add_tag_suffix
|
65
|
+
end
|
66
|
+
|
67
|
+
def parse_url(url_string)
|
68
|
+
URI.parse(url_string)
|
69
|
+
rescue URI::InvalidURIError
|
70
|
+
URI.parse(WEBrick::HTTPUtils.escape(url_string))
|
71
|
+
end
|
72
|
+
|
73
|
+
def create_field_key(field_key)
|
74
|
+
if add_field_prefix?
|
75
|
+
"#{@add_field_prefix}#{field_key}"
|
76
|
+
else
|
77
|
+
field_key
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def add_url_scheme(url, record)
|
82
|
+
return unless @add_url_scheme
|
83
|
+
url_scheme_key = create_field_key('url_scheme')
|
84
|
+
record[url_scheme_key] = url.scheme || ''
|
85
|
+
end
|
86
|
+
|
87
|
+
def add_url_host(url, record)
|
88
|
+
return unless @add_url_host
|
89
|
+
url_host_key = create_field_key('url_host')
|
90
|
+
record[url_host_key] = url.host || ''
|
91
|
+
end
|
92
|
+
|
93
|
+
def add_url_port(url, record)
|
94
|
+
return unless @add_url_port
|
95
|
+
url_port_key = create_field_key('url_port')
|
96
|
+
record[url_port_key] = url.port || ''
|
97
|
+
end
|
98
|
+
|
99
|
+
def add_url_path(url, record)
|
100
|
+
return unless @add_url_path
|
101
|
+
url_path_key = create_field_key('url_path')
|
102
|
+
record[url_path_key] = url.path || ''
|
103
|
+
end
|
104
|
+
|
105
|
+
def add_field_prefix?
|
106
|
+
!!@add_field_prefix
|
107
|
+
end
|
108
|
+
|
109
|
+
def permit_blank_key?
|
110
|
+
@permit_blank_key
|
111
|
+
end
|
112
|
+
|
113
|
+
def add_query_params(url, record)
|
114
|
+
return if url.query.nil?
|
115
|
+
url.query.split('&').each do |pair|
|
116
|
+
key, value = pair.split('=', 2).map { |i| CGI.unescape(i) }
|
117
|
+
next if (key.nil? || key.empty?) && (!permit_blank_key? || value.nil? || value.empty?)
|
118
|
+
key ||= ''
|
119
|
+
value ||= ''
|
120
|
+
|
121
|
+
key = create_field_key(key)
|
122
|
+
if @only
|
123
|
+
record[key] = value if @include_keys.has_key?(key)
|
124
|
+
elsif @except
|
125
|
+
record[key] = value if !@exclude_keys.has_key?(key)
|
126
|
+
else
|
127
|
+
record[key] = value
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
@@ -0,0 +1,332 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
if Gem::Version.new(Fluent::VERSION) > Gem::Version.new('0.12')
|
4
|
+
class ExtractQueryParamsFilterTest < Test::Unit::TestCase
|
5
|
+
URL = 'http://example.com:80/?foo=bar&baz=qux&%E3%83%A2%E3%83%AA%E3%82%B9=%E3%81%99%E3%81%9F%E3%81%98%E3%81%8A'
|
6
|
+
QUERY_ONLY = '?foo=bar&baz=qux&%E3%83%A2%E3%83%AA%E3%82%B9=%E3%81%99%E3%81%9F%E3%81%98%E3%81%8A'
|
7
|
+
|
8
|
+
def setup
|
9
|
+
Fluent::Test.setup
|
10
|
+
@time = Fluent::Engine.now
|
11
|
+
end
|
12
|
+
|
13
|
+
def create_driver(conf, tag = 'test')
|
14
|
+
Fluent::Test::FilterTestDriver.new(
|
15
|
+
Fluent::ExtractQueryParamsFilter, tag
|
16
|
+
).configure(conf)
|
17
|
+
end
|
18
|
+
|
19
|
+
def filter(config, messages)
|
20
|
+
d = create_driver(config, 'test')
|
21
|
+
d.run {
|
22
|
+
messages.each {|message|
|
23
|
+
d.filter(message, @time)
|
24
|
+
}
|
25
|
+
}
|
26
|
+
filtered = d.filtered_as_array
|
27
|
+
filtered.map {|m| m[2] }
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_configure
|
31
|
+
d = create_driver(%[
|
32
|
+
key url
|
33
|
+
only foo, baz
|
34
|
+
])
|
35
|
+
|
36
|
+
assert_equal 'url', d.instance.key
|
37
|
+
assert_equal 'foo, baz', d.instance.only
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_filter
|
41
|
+
config = %[
|
42
|
+
key url
|
43
|
+
]
|
44
|
+
|
45
|
+
record = {
|
46
|
+
'url' => URL,
|
47
|
+
}
|
48
|
+
expected = {
|
49
|
+
'url' => URL,
|
50
|
+
'foo' => 'bar',
|
51
|
+
'baz' => 'qux',
|
52
|
+
'モリス' => 'すたじお'
|
53
|
+
}
|
54
|
+
filtered = filter(config, [record])
|
55
|
+
assert_equal(expected, filtered[0])
|
56
|
+
end
|
57
|
+
|
58
|
+
def test_filter_with_field_prefix
|
59
|
+
config = %[
|
60
|
+
key url
|
61
|
+
add_field_prefix query_
|
62
|
+
]
|
63
|
+
|
64
|
+
record = {
|
65
|
+
'url' => URL,
|
66
|
+
}
|
67
|
+
expected = {
|
68
|
+
'url' => URL,
|
69
|
+
'query_foo' => 'bar',
|
70
|
+
'query_baz' => 'qux',
|
71
|
+
'query_モリス' => 'すたじお'
|
72
|
+
}
|
73
|
+
filtered = filter(config, [record])
|
74
|
+
assert_equal(expected, filtered[0])
|
75
|
+
end
|
76
|
+
|
77
|
+
def test_filter_with_only
|
78
|
+
config = %[
|
79
|
+
key url
|
80
|
+
only foo, baz
|
81
|
+
]
|
82
|
+
|
83
|
+
record = { 'url' => URL }
|
84
|
+
expected = {
|
85
|
+
'url' => URL,
|
86
|
+
'foo' => 'bar',
|
87
|
+
'baz' => 'qux',
|
88
|
+
}
|
89
|
+
filtered = filter(config, [record])
|
90
|
+
assert_equal(expected, filtered[0])
|
91
|
+
end
|
92
|
+
|
93
|
+
def test_filter_with_except
|
94
|
+
config = %[
|
95
|
+
key url
|
96
|
+
except baz, モリス
|
97
|
+
]
|
98
|
+
|
99
|
+
record = { 'url' => URL }
|
100
|
+
expected = {
|
101
|
+
'url' => URL,
|
102
|
+
'foo' => 'bar',
|
103
|
+
}
|
104
|
+
filtered = filter(config, [record])
|
105
|
+
assert_equal(expected, filtered[0])
|
106
|
+
end
|
107
|
+
|
108
|
+
def test_filter_with_discard
|
109
|
+
config = %[
|
110
|
+
key url
|
111
|
+
discard_key true
|
112
|
+
]
|
113
|
+
|
114
|
+
record = { 'url' => URL }
|
115
|
+
expected = {
|
116
|
+
'foo' => 'bar',
|
117
|
+
'baz' => 'qux',
|
118
|
+
'モリス' => 'すたじお'
|
119
|
+
}
|
120
|
+
filtered = filter(config, [record])
|
121
|
+
assert_equal(expected, filtered[0])
|
122
|
+
end
|
123
|
+
|
124
|
+
def test_filter_multi_records
|
125
|
+
config = %[
|
126
|
+
key url
|
127
|
+
only foo, baz
|
128
|
+
]
|
129
|
+
records = [
|
130
|
+
{ 'url' => URL },
|
131
|
+
{ 'url' => URL },
|
132
|
+
{ 'url' => URL }
|
133
|
+
]
|
134
|
+
expected = [
|
135
|
+
{ 'url' => URL, 'foo' => 'bar', 'baz' => 'qux' },
|
136
|
+
{ 'url' => URL, 'foo' => 'bar', 'baz' => 'qux' },
|
137
|
+
{ 'url' => URL, 'foo' => 'bar', 'baz' => 'qux' }
|
138
|
+
]
|
139
|
+
filtered = filter(config, records)
|
140
|
+
assert_equal(expected, filtered)
|
141
|
+
end
|
142
|
+
|
143
|
+
def test_emit_without_match_key
|
144
|
+
config = %[
|
145
|
+
key no_such_key
|
146
|
+
only foo, baz
|
147
|
+
]
|
148
|
+
record = { 'url' => URL }
|
149
|
+
filtered = filter(config, [record])
|
150
|
+
assert_equal(record, filtered[0])
|
151
|
+
end
|
152
|
+
|
153
|
+
def test_emit_with_invalid_url
|
154
|
+
config = %[
|
155
|
+
key url
|
156
|
+
]
|
157
|
+
record = { 'url' => URL }
|
158
|
+
filtered = filter(config, [record])
|
159
|
+
assert_equal([record], filtered)
|
160
|
+
end
|
161
|
+
|
162
|
+
DIRTY_PATH_BLANK_1 = '/dummy?&baz=qux'
|
163
|
+
DIRTY_PATH_BLANK_2 = '/dummy?foo=bar&'
|
164
|
+
DIRTY_PATH_BLANK_3 = '/dummy?foo=bar&&baz=qux'
|
165
|
+
DIRTY_PATH_BLANK_4 = '/dummy?=&baz=qux'
|
166
|
+
DIRTY_PATH_KEY_ONLY_1 = '/dummy?foo=&baz=qux'
|
167
|
+
DIRTY_PATH_KEY_ONLY_2 = '/dummy?foo&baz=qux'
|
168
|
+
DIRTY_PATH_KEY_ONLY_3 = '/dummy?baz=qux&foo'
|
169
|
+
DIRTY_PATH_VALUE_ONLY_1 = '/dummy?=bar&baz=qux'
|
170
|
+
DIRTY_PATH_VALUE_ONLY_2 = '/dummy?baz=qux&=bar'
|
171
|
+
DIRTY_PATH_BASE64_1 = '/dummy?foo=ZXh0cmE=&baz=qux'
|
172
|
+
DIRTY_PATH_BASE64_2 = '/dummy?baz=qux&foo=ZXh0cmE='
|
173
|
+
DIRTY_PATH_BASE64_3 = '/dummy?foo=cGFkZGluZw==&baz=qux'
|
174
|
+
DIRTY_PATH_BASE64_4 = '/dummy?baz=qux&foo=cGFkZGluZw=='
|
175
|
+
|
176
|
+
def test_emit_with_dirty_paths
|
177
|
+
config = %[
|
178
|
+
key path
|
179
|
+
]
|
180
|
+
records = [
|
181
|
+
{ 'path' => DIRTY_PATH_BLANK_1 },
|
182
|
+
{ 'path' => DIRTY_PATH_BLANK_2 },
|
183
|
+
{ 'path' => DIRTY_PATH_BLANK_3 },
|
184
|
+
{ 'path' => DIRTY_PATH_BLANK_4 },
|
185
|
+
{ 'path' => DIRTY_PATH_KEY_ONLY_1 },
|
186
|
+
{ 'path' => DIRTY_PATH_KEY_ONLY_2 },
|
187
|
+
{ 'path' => DIRTY_PATH_KEY_ONLY_3 },
|
188
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_1 },
|
189
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_2 },
|
190
|
+
{ 'path' => DIRTY_PATH_BASE64_1 },
|
191
|
+
{ 'path' => DIRTY_PATH_BASE64_2 },
|
192
|
+
{ 'path' => DIRTY_PATH_BASE64_3 },
|
193
|
+
{ 'path' => DIRTY_PATH_BASE64_4 }
|
194
|
+
]
|
195
|
+
expected = [
|
196
|
+
{ 'path' => DIRTY_PATH_BLANK_1, 'baz' => 'qux' },
|
197
|
+
{ 'path' => DIRTY_PATH_BLANK_2, 'foo' => 'bar' },
|
198
|
+
{ 'path' => DIRTY_PATH_BLANK_3, 'foo' => 'bar', 'baz' => 'qux' },
|
199
|
+
{ 'path' => DIRTY_PATH_BLANK_4, 'baz' => 'qux' },
|
200
|
+
{ 'path' => DIRTY_PATH_KEY_ONLY_1, 'foo' => '', 'baz' => 'qux' },
|
201
|
+
{ 'path' => DIRTY_PATH_KEY_ONLY_2, 'foo' => '', 'baz' => 'qux' },
|
202
|
+
{ 'path' => DIRTY_PATH_KEY_ONLY_3, 'foo' => '', 'baz' => 'qux' },
|
203
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_1, 'baz' => 'qux' },
|
204
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_2, 'baz' => 'qux' },
|
205
|
+
{ 'path' => DIRTY_PATH_BASE64_1, 'baz' => 'qux', 'foo' => 'ZXh0cmE=' },
|
206
|
+
{ 'path' => DIRTY_PATH_BASE64_2, 'baz' => 'qux', 'foo' => 'ZXh0cmE=' },
|
207
|
+
{ 'path' => DIRTY_PATH_BASE64_3, 'baz' => 'qux', 'foo' => 'cGFkZGluZw==' },
|
208
|
+
{ 'path' => DIRTY_PATH_BASE64_4, 'baz' => 'qux', 'foo' => 'cGFkZGluZw==' }
|
209
|
+
]
|
210
|
+
filtered = filter(config, records)
|
211
|
+
assert_equal(expected, filtered)
|
212
|
+
end
|
213
|
+
|
214
|
+
def test_emit_with_permit_blank_key
|
215
|
+
config = %[
|
216
|
+
key path
|
217
|
+
permit_blank_key yes
|
218
|
+
]
|
219
|
+
records = [
|
220
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_1 },
|
221
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_2 }
|
222
|
+
]
|
223
|
+
expected = [
|
224
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_1, '' => 'bar', 'baz' => 'qux' },
|
225
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_2, '' => 'bar', 'baz' => 'qux' }
|
226
|
+
]
|
227
|
+
filtered = filter(config, records)
|
228
|
+
assert_equal(expected, filtered)
|
229
|
+
end
|
230
|
+
|
231
|
+
def test_raw_multibyte_chars
|
232
|
+
config = %[
|
233
|
+
key path
|
234
|
+
permit_blank_key yes
|
235
|
+
]
|
236
|
+
|
237
|
+
raw_multibytes_src = '/path/to/ほげぽす/x?a=b'
|
238
|
+
|
239
|
+
records = [
|
240
|
+
{ 'path' => raw_multibytes_src.dup.encode('sjis').force_encoding('ascii-8bit') },
|
241
|
+
{ 'path' => raw_multibytes_src.dup.encode('eucjp').force_encoding('ascii-8bit') }
|
242
|
+
]
|
243
|
+
expected = [
|
244
|
+
{ 'path' => raw_multibytes_src.dup.encode('sjis').force_encoding('ascii-8bit'), 'a' => 'b' },
|
245
|
+
{ 'path' => raw_multibytes_src.dup.encode('eucjp').force_encoding('ascii-8bit'), 'a' => 'b' }
|
246
|
+
]
|
247
|
+
filtered = filter(config, records)
|
248
|
+
assert_equal(expected, filtered)
|
249
|
+
end
|
250
|
+
|
251
|
+
def test_filter_with_url_scheme_host_port_path
|
252
|
+
config = %[
|
253
|
+
key url
|
254
|
+
|
255
|
+
add_url_scheme true
|
256
|
+
add_url_host true
|
257
|
+
add_url_port true
|
258
|
+
add_url_path true
|
259
|
+
]
|
260
|
+
|
261
|
+
record = {
|
262
|
+
'url' => URL,
|
263
|
+
}
|
264
|
+
expected = {
|
265
|
+
'url' => URL,
|
266
|
+
'foo' => 'bar',
|
267
|
+
'baz' => 'qux',
|
268
|
+
'モリス' => 'すたじお',
|
269
|
+
'url_scheme' => 'http',
|
270
|
+
'url_host' => 'example.com',
|
271
|
+
'url_port' => 80,
|
272
|
+
'url_path' => '/'
|
273
|
+
}
|
274
|
+
filtered = filter(config, [record])
|
275
|
+
assert_equal(expected, filtered[0])
|
276
|
+
end
|
277
|
+
|
278
|
+
def test_filter_with_field_prefix_and_url_scheme_host_port_path
|
279
|
+
config = %[
|
280
|
+
key url
|
281
|
+
add_field_prefix query_
|
282
|
+
|
283
|
+
add_url_scheme true
|
284
|
+
add_url_host true
|
285
|
+
add_url_port true
|
286
|
+
add_url_path true
|
287
|
+
]
|
288
|
+
|
289
|
+
record = {
|
290
|
+
'url' => URL,
|
291
|
+
}
|
292
|
+
expected = {
|
293
|
+
'url' => URL,
|
294
|
+
'query_foo' => 'bar',
|
295
|
+
'query_baz' => 'qux',
|
296
|
+
'query_モリス' => 'すたじお',
|
297
|
+
'query_url_scheme' => 'http',
|
298
|
+
'query_url_host' => 'example.com',
|
299
|
+
'query_url_port' => 80,
|
300
|
+
'query_url_path' => '/'
|
301
|
+
}
|
302
|
+
filtered = filter(config, [record])
|
303
|
+
assert_equal(expected, filtered[0])
|
304
|
+
end
|
305
|
+
|
306
|
+
def test_filter_from_query_only_url_with_url_scheme_host_port_path
|
307
|
+
config = %[
|
308
|
+
key url
|
309
|
+
|
310
|
+
add_url_scheme true
|
311
|
+
add_url_host true
|
312
|
+
add_url_port true
|
313
|
+
add_url_path true
|
314
|
+
]
|
315
|
+
record = {
|
316
|
+
'url' => QUERY_ONLY,
|
317
|
+
}
|
318
|
+
expected = {
|
319
|
+
'url' => QUERY_ONLY,
|
320
|
+
'foo' => 'bar',
|
321
|
+
'baz' => 'qux',
|
322
|
+
'モリス' => 'すたじお',
|
323
|
+
'url_scheme' => '',
|
324
|
+
'url_host' => '',
|
325
|
+
'url_port' => '',
|
326
|
+
'url_path' => ''
|
327
|
+
}
|
328
|
+
filtered = filter(config, [record])
|
329
|
+
assert_equal(expected, filtered[0])
|
330
|
+
end
|
331
|
+
end
|
332
|
+
end
|
data/test/test_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-extract_query_params
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kentaro Kuribayashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-09-
|
11
|
+
date: 2015-09-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: test-unit
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: appraisal
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
55
69
|
description: Fluentd plugin to extract key/values from URL query parameters.
|
56
70
|
email:
|
57
71
|
- kentarok@gmail.com
|
@@ -61,12 +75,18 @@ extra_rdoc_files: []
|
|
61
75
|
files:
|
62
76
|
- ".gitignore"
|
63
77
|
- ".travis.yml"
|
78
|
+
- Appraisals
|
64
79
|
- Gemfile
|
65
80
|
- LICENSE
|
66
81
|
- README.md
|
67
82
|
- Rakefile
|
68
83
|
- fluent-plugin-extract_query_params.gemspec
|
84
|
+
- gemfiles/fluentd_0.10.gemfile
|
85
|
+
- gemfiles/fluentd_0.12.gemfile
|
86
|
+
- lib/fluent/plugin/filter_extract_query_params.rb
|
69
87
|
- lib/fluent/plugin/out_extract_query_params.rb
|
88
|
+
- lib/fluent/plugin/query_params_extractor.rb
|
89
|
+
- test/plugin/test_filter_extract_query_params.rb
|
70
90
|
- test/plugin/test_out_extract_query_params.rb
|
71
91
|
- test/test_helper.rb
|
72
92
|
homepage: http://github.com/kentaro/fluent-plugin-extract_query_params
|
@@ -94,5 +114,6 @@ signing_key:
|
|
94
114
|
specification_version: 4
|
95
115
|
summary: Fluentd plugin to extract key/values from URL query parameters
|
96
116
|
test_files:
|
117
|
+
- test/plugin/test_filter_extract_query_params.rb
|
97
118
|
- test/plugin/test_out_extract_query_params.rb
|
98
119
|
- test/test_helper.rb
|