fluent-plugin-extract_query_params 0.0.10 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -1
- data/.travis.yml +3 -7
- data/Appraisals +7 -0
- data/README.md +15 -1
- data/fluent-plugin-extract_query_params.gemspec +2 -2
- data/gemfiles/fluentd_0.10.gemfile +7 -0
- data/gemfiles/fluentd_0.12.gemfile +7 -0
- data/lib/fluent/plugin/filter_extract_query_params.rb +32 -0
- data/lib/fluent/plugin/out_extract_query_params.rb +6 -80
- data/lib/fluent/plugin/query_params_extractor.rb +132 -0
- data/test/plugin/test_filter_extract_query_params.rb +332 -0
- data/test/test_helper.rb +4 -0
- metadata +23 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ac87462a28ad9c0a04464033080272f321863f1d
|
4
|
+
data.tar.gz: b4304e2e55078d5d9554b788f4ad95ee1d2d1016
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2247eb4e072f07d11bb16477de324bae075fcd8cd455401463b007b7503f354374e816ba02d7693a08052c997cee4c5fd9c647d0d88833c5358db1cf4eaf0a66
|
7
|
+
data.tar.gz: e90bcefe1aba7baaaa115a79e4d7e91d53b9d2dd2f7f93050ef147bc275f30219148070141bb5ee01d4d5ad90cfae396520a4f163e6f21b188a2c199eb3cbb37
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
@@ -3,13 +3,9 @@ language: ruby
|
|
3
3
|
rvm:
|
4
4
|
- 2.1
|
5
5
|
- 2.2
|
6
|
-
- rbx
|
7
6
|
|
8
7
|
gemfile:
|
9
|
-
|
8
|
+
- gemfiles/fluentd_0.10.gemfile
|
9
|
+
- gemfiles/fluentd_0.12.gemfile
|
10
10
|
|
11
|
-
script: bundle exec rake test
|
12
|
-
|
13
|
-
matrix:
|
14
|
-
allow_failures:
|
15
|
-
- rvm: rbx
|
11
|
+
script: bundle exec rake test
|
data/Appraisals
ADDED
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# fluent-plugin-extract_query_params, a plugin for [Fluentd](http://fluentd.org)
|
1
|
+
# fluent-plugin-extract_query_params, a plugin for [Fluentd](http://fluentd.org) [](https://travis-ci.org/kentaro/fluent-plugin-extract_query_params)
|
2
2
|
|
3
3
|
## Component
|
4
4
|
|
@@ -10,6 +10,8 @@ Fluentd plugin to extract key/values from URL query parameters.
|
|
10
10
|
|
11
11
|
Imagin you have a config as below:
|
12
12
|
|
13
|
+
fluentd `< 0.12`:
|
14
|
+
|
13
15
|
```
|
14
16
|
<match test.**>
|
15
17
|
type extract_query_params
|
@@ -20,6 +22,18 @@ Imagin you have a config as below:
|
|
20
22
|
</match>
|
21
23
|
```
|
22
24
|
|
25
|
+
fluentd `>= 0.12`:
|
26
|
+
|
27
|
+
```
|
28
|
+
<filter test.**>
|
29
|
+
type extract_query_params
|
30
|
+
|
31
|
+
key url
|
32
|
+
add_tag_prefix extracted.
|
33
|
+
only foo, baz
|
34
|
+
</match>
|
35
|
+
```
|
36
|
+
|
23
37
|
And you feed such a value into fluentd:
|
24
38
|
|
25
39
|
```
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |gem|
|
2
2
|
gem.name = 'fluent-plugin-extract_query_params'
|
3
|
-
gem.version = '0.0.
|
3
|
+
gem.version = '0.0.11'
|
4
4
|
gem.authors = ['Kentaro Kuribayashi']
|
5
5
|
gem.email = ['kentarok@gmail.com']
|
6
6
|
gem.homepage = 'http://github.com/kentaro/fluent-plugin-extract_query_params'
|
@@ -19,5 +19,5 @@ Gem::Specification.new do |gem|
|
|
19
19
|
|
20
20
|
gem.add_development_dependency 'rake'
|
21
21
|
gem.add_runtime_dependency 'fluentd'
|
22
|
+
gem.add_runtime_dependency 'appraisal'
|
22
23
|
end
|
23
|
-
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Fluent
|
2
|
+
class ExtractQueryParamsFilter < Filter
|
3
|
+
|
4
|
+
Fluent::Plugin.register_filter('extract_query_params', self)
|
5
|
+
|
6
|
+
config_param :key, :string
|
7
|
+
config_param :only, :string, :default => nil
|
8
|
+
config_param :except, :string, :default => nil
|
9
|
+
config_param :discard_key, :bool, :default => false
|
10
|
+
config_param :add_field_prefix, :string, :default => nil
|
11
|
+
config_param :permit_blank_key, :bool, :default => false
|
12
|
+
|
13
|
+
config_param :add_url_scheme, :bool, :default => false
|
14
|
+
config_param :add_url_host, :bool, :default => false
|
15
|
+
config_param :add_url_port, :bool, :default => false
|
16
|
+
config_param :add_url_path, :bool, :default => false
|
17
|
+
|
18
|
+
def initialize
|
19
|
+
super
|
20
|
+
require 'fluent/plugin/query_params_extractor'
|
21
|
+
end
|
22
|
+
|
23
|
+
def configure(conf)
|
24
|
+
super
|
25
|
+
@extractor = QueryParamsExtractor.new(self, conf)
|
26
|
+
end
|
27
|
+
|
28
|
+
def filter(tag, time, record)
|
29
|
+
@extractor.add_query_params_field(record)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -1,6 +1,3 @@
|
|
1
|
-
require 'uri'
|
2
|
-
require 'cgi/util'
|
3
|
-
|
4
1
|
module Fluent
|
5
2
|
class ExtractQueryParamsOutput < Output
|
6
3
|
include Fluent::HandleTagNameMixin
|
@@ -25,30 +22,18 @@ module Fluent
|
|
25
22
|
config_param :add_url_path, :bool, :default => false
|
26
23
|
|
27
24
|
def initialize
|
25
|
+
require 'fluent/plugin/query_params_extractor'
|
28
26
|
super
|
29
|
-
require 'webrick'
|
30
27
|
end
|
31
28
|
|
32
29
|
def configure(conf)
|
33
30
|
super
|
31
|
+
@extractor = QueryParamsExtractor.new(self, conf)
|
32
|
+
end
|
34
33
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
!add_tag_prefix &&
|
39
|
-
!add_tag_suffix
|
40
|
-
)
|
41
|
-
raise ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
|
42
|
-
end
|
43
|
-
|
44
|
-
@include_keys = only && only.split(/\s*,\s*/).inject({}) do |hash, i|
|
45
|
-
hash[i] = true
|
46
|
-
hash
|
47
|
-
end
|
48
|
-
@exclude_keys = except && except.split(/\s*,\s*/).inject({}) do |hash, i|
|
49
|
-
hash[i] = true
|
50
|
-
hash
|
51
|
-
end
|
34
|
+
def filter_record(tag, time, record)
|
35
|
+
record = @extractor.add_query_params_field(record)
|
36
|
+
super(tag, time, record)
|
52
37
|
end
|
53
38
|
|
54
39
|
def emit(tag, es, chain)
|
@@ -60,64 +45,5 @@ module Fluent
|
|
60
45
|
|
61
46
|
chain.next
|
62
47
|
end
|
63
|
-
|
64
|
-
def filter_record(tag, time, record)
|
65
|
-
if record[key]
|
66
|
-
begin
|
67
|
-
url = begin
|
68
|
-
URI.parse(record[key])
|
69
|
-
rescue URI::InvalidURIError => e
|
70
|
-
URI.parse(WEBrick::HTTPUtils.escape(record[key]))
|
71
|
-
end
|
72
|
-
|
73
|
-
if @add_url_scheme
|
74
|
-
url_scheme_key = 'url_scheme'
|
75
|
-
url_scheme_key = @add_field_prefix + url_scheme_key if @add_field_prefix
|
76
|
-
record[url_scheme_key] = url.scheme || ''
|
77
|
-
end
|
78
|
-
|
79
|
-
if @add_url_host
|
80
|
-
url_host_key = 'url_host'
|
81
|
-
url_host_key = @add_field_prefix + url_host_key if @add_field_prefix
|
82
|
-
record[url_host_key] = url.host || ''
|
83
|
-
end
|
84
|
-
|
85
|
-
if @add_url_port
|
86
|
-
url_port_key = 'url_port'
|
87
|
-
url_port_key = @add_field_prefix + url_port_key if @add_field_prefix
|
88
|
-
record[url_port_key] = url.port || ''
|
89
|
-
end
|
90
|
-
|
91
|
-
if @add_url_path
|
92
|
-
url_path_key = 'url_path'
|
93
|
-
url_path_key = @add_field_prefix + url_path_key if @add_field_prefix
|
94
|
-
record[url_path_key] = url.path || ''
|
95
|
-
end
|
96
|
-
|
97
|
-
unless url.query.nil?
|
98
|
-
url.query.split('&').each do |pair|
|
99
|
-
key, value = pair.split('=', 2).map { |i| CGI.unescape(i) }
|
100
|
-
next if (key.nil? || key.empty?) && (!@permit_blank_key || value.nil? || value.empty?)
|
101
|
-
key ||= ''
|
102
|
-
value ||= ''
|
103
|
-
|
104
|
-
key = @add_field_prefix + key if @add_field_prefix
|
105
|
-
if only
|
106
|
-
record[key] = value if @include_keys.has_key?(key)
|
107
|
-
elsif except
|
108
|
-
record[key] = value if !@exclude_keys.has_key?(key)
|
109
|
-
else
|
110
|
-
record[key] = value
|
111
|
-
end
|
112
|
-
end
|
113
|
-
end
|
114
|
-
record.delete key if discard_key
|
115
|
-
rescue URI::InvalidURIError => error
|
116
|
-
$log.warn("out_extract_query_params: #{error.message}")
|
117
|
-
end
|
118
|
-
end
|
119
|
-
|
120
|
-
super(tag, time, record)
|
121
|
-
end
|
122
48
|
end
|
123
49
|
end
|
@@ -0,0 +1,132 @@
|
|
1
|
+
require 'uri'
|
2
|
+
require 'cgi/util'
|
3
|
+
require 'webrick'
|
4
|
+
|
5
|
+
module Fluent
|
6
|
+
class QueryParamsExtractor
|
7
|
+
|
8
|
+
attr_reader :log
|
9
|
+
|
10
|
+
def initialize(plugin, conf)
|
11
|
+
@log = plugin.log
|
12
|
+
|
13
|
+
if plugin.instance_of?(Fluent::ExtractQueryParamsOutput)
|
14
|
+
unless have_tag_option?(plugin)
|
15
|
+
raise ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
@key = plugin.key
|
20
|
+
@only = plugin.only
|
21
|
+
@except = plugin.except
|
22
|
+
@discard_key = plugin.discard_key
|
23
|
+
@add_field_prefix = plugin.add_field_prefix
|
24
|
+
@permit_blank_key = plugin.permit_blank_key
|
25
|
+
|
26
|
+
@add_url_scheme = plugin.add_url_scheme
|
27
|
+
@add_url_host = plugin.add_url_host
|
28
|
+
@add_url_port = plugin.add_url_port
|
29
|
+
@add_url_path = plugin.add_url_path
|
30
|
+
|
31
|
+
if @only
|
32
|
+
@include_keys = @only.split(/\s*,\s*/).inject({}) do |hash, i|
|
33
|
+
hash[i] = true
|
34
|
+
hash
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
if @except
|
39
|
+
@exclude_keys = @except.split(/\s*,\s*/).inject({}) do |hash, i|
|
40
|
+
hash[i] = true
|
41
|
+
hash
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def add_query_params_field(record)
|
47
|
+
return record unless record[@key]
|
48
|
+
url = parse_url(record[@key])
|
49
|
+
add_url_scheme(url, record)
|
50
|
+
add_url_host(url, record)
|
51
|
+
add_url_port(url, record)
|
52
|
+
add_url_path(url, record)
|
53
|
+
add_query_params(url, record)
|
54
|
+
record.delete(@key) if @discard_key
|
55
|
+
record
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
def have_tag_option?(plugin)
|
61
|
+
plugin.remove_tag_prefix ||
|
62
|
+
plugin.remove_tag_suffix ||
|
63
|
+
plugin.add_tag_prefix ||
|
64
|
+
plugin.add_tag_suffix
|
65
|
+
end
|
66
|
+
|
67
|
+
def parse_url(url_string)
|
68
|
+
URI.parse(url_string)
|
69
|
+
rescue URI::InvalidURIError
|
70
|
+
URI.parse(WEBrick::HTTPUtils.escape(url_string))
|
71
|
+
end
|
72
|
+
|
73
|
+
def create_field_key(field_key)
|
74
|
+
if add_field_prefix?
|
75
|
+
"#{@add_field_prefix}#{field_key}"
|
76
|
+
else
|
77
|
+
field_key
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def add_url_scheme(url, record)
|
82
|
+
return unless @add_url_scheme
|
83
|
+
url_scheme_key = create_field_key('url_scheme')
|
84
|
+
record[url_scheme_key] = url.scheme || ''
|
85
|
+
end
|
86
|
+
|
87
|
+
def add_url_host(url, record)
|
88
|
+
return unless @add_url_host
|
89
|
+
url_host_key = create_field_key('url_host')
|
90
|
+
record[url_host_key] = url.host || ''
|
91
|
+
end
|
92
|
+
|
93
|
+
def add_url_port(url, record)
|
94
|
+
return unless @add_url_port
|
95
|
+
url_port_key = create_field_key('url_port')
|
96
|
+
record[url_port_key] = url.port || ''
|
97
|
+
end
|
98
|
+
|
99
|
+
def add_url_path(url, record)
|
100
|
+
return unless @add_url_path
|
101
|
+
url_path_key = create_field_key('url_path')
|
102
|
+
record[url_path_key] = url.path || ''
|
103
|
+
end
|
104
|
+
|
105
|
+
def add_field_prefix?
|
106
|
+
!!@add_field_prefix
|
107
|
+
end
|
108
|
+
|
109
|
+
def permit_blank_key?
|
110
|
+
@permit_blank_key
|
111
|
+
end
|
112
|
+
|
113
|
+
def add_query_params(url, record)
|
114
|
+
return if url.query.nil?
|
115
|
+
url.query.split('&').each do |pair|
|
116
|
+
key, value = pair.split('=', 2).map { |i| CGI.unescape(i) }
|
117
|
+
next if (key.nil? || key.empty?) && (!permit_blank_key? || value.nil? || value.empty?)
|
118
|
+
key ||= ''
|
119
|
+
value ||= ''
|
120
|
+
|
121
|
+
key = create_field_key(key)
|
122
|
+
if @only
|
123
|
+
record[key] = value if @include_keys.has_key?(key)
|
124
|
+
elsif @except
|
125
|
+
record[key] = value if !@exclude_keys.has_key?(key)
|
126
|
+
else
|
127
|
+
record[key] = value
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
@@ -0,0 +1,332 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
if Gem::Version.new(Fluent::VERSION) > Gem::Version.new('0.12')
|
4
|
+
class ExtractQueryParamsFilterTest < Test::Unit::TestCase
|
5
|
+
URL = 'http://example.com:80/?foo=bar&baz=qux&%E3%83%A2%E3%83%AA%E3%82%B9=%E3%81%99%E3%81%9F%E3%81%98%E3%81%8A'
|
6
|
+
QUERY_ONLY = '?foo=bar&baz=qux&%E3%83%A2%E3%83%AA%E3%82%B9=%E3%81%99%E3%81%9F%E3%81%98%E3%81%8A'
|
7
|
+
|
8
|
+
def setup
|
9
|
+
Fluent::Test.setup
|
10
|
+
@time = Fluent::Engine.now
|
11
|
+
end
|
12
|
+
|
13
|
+
def create_driver(conf, tag = 'test')
|
14
|
+
Fluent::Test::FilterTestDriver.new(
|
15
|
+
Fluent::ExtractQueryParamsFilter, tag
|
16
|
+
).configure(conf)
|
17
|
+
end
|
18
|
+
|
19
|
+
def filter(config, messages)
|
20
|
+
d = create_driver(config, 'test')
|
21
|
+
d.run {
|
22
|
+
messages.each {|message|
|
23
|
+
d.filter(message, @time)
|
24
|
+
}
|
25
|
+
}
|
26
|
+
filtered = d.filtered_as_array
|
27
|
+
filtered.map {|m| m[2] }
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_configure
|
31
|
+
d = create_driver(%[
|
32
|
+
key url
|
33
|
+
only foo, baz
|
34
|
+
])
|
35
|
+
|
36
|
+
assert_equal 'url', d.instance.key
|
37
|
+
assert_equal 'foo, baz', d.instance.only
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_filter
|
41
|
+
config = %[
|
42
|
+
key url
|
43
|
+
]
|
44
|
+
|
45
|
+
record = {
|
46
|
+
'url' => URL,
|
47
|
+
}
|
48
|
+
expected = {
|
49
|
+
'url' => URL,
|
50
|
+
'foo' => 'bar',
|
51
|
+
'baz' => 'qux',
|
52
|
+
'モリス' => 'すたじお'
|
53
|
+
}
|
54
|
+
filtered = filter(config, [record])
|
55
|
+
assert_equal(expected, filtered[0])
|
56
|
+
end
|
57
|
+
|
58
|
+
def test_filter_with_field_prefix
|
59
|
+
config = %[
|
60
|
+
key url
|
61
|
+
add_field_prefix query_
|
62
|
+
]
|
63
|
+
|
64
|
+
record = {
|
65
|
+
'url' => URL,
|
66
|
+
}
|
67
|
+
expected = {
|
68
|
+
'url' => URL,
|
69
|
+
'query_foo' => 'bar',
|
70
|
+
'query_baz' => 'qux',
|
71
|
+
'query_モリス' => 'すたじお'
|
72
|
+
}
|
73
|
+
filtered = filter(config, [record])
|
74
|
+
assert_equal(expected, filtered[0])
|
75
|
+
end
|
76
|
+
|
77
|
+
def test_filter_with_only
|
78
|
+
config = %[
|
79
|
+
key url
|
80
|
+
only foo, baz
|
81
|
+
]
|
82
|
+
|
83
|
+
record = { 'url' => URL }
|
84
|
+
expected = {
|
85
|
+
'url' => URL,
|
86
|
+
'foo' => 'bar',
|
87
|
+
'baz' => 'qux',
|
88
|
+
}
|
89
|
+
filtered = filter(config, [record])
|
90
|
+
assert_equal(expected, filtered[0])
|
91
|
+
end
|
92
|
+
|
93
|
+
def test_filter_with_except
|
94
|
+
config = %[
|
95
|
+
key url
|
96
|
+
except baz, モリス
|
97
|
+
]
|
98
|
+
|
99
|
+
record = { 'url' => URL }
|
100
|
+
expected = {
|
101
|
+
'url' => URL,
|
102
|
+
'foo' => 'bar',
|
103
|
+
}
|
104
|
+
filtered = filter(config, [record])
|
105
|
+
assert_equal(expected, filtered[0])
|
106
|
+
end
|
107
|
+
|
108
|
+
def test_filter_with_discard
|
109
|
+
config = %[
|
110
|
+
key url
|
111
|
+
discard_key true
|
112
|
+
]
|
113
|
+
|
114
|
+
record = { 'url' => URL }
|
115
|
+
expected = {
|
116
|
+
'foo' => 'bar',
|
117
|
+
'baz' => 'qux',
|
118
|
+
'モリス' => 'すたじお'
|
119
|
+
}
|
120
|
+
filtered = filter(config, [record])
|
121
|
+
assert_equal(expected, filtered[0])
|
122
|
+
end
|
123
|
+
|
124
|
+
def test_filter_multi_records
|
125
|
+
config = %[
|
126
|
+
key url
|
127
|
+
only foo, baz
|
128
|
+
]
|
129
|
+
records = [
|
130
|
+
{ 'url' => URL },
|
131
|
+
{ 'url' => URL },
|
132
|
+
{ 'url' => URL }
|
133
|
+
]
|
134
|
+
expected = [
|
135
|
+
{ 'url' => URL, 'foo' => 'bar', 'baz' => 'qux' },
|
136
|
+
{ 'url' => URL, 'foo' => 'bar', 'baz' => 'qux' },
|
137
|
+
{ 'url' => URL, 'foo' => 'bar', 'baz' => 'qux' }
|
138
|
+
]
|
139
|
+
filtered = filter(config, records)
|
140
|
+
assert_equal(expected, filtered)
|
141
|
+
end
|
142
|
+
|
143
|
+
def test_emit_without_match_key
|
144
|
+
config = %[
|
145
|
+
key no_such_key
|
146
|
+
only foo, baz
|
147
|
+
]
|
148
|
+
record = { 'url' => URL }
|
149
|
+
filtered = filter(config, [record])
|
150
|
+
assert_equal(record, filtered[0])
|
151
|
+
end
|
152
|
+
|
153
|
+
def test_emit_with_invalid_url
|
154
|
+
config = %[
|
155
|
+
key url
|
156
|
+
]
|
157
|
+
record = { 'url' => URL }
|
158
|
+
filtered = filter(config, [record])
|
159
|
+
assert_equal([record], filtered)
|
160
|
+
end
|
161
|
+
|
162
|
+
DIRTY_PATH_BLANK_1 = '/dummy?&baz=qux'
|
163
|
+
DIRTY_PATH_BLANK_2 = '/dummy?foo=bar&'
|
164
|
+
DIRTY_PATH_BLANK_3 = '/dummy?foo=bar&&baz=qux'
|
165
|
+
DIRTY_PATH_BLANK_4 = '/dummy?=&baz=qux'
|
166
|
+
DIRTY_PATH_KEY_ONLY_1 = '/dummy?foo=&baz=qux'
|
167
|
+
DIRTY_PATH_KEY_ONLY_2 = '/dummy?foo&baz=qux'
|
168
|
+
DIRTY_PATH_KEY_ONLY_3 = '/dummy?baz=qux&foo'
|
169
|
+
DIRTY_PATH_VALUE_ONLY_1 = '/dummy?=bar&baz=qux'
|
170
|
+
DIRTY_PATH_VALUE_ONLY_2 = '/dummy?baz=qux&=bar'
|
171
|
+
DIRTY_PATH_BASE64_1 = '/dummy?foo=ZXh0cmE=&baz=qux'
|
172
|
+
DIRTY_PATH_BASE64_2 = '/dummy?baz=qux&foo=ZXh0cmE='
|
173
|
+
DIRTY_PATH_BASE64_3 = '/dummy?foo=cGFkZGluZw==&baz=qux'
|
174
|
+
DIRTY_PATH_BASE64_4 = '/dummy?baz=qux&foo=cGFkZGluZw=='
|
175
|
+
|
176
|
+
def test_emit_with_dirty_paths
|
177
|
+
config = %[
|
178
|
+
key path
|
179
|
+
]
|
180
|
+
records = [
|
181
|
+
{ 'path' => DIRTY_PATH_BLANK_1 },
|
182
|
+
{ 'path' => DIRTY_PATH_BLANK_2 },
|
183
|
+
{ 'path' => DIRTY_PATH_BLANK_3 },
|
184
|
+
{ 'path' => DIRTY_PATH_BLANK_4 },
|
185
|
+
{ 'path' => DIRTY_PATH_KEY_ONLY_1 },
|
186
|
+
{ 'path' => DIRTY_PATH_KEY_ONLY_2 },
|
187
|
+
{ 'path' => DIRTY_PATH_KEY_ONLY_3 },
|
188
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_1 },
|
189
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_2 },
|
190
|
+
{ 'path' => DIRTY_PATH_BASE64_1 },
|
191
|
+
{ 'path' => DIRTY_PATH_BASE64_2 },
|
192
|
+
{ 'path' => DIRTY_PATH_BASE64_3 },
|
193
|
+
{ 'path' => DIRTY_PATH_BASE64_4 }
|
194
|
+
]
|
195
|
+
expected = [
|
196
|
+
{ 'path' => DIRTY_PATH_BLANK_1, 'baz' => 'qux' },
|
197
|
+
{ 'path' => DIRTY_PATH_BLANK_2, 'foo' => 'bar' },
|
198
|
+
{ 'path' => DIRTY_PATH_BLANK_3, 'foo' => 'bar', 'baz' => 'qux' },
|
199
|
+
{ 'path' => DIRTY_PATH_BLANK_4, 'baz' => 'qux' },
|
200
|
+
{ 'path' => DIRTY_PATH_KEY_ONLY_1, 'foo' => '', 'baz' => 'qux' },
|
201
|
+
{ 'path' => DIRTY_PATH_KEY_ONLY_2, 'foo' => '', 'baz' => 'qux' },
|
202
|
+
{ 'path' => DIRTY_PATH_KEY_ONLY_3, 'foo' => '', 'baz' => 'qux' },
|
203
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_1, 'baz' => 'qux' },
|
204
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_2, 'baz' => 'qux' },
|
205
|
+
{ 'path' => DIRTY_PATH_BASE64_1, 'baz' => 'qux', 'foo' => 'ZXh0cmE=' },
|
206
|
+
{ 'path' => DIRTY_PATH_BASE64_2, 'baz' => 'qux', 'foo' => 'ZXh0cmE=' },
|
207
|
+
{ 'path' => DIRTY_PATH_BASE64_3, 'baz' => 'qux', 'foo' => 'cGFkZGluZw==' },
|
208
|
+
{ 'path' => DIRTY_PATH_BASE64_4, 'baz' => 'qux', 'foo' => 'cGFkZGluZw==' }
|
209
|
+
]
|
210
|
+
filtered = filter(config, records)
|
211
|
+
assert_equal(expected, filtered)
|
212
|
+
end
|
213
|
+
|
214
|
+
def test_emit_with_permit_blank_key
|
215
|
+
config = %[
|
216
|
+
key path
|
217
|
+
permit_blank_key yes
|
218
|
+
]
|
219
|
+
records = [
|
220
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_1 },
|
221
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_2 }
|
222
|
+
]
|
223
|
+
expected = [
|
224
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_1, '' => 'bar', 'baz' => 'qux' },
|
225
|
+
{ 'path' => DIRTY_PATH_VALUE_ONLY_2, '' => 'bar', 'baz' => 'qux' }
|
226
|
+
]
|
227
|
+
filtered = filter(config, records)
|
228
|
+
assert_equal(expected, filtered)
|
229
|
+
end
|
230
|
+
|
231
|
+
def test_raw_multibyte_chars
|
232
|
+
config = %[
|
233
|
+
key path
|
234
|
+
permit_blank_key yes
|
235
|
+
]
|
236
|
+
|
237
|
+
raw_multibytes_src = '/path/to/ほげぽす/x?a=b'
|
238
|
+
|
239
|
+
records = [
|
240
|
+
{ 'path' => raw_multibytes_src.dup.encode('sjis').force_encoding('ascii-8bit') },
|
241
|
+
{ 'path' => raw_multibytes_src.dup.encode('eucjp').force_encoding('ascii-8bit') }
|
242
|
+
]
|
243
|
+
expected = [
|
244
|
+
{ 'path' => raw_multibytes_src.dup.encode('sjis').force_encoding('ascii-8bit'), 'a' => 'b' },
|
245
|
+
{ 'path' => raw_multibytes_src.dup.encode('eucjp').force_encoding('ascii-8bit'), 'a' => 'b' }
|
246
|
+
]
|
247
|
+
filtered = filter(config, records)
|
248
|
+
assert_equal(expected, filtered)
|
249
|
+
end
|
250
|
+
|
251
|
+
def test_filter_with_url_scheme_host_port_path
|
252
|
+
config = %[
|
253
|
+
key url
|
254
|
+
|
255
|
+
add_url_scheme true
|
256
|
+
add_url_host true
|
257
|
+
add_url_port true
|
258
|
+
add_url_path true
|
259
|
+
]
|
260
|
+
|
261
|
+
record = {
|
262
|
+
'url' => URL,
|
263
|
+
}
|
264
|
+
expected = {
|
265
|
+
'url' => URL,
|
266
|
+
'foo' => 'bar',
|
267
|
+
'baz' => 'qux',
|
268
|
+
'モリス' => 'すたじお',
|
269
|
+
'url_scheme' => 'http',
|
270
|
+
'url_host' => 'example.com',
|
271
|
+
'url_port' => 80,
|
272
|
+
'url_path' => '/'
|
273
|
+
}
|
274
|
+
filtered = filter(config, [record])
|
275
|
+
assert_equal(expected, filtered[0])
|
276
|
+
end
|
277
|
+
|
278
|
+
def test_filter_with_field_prefix_and_url_scheme_host_port_path
|
279
|
+
config = %[
|
280
|
+
key url
|
281
|
+
add_field_prefix query_
|
282
|
+
|
283
|
+
add_url_scheme true
|
284
|
+
add_url_host true
|
285
|
+
add_url_port true
|
286
|
+
add_url_path true
|
287
|
+
]
|
288
|
+
|
289
|
+
record = {
|
290
|
+
'url' => URL,
|
291
|
+
}
|
292
|
+
expected = {
|
293
|
+
'url' => URL,
|
294
|
+
'query_foo' => 'bar',
|
295
|
+
'query_baz' => 'qux',
|
296
|
+
'query_モリス' => 'すたじお',
|
297
|
+
'query_url_scheme' => 'http',
|
298
|
+
'query_url_host' => 'example.com',
|
299
|
+
'query_url_port' => 80,
|
300
|
+
'query_url_path' => '/'
|
301
|
+
}
|
302
|
+
filtered = filter(config, [record])
|
303
|
+
assert_equal(expected, filtered[0])
|
304
|
+
end
|
305
|
+
|
306
|
+
def test_filter_from_query_only_url_with_url_scheme_host_port_path
|
307
|
+
config = %[
|
308
|
+
key url
|
309
|
+
|
310
|
+
add_url_scheme true
|
311
|
+
add_url_host true
|
312
|
+
add_url_port true
|
313
|
+
add_url_path true
|
314
|
+
]
|
315
|
+
record = {
|
316
|
+
'url' => QUERY_ONLY,
|
317
|
+
}
|
318
|
+
expected = {
|
319
|
+
'url' => QUERY_ONLY,
|
320
|
+
'foo' => 'bar',
|
321
|
+
'baz' => 'qux',
|
322
|
+
'モリス' => 'すたじお',
|
323
|
+
'url_scheme' => '',
|
324
|
+
'url_host' => '',
|
325
|
+
'url_port' => '',
|
326
|
+
'url_path' => ''
|
327
|
+
}
|
328
|
+
filtered = filter(config, [record])
|
329
|
+
assert_equal(expected, filtered[0])
|
330
|
+
end
|
331
|
+
end
|
332
|
+
end
|
data/test/test_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-extract_query_params
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kentaro Kuribayashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-09-
|
11
|
+
date: 2015-09-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: test-unit
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: appraisal
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
55
69
|
description: Fluentd plugin to extract key/values from URL query parameters.
|
56
70
|
email:
|
57
71
|
- kentarok@gmail.com
|
@@ -61,12 +75,18 @@ extra_rdoc_files: []
|
|
61
75
|
files:
|
62
76
|
- ".gitignore"
|
63
77
|
- ".travis.yml"
|
78
|
+
- Appraisals
|
64
79
|
- Gemfile
|
65
80
|
- LICENSE
|
66
81
|
- README.md
|
67
82
|
- Rakefile
|
68
83
|
- fluent-plugin-extract_query_params.gemspec
|
84
|
+
- gemfiles/fluentd_0.10.gemfile
|
85
|
+
- gemfiles/fluentd_0.12.gemfile
|
86
|
+
- lib/fluent/plugin/filter_extract_query_params.rb
|
69
87
|
- lib/fluent/plugin/out_extract_query_params.rb
|
88
|
+
- lib/fluent/plugin/query_params_extractor.rb
|
89
|
+
- test/plugin/test_filter_extract_query_params.rb
|
70
90
|
- test/plugin/test_out_extract_query_params.rb
|
71
91
|
- test/test_helper.rb
|
72
92
|
homepage: http://github.com/kentaro/fluent-plugin-extract_query_params
|
@@ -94,5 +114,6 @@ signing_key:
|
|
94
114
|
specification_version: 4
|
95
115
|
summary: Fluentd plugin to extract key/values from URL query parameters
|
96
116
|
test_files:
|
117
|
+
- test/plugin/test_filter_extract_query_params.rb
|
97
118
|
- test/plugin/test_out_extract_query_params.rb
|
98
119
|
- test/test_helper.rb
|