fluent-plugin-extract_query_params 0.0.12 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +4 -2
- data/Appraisals +2 -6
- data/README.md +2 -3
- data/fluent-plugin-extract_query_params.gemspec +3 -3
- data/gemfiles/{fluentd_0.10.gemfile → fluentd_0.14.gemfile} +1 -1
- data/lib/fluent/plugin/filter_extract_query_params.rb +25 -17
- data/lib/fluent/plugin/out_extract_query_params.rb +28 -25
- data/lib/fluent/plugin/query_params_extractor.rb +3 -3
- data/test/plugin/test_filter_extract_query_params.rb +8 -10
- data/test/plugin/test_out_extract_query_params.rb +72 -81
- data/test/test_helper.rb +5 -21
- metadata +15 -10
- data/gemfiles/fluentd_0.12.gemfile +0 -7
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 993a35a9a1e1b944a3cde32b2b770054a261d217
|
|
4
|
+
data.tar.gz: e7a782eb22c24c36426ac9923dae9828e73697e1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a1e100c5cbf7ebe173431b5afd06557d1277aa27316ec54d0d92ac1bb3a81abf6d7930c31657686ef5520e5119707db4ec1c2f401801367ff875ddb32574a943
|
|
7
|
+
data.tar.gz: c281cdc98702219c3761f9c298cc597f18b65381f194286ae621c716750fa89e7c995152d8a141614fef72ddce658f6f459e03312d36153ea6e278abb04bdf72
|
data/.travis.yml
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
|
+
sudo: false
|
|
1
2
|
language: ruby
|
|
2
3
|
|
|
3
4
|
rvm:
|
|
4
5
|
- 2.1
|
|
5
6
|
- 2.2
|
|
7
|
+
- 2.3.1
|
|
6
8
|
|
|
7
9
|
gemfile:
|
|
8
|
-
- gemfiles/fluentd_0.
|
|
9
|
-
- gemfiles/fluentd_0.12.gemfile
|
|
10
|
+
- gemfiles/fluentd_0.14.gemfile
|
|
10
11
|
|
|
12
|
+
before_script: gem update bundler
|
|
11
13
|
script: bundle exec rake test
|
data/Appraisals
CHANGED
data/README.md
CHANGED
|
@@ -14,7 +14,7 @@ fluentd `< 0.12`:
|
|
|
14
14
|
|
|
15
15
|
```
|
|
16
16
|
<match test.**>
|
|
17
|
-
type extract_query_params
|
|
17
|
+
@type extract_query_params
|
|
18
18
|
|
|
19
19
|
key url
|
|
20
20
|
add_tag_prefix extracted.
|
|
@@ -26,10 +26,9 @@ fluentd `>= 0.12`:
|
|
|
26
26
|
|
|
27
27
|
```
|
|
28
28
|
<filter test.**>
|
|
29
|
-
type extract_query_params
|
|
29
|
+
@type extract_query_params
|
|
30
30
|
|
|
31
31
|
key url
|
|
32
|
-
add_tag_prefix extracted.
|
|
33
32
|
only foo, baz
|
|
34
33
|
</match>
|
|
35
34
|
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |gem|
|
|
2
2
|
gem.name = 'fluent-plugin-extract_query_params'
|
|
3
|
-
gem.version = '0.0
|
|
3
|
+
gem.version = '0.1.0'
|
|
4
4
|
gem.authors = ['Kentaro Kuribayashi']
|
|
5
5
|
gem.email = ['kentarok@gmail.com']
|
|
6
6
|
gem.homepage = 'http://github.com/kentaro/fluent-plugin-extract_query_params'
|
|
@@ -18,6 +18,6 @@ Gem::Specification.new do |gem|
|
|
|
18
18
|
end
|
|
19
19
|
|
|
20
20
|
gem.add_development_dependency 'rake'
|
|
21
|
-
gem.
|
|
22
|
-
gem.add_runtime_dependency '
|
|
21
|
+
gem.add_development_dependency 'appraisal'
|
|
22
|
+
gem.add_runtime_dependency 'fluentd', ['>= 0.14.8', '< 2']
|
|
23
23
|
end
|
|
@@ -1,28 +1,36 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
require "fluent/plugin/filter"
|
|
2
|
+
require "fluent/plugin/query_params_extractor"
|
|
3
|
+
|
|
4
|
+
module Fluent::Plugin
|
|
5
|
+
class ExtractQueryParamsFilter < Fluent::Plugin::Filter
|
|
3
6
|
|
|
4
7
|
Fluent::Plugin.register_filter('extract_query_params', self)
|
|
5
8
|
|
|
9
|
+
desc "point a key whose value contains URL string."
|
|
6
10
|
config_param :key, :string
|
|
7
|
-
|
|
8
|
-
config_param :
|
|
9
|
-
|
|
10
|
-
config_param :
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
config_param :
|
|
15
|
-
|
|
16
|
-
config_param :
|
|
11
|
+
desc "If set, only the key/value whose key is included only will be added to the record."
|
|
12
|
+
config_param :only, :string, default: nil
|
|
13
|
+
desc "If set, the key/value whose key is included except will NOT be added to the record."
|
|
14
|
+
config_param :except, :string, default: nil
|
|
15
|
+
desc "If set to true, the original key url will be discarded from the record."
|
|
16
|
+
config_param :discard_key, :bool, default: false
|
|
17
|
+
desc "Prefix of fields."
|
|
18
|
+
config_param :add_field_prefix, :string, default: nil
|
|
19
|
+
desc "If set to true, permit blank key."
|
|
20
|
+
config_param :permit_blank_key, :bool, default: false
|
|
17
21
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
+
desc "If set to true, scheme (use url_scheme key) will be added to the record."
|
|
23
|
+
config_param :add_url_scheme, :bool, default: false
|
|
24
|
+
desc "If set to true, host (use url_host key) will be added to the record."
|
|
25
|
+
config_param :add_url_host, :bool, default: false
|
|
26
|
+
desc "If set to true, port (use url_port key) will be added to the record."
|
|
27
|
+
config_param :add_url_port, :bool, default: false
|
|
28
|
+
desc "If set to true, path (use url_path key) will be added to the record."
|
|
29
|
+
config_param :add_url_path, :bool, default: false
|
|
22
30
|
|
|
23
31
|
def configure(conf)
|
|
24
32
|
super
|
|
25
|
-
@extractor = QueryParamsExtractor.new(self, conf)
|
|
33
|
+
@extractor = Fluent::Plugin::QueryParamsExtractor.new(self, conf)
|
|
26
34
|
end
|
|
27
35
|
|
|
28
36
|
def filter(tag, time, record)
|
|
@@ -1,34 +1,39 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
require "fluent/plugin/output"
|
|
2
|
+
require "fluent/plugin/query_params_extractor"
|
|
3
|
+
|
|
4
|
+
module Fluent::Plugin
|
|
5
|
+
class ExtractQueryParamsOutput < Fluent::Plugin::Output
|
|
3
6
|
include Fluent::HandleTagNameMixin
|
|
4
7
|
|
|
5
8
|
Fluent::Plugin.register_output('extract_query_params', self)
|
|
6
9
|
|
|
7
|
-
|
|
8
|
-
unless method_defined?(:router)
|
|
9
|
-
define_method("router") { Fluent::Engine }
|
|
10
|
-
end
|
|
10
|
+
helpers :event_emitter
|
|
11
11
|
|
|
12
|
+
desc "point a key whose value contains URL string."
|
|
12
13
|
config_param :key, :string
|
|
13
|
-
|
|
14
|
-
config_param :
|
|
15
|
-
|
|
16
|
-
config_param :
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
config_param :
|
|
21
|
-
|
|
22
|
-
config_param :
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
14
|
+
desc "If set, only the key/value whose key is included only will be added to the record."
|
|
15
|
+
config_param :only, :string, default: nil
|
|
16
|
+
desc "If set, the key/value whose key is included except will NOT be added to the record."
|
|
17
|
+
config_param :except, :string, default: nil
|
|
18
|
+
desc "If set to true, the original key url will be discarded from the record."
|
|
19
|
+
config_param :discard_key, :bool, default: false
|
|
20
|
+
desc "Prefix of fields."
|
|
21
|
+
config_param :add_field_prefix, :string, default: nil
|
|
22
|
+
desc "If set to true, permit blank key."
|
|
23
|
+
config_param :permit_blank_key, :bool, default: false
|
|
24
|
+
|
|
25
|
+
desc "If set to true, scheme (use url_scheme key) will be added to the record."
|
|
26
|
+
config_param :add_url_scheme, :bool, default: false
|
|
27
|
+
desc "If set to true, host (use url_host key) will be added to the record."
|
|
28
|
+
config_param :add_url_host, :bool, default: false
|
|
29
|
+
desc "If set to true, port (use url_port key) will be added to the record."
|
|
30
|
+
config_param :add_url_port, :bool, default: false
|
|
31
|
+
desc "If set to true, path (use url_path key) will be added to the record."
|
|
32
|
+
config_param :add_url_path, :bool, default: false
|
|
28
33
|
|
|
29
34
|
def configure(conf)
|
|
30
35
|
super
|
|
31
|
-
@extractor = QueryParamsExtractor.new(self, conf)
|
|
36
|
+
@extractor = Fluent::Plugin::QueryParamsExtractor.new(self, conf)
|
|
32
37
|
end
|
|
33
38
|
|
|
34
39
|
def filter_record(tag, time, record)
|
|
@@ -36,14 +41,12 @@ module Fluent
|
|
|
36
41
|
super(tag, time, record)
|
|
37
42
|
end
|
|
38
43
|
|
|
39
|
-
def
|
|
44
|
+
def process(tag, es)
|
|
40
45
|
es.each do |time, record|
|
|
41
46
|
t = tag.dup
|
|
42
47
|
filter_record(t, time, record)
|
|
43
48
|
router.emit(t, time, record)
|
|
44
49
|
end
|
|
45
|
-
|
|
46
|
-
chain.next
|
|
47
50
|
end
|
|
48
51
|
end
|
|
49
52
|
end
|
|
@@ -2,7 +2,7 @@ require 'uri'
|
|
|
2
2
|
require 'cgi/util'
|
|
3
3
|
require 'webrick'
|
|
4
4
|
|
|
5
|
-
module Fluent
|
|
5
|
+
module Fluent::Plugin
|
|
6
6
|
class QueryParamsExtractor
|
|
7
7
|
|
|
8
8
|
attr_reader :log
|
|
@@ -10,9 +10,9 @@ module Fluent
|
|
|
10
10
|
def initialize(plugin, conf)
|
|
11
11
|
@log = plugin.log
|
|
12
12
|
|
|
13
|
-
if plugin.is_a?(Fluent::Output)
|
|
13
|
+
if plugin.is_a?(Fluent::Plugin::Output)
|
|
14
14
|
unless have_tag_option?(plugin)
|
|
15
|
-
raise ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
|
|
15
|
+
raise Fluent::ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
|
|
16
16
|
end
|
|
17
17
|
end
|
|
18
18
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
require 'test_helper'
|
|
2
|
+
require 'fluent/plugin/filter_extract_query_params'
|
|
2
3
|
|
|
3
|
-
if Gem::Version.new(Fluent::VERSION) > Gem::Version.new('0.12')
|
|
4
4
|
class ExtractQueryParamsFilterTest < Test::Unit::TestCase
|
|
5
5
|
URL = 'http://example.com:80/?foo=bar&baz=qux&%E3%83%A2%E3%83%AA%E3%82%B9=%E3%81%99%E3%81%9F%E3%81%98%E3%81%8A'
|
|
6
6
|
QUERY_ONLY = '?foo=bar&baz=qux&%E3%83%A2%E3%83%AA%E3%82%B9=%E3%81%99%E3%81%9F%E3%81%98%E3%81%8A'
|
|
@@ -10,21 +10,20 @@ class ExtractQueryParamsFilterTest < Test::Unit::TestCase
|
|
|
10
10
|
@time = Fluent::Engine.now
|
|
11
11
|
end
|
|
12
12
|
|
|
13
|
-
def create_driver(conf
|
|
14
|
-
Fluent::Test::
|
|
15
|
-
Fluent::ExtractQueryParamsFilter
|
|
13
|
+
def create_driver(conf)
|
|
14
|
+
Fluent::Test::Driver::Filter.new(
|
|
15
|
+
Fluent::Plugin::ExtractQueryParamsFilter
|
|
16
16
|
).configure(conf)
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
def filter(config, messages)
|
|
20
|
-
d = create_driver(config
|
|
21
|
-
d.run {
|
|
20
|
+
d = create_driver(config)
|
|
21
|
+
d.run(default_tag: "test") {
|
|
22
22
|
messages.each {|message|
|
|
23
|
-
d.
|
|
23
|
+
d.feed(@time, message)
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
|
-
|
|
27
|
-
filtered.map {|m| m[2] }
|
|
26
|
+
d.filtered_records
|
|
28
27
|
end
|
|
29
28
|
|
|
30
29
|
def test_configure
|
|
@@ -329,4 +328,3 @@ class ExtractQueryParamsFilterTest < Test::Unit::TestCase
|
|
|
329
328
|
assert_equal(expected, filtered[0])
|
|
330
329
|
end
|
|
331
330
|
end
|
|
332
|
-
end
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
# -*- encoding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
require 'test_helper'
|
|
2
|
+
require 'fluent/plugin/out_extract_query_params'
|
|
4
3
|
|
|
5
4
|
class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
6
5
|
URL = 'http://example.com:80/?foo=bar&baz=qux&%E3%83%A2%E3%83%AA%E3%82%B9=%E3%81%99%E3%81%9F%E3%81%98%E3%81%8A'
|
|
@@ -10,9 +9,9 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
10
9
|
Fluent::Test.setup
|
|
11
10
|
end
|
|
12
11
|
|
|
13
|
-
def create_driver(conf
|
|
14
|
-
Fluent::Test::
|
|
15
|
-
Fluent::ExtractQueryParamsOutput
|
|
12
|
+
def create_driver(conf)
|
|
13
|
+
Fluent::Test::Driver::Output.new(
|
|
14
|
+
Fluent::Plugin::ExtractQueryParamsOutput
|
|
16
15
|
).configure(conf)
|
|
17
16
|
end
|
|
18
17
|
|
|
@@ -41,7 +40,6 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
41
40
|
add_tag_prefix extracted.
|
|
42
41
|
])
|
|
43
42
|
|
|
44
|
-
tag = 'test'
|
|
45
43
|
record = {
|
|
46
44
|
'url' => URL,
|
|
47
45
|
}
|
|
@@ -60,7 +58,6 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
60
58
|
add_tag_prefix extracted.
|
|
61
59
|
])
|
|
62
60
|
|
|
63
|
-
tag = 'test'
|
|
64
61
|
record = {
|
|
65
62
|
'url' => URL,
|
|
66
63
|
}
|
|
@@ -82,7 +79,6 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
82
79
|
only foo, baz
|
|
83
80
|
])
|
|
84
81
|
|
|
85
|
-
tag = 'test'
|
|
86
82
|
record = { 'url' => URL }
|
|
87
83
|
d.instance.filter_record('test', Time.now, record)
|
|
88
84
|
|
|
@@ -99,7 +95,6 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
99
95
|
except baz, モリス
|
|
100
96
|
])
|
|
101
97
|
|
|
102
|
-
tag = 'test'
|
|
103
98
|
record = { 'url' => URL }
|
|
104
99
|
d.instance.filter_record('test', Time.now, record)
|
|
105
100
|
|
|
@@ -116,7 +111,6 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
116
111
|
discard_key true
|
|
117
112
|
])
|
|
118
113
|
|
|
119
|
-
tag = 'test'
|
|
120
114
|
record = { 'url' => URL }
|
|
121
115
|
d.instance.filter_record('test', Time.now, record)
|
|
122
116
|
|
|
@@ -133,14 +127,14 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
133
127
|
add_tag_prefix extracted.
|
|
134
128
|
only foo, baz
|
|
135
129
|
])
|
|
136
|
-
d.run { d.
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
assert_equal 1,
|
|
140
|
-
assert_equal 'extracted.test',
|
|
141
|
-
assert_equal URL,
|
|
142
|
-
assert_equal 'bar',
|
|
143
|
-
assert_equal 'qux',
|
|
130
|
+
d.run(default_tag: "test") { d.feed('url' => URL) }
|
|
131
|
+
events = d.events
|
|
132
|
+
|
|
133
|
+
assert_equal 1, events.count
|
|
134
|
+
assert_equal 'extracted.test', events[0][0]
|
|
135
|
+
assert_equal URL, events[0][2]['url']
|
|
136
|
+
assert_equal 'bar', events[0][2]['foo']
|
|
137
|
+
assert_equal 'qux', events[0][2]['baz']
|
|
144
138
|
end
|
|
145
139
|
|
|
146
140
|
def test_emit_multi
|
|
@@ -149,16 +143,16 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
149
143
|
add_tag_prefix extracted.
|
|
150
144
|
only foo, baz
|
|
151
145
|
])
|
|
152
|
-
d.run do
|
|
153
|
-
d.
|
|
154
|
-
d.
|
|
155
|
-
d.
|
|
146
|
+
d.run(default_tag: "test") do
|
|
147
|
+
d.feed('url' => URL)
|
|
148
|
+
d.feed('url' => URL)
|
|
149
|
+
d.feed('url' => URL)
|
|
156
150
|
end
|
|
157
|
-
|
|
151
|
+
events = d.events
|
|
158
152
|
|
|
159
|
-
assert_equal 3,
|
|
153
|
+
assert_equal 3, events.count
|
|
160
154
|
|
|
161
|
-
|
|
155
|
+
events.each do |e|
|
|
162
156
|
assert_equal 'extracted.test', e[0]
|
|
163
157
|
assert_equal URL, e[2]['url']
|
|
164
158
|
assert_equal 'bar', e[2]['foo']
|
|
@@ -172,12 +166,12 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
172
166
|
add_tag_prefix extracted.
|
|
173
167
|
only foo, baz
|
|
174
168
|
])
|
|
175
|
-
d.run { d.
|
|
176
|
-
|
|
169
|
+
d.run(default_tag: "test") { d.feed('url' => URL) }
|
|
170
|
+
events = d.events
|
|
177
171
|
|
|
178
|
-
assert_equal 1,
|
|
179
|
-
assert_equal 'extracted.test',
|
|
180
|
-
assert_equal URL,
|
|
172
|
+
assert_equal 1, events.count
|
|
173
|
+
assert_equal 'extracted.test', events[0][0]
|
|
174
|
+
assert_equal URL, events[0][2]['url']
|
|
181
175
|
end
|
|
182
176
|
|
|
183
177
|
def test_emit_with_invalid_url
|
|
@@ -185,12 +179,12 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
185
179
|
key url
|
|
186
180
|
add_tag_prefix extracted.
|
|
187
181
|
])
|
|
188
|
-
d.run { d.
|
|
189
|
-
|
|
182
|
+
d.run(default_tag: "test") { d.feed('url' => 'invalid url') }
|
|
183
|
+
events = d.events
|
|
190
184
|
|
|
191
|
-
assert_equal 1,
|
|
192
|
-
assert_equal 'extracted.test',
|
|
193
|
-
assert_equal 'invalid url',
|
|
185
|
+
assert_equal 1, events.count
|
|
186
|
+
assert_equal 'extracted.test', events[0][0]
|
|
187
|
+
assert_equal 'invalid url', events[0][2]['url']
|
|
194
188
|
end
|
|
195
189
|
|
|
196
190
|
DIRTY_PATH_BLANK_1 = '/dummy?&baz=qux'
|
|
@@ -212,93 +206,93 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
212
206
|
key path
|
|
213
207
|
add_tag_prefix a.
|
|
214
208
|
])
|
|
215
|
-
d.run {
|
|
216
|
-
d.
|
|
217
|
-
d.
|
|
218
|
-
d.
|
|
219
|
-
d.
|
|
220
|
-
d.
|
|
221
|
-
d.
|
|
222
|
-
d.
|
|
223
|
-
d.
|
|
224
|
-
d.
|
|
225
|
-
d.
|
|
226
|
-
d.
|
|
227
|
-
d.
|
|
228
|
-
d.
|
|
209
|
+
d.run(default_tag: "test") {
|
|
210
|
+
d.feed({ 'path' => DIRTY_PATH_BLANK_1 })
|
|
211
|
+
d.feed({ 'path' => DIRTY_PATH_BLANK_2 })
|
|
212
|
+
d.feed({ 'path' => DIRTY_PATH_BLANK_3 })
|
|
213
|
+
d.feed({ 'path' => DIRTY_PATH_BLANK_4 })
|
|
214
|
+
d.feed({ 'path' => DIRTY_PATH_KEY_ONLY_1 })
|
|
215
|
+
d.feed({ 'path' => DIRTY_PATH_KEY_ONLY_2 })
|
|
216
|
+
d.feed({ 'path' => DIRTY_PATH_KEY_ONLY_3 })
|
|
217
|
+
d.feed({ 'path' => DIRTY_PATH_VALUE_ONLY_1 })
|
|
218
|
+
d.feed({ 'path' => DIRTY_PATH_VALUE_ONLY_2 })
|
|
219
|
+
d.feed({ 'path' => DIRTY_PATH_BASE64_1 })
|
|
220
|
+
d.feed({ 'path' => DIRTY_PATH_BASE64_2 })
|
|
221
|
+
d.feed({ 'path' => DIRTY_PATH_BASE64_3 })
|
|
222
|
+
d.feed({ 'path' => DIRTY_PATH_BASE64_4 })
|
|
229
223
|
}
|
|
230
|
-
|
|
224
|
+
events = d.events
|
|
231
225
|
|
|
232
|
-
assert_equal 13,
|
|
226
|
+
assert_equal 13, events.count
|
|
233
227
|
|
|
234
|
-
r =
|
|
228
|
+
r = events.shift[2]
|
|
235
229
|
assert_equal 2, r.size
|
|
236
230
|
assert_equal DIRTY_PATH_BLANK_1, r['path']
|
|
237
231
|
assert_equal 'qux', r['baz']
|
|
238
232
|
|
|
239
|
-
r =
|
|
233
|
+
r = events.shift[2]
|
|
240
234
|
assert_equal 2, r.size
|
|
241
235
|
assert_equal DIRTY_PATH_BLANK_2, r['path']
|
|
242
236
|
assert_equal 'bar', r['foo']
|
|
243
237
|
|
|
244
|
-
r =
|
|
238
|
+
r = events.shift[2]
|
|
245
239
|
assert_equal 3, r.size
|
|
246
240
|
assert_equal DIRTY_PATH_BLANK_3, r['path']
|
|
247
241
|
assert_equal 'bar', r['foo']
|
|
248
242
|
assert_equal 'qux', r['baz']
|
|
249
243
|
|
|
250
|
-
r =
|
|
244
|
+
r = events.shift[2]
|
|
251
245
|
assert_equal 2, r.size
|
|
252
246
|
assert_equal DIRTY_PATH_BLANK_4, r['path']
|
|
253
247
|
assert_equal 'qux', r['baz']
|
|
254
248
|
|
|
255
|
-
r =
|
|
249
|
+
r = events.shift[2]
|
|
256
250
|
assert_equal 3, r.size
|
|
257
251
|
assert_equal DIRTY_PATH_KEY_ONLY_1, r['path']
|
|
258
252
|
assert_equal '', r['foo']
|
|
259
253
|
assert_equal 'qux', r['baz']
|
|
260
254
|
|
|
261
|
-
r =
|
|
255
|
+
r = events.shift[2]
|
|
262
256
|
assert_equal 3, r.size
|
|
263
257
|
assert_equal DIRTY_PATH_KEY_ONLY_2, r['path']
|
|
264
258
|
assert_equal '', r['foo']
|
|
265
259
|
assert_equal 'qux', r['baz']
|
|
266
260
|
|
|
267
|
-
r =
|
|
261
|
+
r = events.shift[2]
|
|
268
262
|
assert_equal 3, r.size
|
|
269
263
|
assert_equal DIRTY_PATH_KEY_ONLY_3, r['path']
|
|
270
264
|
assert_equal '', r['foo']
|
|
271
265
|
assert_equal 'qux', r['baz']
|
|
272
266
|
|
|
273
|
-
r =
|
|
267
|
+
r = events.shift[2]
|
|
274
268
|
assert_equal 2, r.size
|
|
275
269
|
assert_equal DIRTY_PATH_VALUE_ONLY_1, r['path']
|
|
276
270
|
assert_equal 'qux', r['baz']
|
|
277
271
|
|
|
278
|
-
r =
|
|
272
|
+
r = events.shift[2]
|
|
279
273
|
assert_equal 2, r.size
|
|
280
274
|
assert_equal DIRTY_PATH_VALUE_ONLY_2, r['path']
|
|
281
275
|
assert_equal 'qux', r['baz']
|
|
282
276
|
|
|
283
|
-
r =
|
|
277
|
+
r = events.shift[2]
|
|
284
278
|
assert_equal 3, r.size
|
|
285
279
|
assert_equal DIRTY_PATH_BASE64_1, r['path']
|
|
286
280
|
assert_equal 'qux', r['baz']
|
|
287
281
|
assert_equal 'ZXh0cmE=', r['foo']
|
|
288
282
|
|
|
289
|
-
r =
|
|
283
|
+
r = events.shift[2]
|
|
290
284
|
assert_equal 3, r.size
|
|
291
285
|
assert_equal DIRTY_PATH_BASE64_2, r['path']
|
|
292
286
|
assert_equal 'qux', r['baz']
|
|
293
287
|
assert_equal 'ZXh0cmE=', r['foo']
|
|
294
288
|
|
|
295
|
-
r =
|
|
289
|
+
r = events.shift[2]
|
|
296
290
|
assert_equal 3, r.size
|
|
297
291
|
assert_equal DIRTY_PATH_BASE64_3, r['path']
|
|
298
292
|
assert_equal 'qux', r['baz']
|
|
299
293
|
assert_equal 'cGFkZGluZw==', r['foo']
|
|
300
294
|
|
|
301
|
-
r =
|
|
295
|
+
r = events.shift[2]
|
|
302
296
|
assert_equal 3, r.size
|
|
303
297
|
assert_equal DIRTY_PATH_BASE64_4, r['path']
|
|
304
298
|
assert_equal 'qux', r['baz']
|
|
@@ -311,21 +305,21 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
311
305
|
add_tag_prefix a.
|
|
312
306
|
permit_blank_key yes
|
|
313
307
|
])
|
|
314
|
-
d.run {
|
|
315
|
-
d.
|
|
316
|
-
d.
|
|
308
|
+
d.run(default_tag: "test") {
|
|
309
|
+
d.feed({ 'path' => DIRTY_PATH_VALUE_ONLY_1 })
|
|
310
|
+
d.feed({ 'path' => DIRTY_PATH_VALUE_ONLY_2 })
|
|
317
311
|
}
|
|
318
|
-
|
|
312
|
+
events = d.events
|
|
319
313
|
|
|
320
|
-
assert_equal 2,
|
|
314
|
+
assert_equal 2, events.count
|
|
321
315
|
|
|
322
|
-
r =
|
|
316
|
+
r = events.shift[2]
|
|
323
317
|
assert_equal 3, r.size
|
|
324
318
|
assert_equal DIRTY_PATH_VALUE_ONLY_1, r['path']
|
|
325
319
|
assert_equal 'bar', r['']
|
|
326
320
|
assert_equal 'qux', r['baz']
|
|
327
321
|
|
|
328
|
-
r =
|
|
322
|
+
r = events.shift[2]
|
|
329
323
|
assert_equal 3, r.size
|
|
330
324
|
assert_equal DIRTY_PATH_VALUE_ONLY_2, r['path']
|
|
331
325
|
assert_equal 'bar', r['']
|
|
@@ -341,20 +335,20 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
341
335
|
|
|
342
336
|
raw_multibytes_src = '/path/to/ほげぽす/x?a=b'
|
|
343
337
|
|
|
344
|
-
d.run {
|
|
345
|
-
d.
|
|
346
|
-
d.
|
|
338
|
+
d.run(default_tag: "test") {
|
|
339
|
+
d.feed({ 'path' => raw_multibytes_src.dup.encode('sjis').force_encoding('ascii-8bit') })
|
|
340
|
+
d.feed({ 'path' => raw_multibytes_src.dup.encode('eucjp').force_encoding('ascii-8bit') })
|
|
347
341
|
}
|
|
348
|
-
|
|
342
|
+
events = d.events
|
|
349
343
|
|
|
350
344
|
# nothing raised is correct
|
|
351
|
-
assert_equal 2,
|
|
345
|
+
assert_equal 2, events.count
|
|
352
346
|
|
|
353
|
-
r =
|
|
347
|
+
r = events.shift[2]
|
|
354
348
|
assert_equal 2, r.size
|
|
355
349
|
assert_equal 'b', r['a']
|
|
356
350
|
|
|
357
|
-
r =
|
|
351
|
+
r = events.shift[2]
|
|
358
352
|
assert_equal 2, r.size
|
|
359
353
|
assert_equal 'b', r['a']
|
|
360
354
|
end
|
|
@@ -369,7 +363,6 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
369
363
|
add_url_port true
|
|
370
364
|
add_url_path true
|
|
371
365
|
])
|
|
372
|
-
tag = 'test'
|
|
373
366
|
record = {
|
|
374
367
|
'url' => URL,
|
|
375
368
|
}
|
|
@@ -397,7 +390,6 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
397
390
|
add_url_path true
|
|
398
391
|
])
|
|
399
392
|
|
|
400
|
-
tag = 'test'
|
|
401
393
|
record = {
|
|
402
394
|
'url' => URL,
|
|
403
395
|
}
|
|
@@ -426,7 +418,6 @@ class ExtractQueryParamsOutputTest < Test::Unit::TestCase
|
|
|
426
418
|
add_url_port true
|
|
427
419
|
add_url_path true
|
|
428
420
|
])
|
|
429
|
-
tag = 'test'
|
|
430
421
|
record = {
|
|
431
422
|
'url' => QUERY_ONLY,
|
|
432
423
|
}
|
data/test/test_helper.rb
CHANGED
|
@@ -1,25 +1,9 @@
|
|
|
1
|
+
require 'bundler/setup'
|
|
1
2
|
require 'test/unit'
|
|
2
3
|
|
|
3
|
-
$LOAD_PATH.unshift(File.join(
|
|
4
|
-
$LOAD_PATH.unshift(
|
|
4
|
+
$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
|
|
5
|
+
$LOAD_PATH.unshift(__dir__)
|
|
5
6
|
|
|
6
7
|
require 'fluent/test'
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
nulllogger = Object.new
|
|
10
|
-
nulllogger.instance_eval {|obj|
|
|
11
|
-
def method_missing(method, *args)
|
|
12
|
-
# pass
|
|
13
|
-
end
|
|
14
|
-
}
|
|
15
|
-
$log = nulllogger
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
require 'fluent/plugin/out_extract_query_params'
|
|
19
|
-
|
|
20
|
-
if Gem::Version.new(Fluent::VERSION) > Gem::Version.new('0.12')
|
|
21
|
-
require 'fluent/plugin/filter_extract_query_params'
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
class Test::Unit::TestCase
|
|
25
|
-
end
|
|
8
|
+
require 'fluent/test/driver/output'
|
|
9
|
+
require 'fluent/test/driver/filter'
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-extract_query_params
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0
|
|
4
|
+
version: 0.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Kentaro Kuribayashi
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2016-11-14 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: test-unit
|
|
@@ -39,13 +39,13 @@ dependencies:
|
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: '0'
|
|
41
41
|
- !ruby/object:Gem::Dependency
|
|
42
|
-
name:
|
|
42
|
+
name: appraisal
|
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
|
44
44
|
requirements:
|
|
45
45
|
- - ">="
|
|
46
46
|
- !ruby/object:Gem::Version
|
|
47
47
|
version: '0'
|
|
48
|
-
type: :
|
|
48
|
+
type: :development
|
|
49
49
|
prerelease: false
|
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
51
|
requirements:
|
|
@@ -53,19 +53,25 @@ dependencies:
|
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
54
|
version: '0'
|
|
55
55
|
- !ruby/object:Gem::Dependency
|
|
56
|
-
name:
|
|
56
|
+
name: fluentd
|
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
|
58
58
|
requirements:
|
|
59
59
|
- - ">="
|
|
60
60
|
- !ruby/object:Gem::Version
|
|
61
|
-
version:
|
|
61
|
+
version: 0.14.8
|
|
62
|
+
- - "<"
|
|
63
|
+
- !ruby/object:Gem::Version
|
|
64
|
+
version: '2'
|
|
62
65
|
type: :runtime
|
|
63
66
|
prerelease: false
|
|
64
67
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
68
|
requirements:
|
|
66
69
|
- - ">="
|
|
67
70
|
- !ruby/object:Gem::Version
|
|
68
|
-
version:
|
|
71
|
+
version: 0.14.8
|
|
72
|
+
- - "<"
|
|
73
|
+
- !ruby/object:Gem::Version
|
|
74
|
+
version: '2'
|
|
69
75
|
description: Fluentd plugin to extract key/values from URL query parameters.
|
|
70
76
|
email:
|
|
71
77
|
- kentarok@gmail.com
|
|
@@ -81,8 +87,7 @@ files:
|
|
|
81
87
|
- README.md
|
|
82
88
|
- Rakefile
|
|
83
89
|
- fluent-plugin-extract_query_params.gemspec
|
|
84
|
-
- gemfiles/fluentd_0.
|
|
85
|
-
- gemfiles/fluentd_0.12.gemfile
|
|
90
|
+
- gemfiles/fluentd_0.14.gemfile
|
|
86
91
|
- lib/fluent/plugin/filter_extract_query_params.rb
|
|
87
92
|
- lib/fluent/plugin/out_extract_query_params.rb
|
|
88
93
|
- lib/fluent/plugin/query_params_extractor.rb
|
|
@@ -109,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
109
114
|
version: '0'
|
|
110
115
|
requirements: []
|
|
111
116
|
rubyforge_project:
|
|
112
|
-
rubygems_version: 2.4.5
|
|
117
|
+
rubygems_version: 2.4.5.1
|
|
113
118
|
signing_key:
|
|
114
119
|
specification_version: 4
|
|
115
120
|
summary: Fluentd plugin to extract key/values from URL query parameters
|