fluent-plugin-extract_query_params 0.0.10 → 0.0.11

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9364cbf126771270192d4ccb7e08ab1212233e97
4
- data.tar.gz: 968525fc0c61795056f3b864a4895e275afd33de
3
+ metadata.gz: ac87462a28ad9c0a04464033080272f321863f1d
4
+ data.tar.gz: b4304e2e55078d5d9554b788f4ad95ee1d2d1016
5
5
  SHA512:
6
- metadata.gz: c5645d2cad890c7db0962e862e356c726b0001fe258e65a5b8adef7497557330c7e660b7624b5f321d0d9842c20860e8b766da55f16c7f148a3a43e701cbf0b8
7
- data.tar.gz: 795e13c0383a603ef2efc627e227c674da64d4ca027eecd9f3a5ec43d3406d921a0037ee956e0de0c1bb59d1eb1b5d413af2832df07091912ee080f90512bfd3
6
+ metadata.gz: 2247eb4e072f07d11bb16477de324bae075fcd8cd455401463b007b7503f354374e816ba02d7693a08052c997cee4c5fd9c647d0d88833c5358db1cf4eaf0a66
7
+ data.tar.gz: e90bcefe1aba7baaaa115a79e4d7e91d53b9d2dd2f7f93050ef147bc275f30219148070141bb5ee01d4d5ad90cfae396520a4f163e6f21b188a2c199eb3cbb37
data/.gitignore CHANGED
@@ -3,7 +3,7 @@
3
3
  .bundle
4
4
  .config
5
5
  .yardoc
6
- Gemfile.lock
6
+ *.lock
7
7
  InstalledFiles
8
8
  _yardoc
9
9
  coverage
@@ -3,13 +3,9 @@ language: ruby
3
3
  rvm:
4
4
  - 2.1
5
5
  - 2.2
6
- - rbx
7
6
 
8
7
  gemfile:
9
- - Gemfile
8
+ - gemfiles/fluentd_0.10.gemfile
9
+ - gemfiles/fluentd_0.12.gemfile
10
10
 
11
- script: bundle exec rake test TESTOPTS="-v"
12
-
13
- matrix:
14
- allow_failures:
15
- - rvm: rbx
11
+ script: bundle exec rake test
@@ -0,0 +1,7 @@
1
+ appraise 'fluentd-0.10' do
2
+ gem 'fluentd', '0.10.61' # the last version of 0.10.x
3
+ end
4
+
5
+ appraise 'fluentd-0.12' do
6
+ gem 'rails', '~> 0.12.0'
7
+ end
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # fluent-plugin-extract_query_params, a plugin for [Fluentd](http://fluentd.org)
1
+ # fluent-plugin-extract_query_params, a plugin for [Fluentd](http://fluentd.org) [![](https://travis-ci.org/kentaro/fluent-plugin-extract_query_params.svg)](https://travis-ci.org/kentaro/fluent-plugin-extract_query_params)
2
2
 
3
3
  ## Component
4
4
 
@@ -10,6 +10,8 @@ Fluentd plugin to extract key/values from URL query parameters.
10
10
 
11
11
  Imagin you have a config as below:
12
12
 
13
+ fluentd `< 0.12`:
14
+
13
15
  ```
14
16
  <match test.**>
15
17
  type extract_query_params
@@ -20,6 +22,18 @@ Imagin you have a config as below:
20
22
  </match>
21
23
  ```
22
24
 
25
+ fluentd `>= 0.12`:
26
+
27
+ ```
28
+ <filter test.**>
29
+ type extract_query_params
30
+
31
+ key url
32
+ add_tag_prefix extracted.
33
+ only foo, baz
34
+ </match>
35
+ ```
36
+
23
37
  And you feed such a value into fluentd:
24
38
 
25
39
  ```
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |gem|
2
2
  gem.name = 'fluent-plugin-extract_query_params'
3
- gem.version = '0.0.10'
3
+ gem.version = '0.0.11'
4
4
  gem.authors = ['Kentaro Kuribayashi']
5
5
  gem.email = ['kentarok@gmail.com']
6
6
  gem.homepage = 'http://github.com/kentaro/fluent-plugin-extract_query_params'
@@ -19,5 +19,5 @@ Gem::Specification.new do |gem|
19
19
 
20
20
  gem.add_development_dependency 'rake'
21
21
  gem.add_runtime_dependency 'fluentd'
22
+ gem.add_runtime_dependency 'appraisal'
22
23
  end
23
-
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "fluentd", "0.10.61"
6
+
7
+ gemspec :path => "../"
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "rails", "~> 0.12.0"
6
+
7
+ gemspec :path => "../"
@@ -0,0 +1,32 @@
1
+ module Fluent
2
+ class ExtractQueryParamsFilter < Filter
3
+
4
+ Fluent::Plugin.register_filter('extract_query_params', self)
5
+
6
+ config_param :key, :string
7
+ config_param :only, :string, :default => nil
8
+ config_param :except, :string, :default => nil
9
+ config_param :discard_key, :bool, :default => false
10
+ config_param :add_field_prefix, :string, :default => nil
11
+ config_param :permit_blank_key, :bool, :default => false
12
+
13
+ config_param :add_url_scheme, :bool, :default => false
14
+ config_param :add_url_host, :bool, :default => false
15
+ config_param :add_url_port, :bool, :default => false
16
+ config_param :add_url_path, :bool, :default => false
17
+
18
+ def initialize
19
+ super
20
+ require 'fluent/plugin/query_params_extractor'
21
+ end
22
+
23
+ def configure(conf)
24
+ super
25
+ @extractor = QueryParamsExtractor.new(self, conf)
26
+ end
27
+
28
+ def filter(tag, time, record)
29
+ @extractor.add_query_params_field(record)
30
+ end
31
+ end
32
+ end
@@ -1,6 +1,3 @@
1
- require 'uri'
2
- require 'cgi/util'
3
-
4
1
  module Fluent
5
2
  class ExtractQueryParamsOutput < Output
6
3
  include Fluent::HandleTagNameMixin
@@ -25,30 +22,18 @@ module Fluent
25
22
  config_param :add_url_path, :bool, :default => false
26
23
 
27
24
  def initialize
25
+ require 'fluent/plugin/query_params_extractor'
28
26
  super
29
- require 'webrick'
30
27
  end
31
28
 
32
29
  def configure(conf)
33
30
  super
31
+ @extractor = QueryParamsExtractor.new(self, conf)
32
+ end
34
33
 
35
- if (
36
- !remove_tag_prefix &&
37
- !remove_tag_suffix &&
38
- !add_tag_prefix &&
39
- !add_tag_suffix
40
- )
41
- raise ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
42
- end
43
-
44
- @include_keys = only && only.split(/\s*,\s*/).inject({}) do |hash, i|
45
- hash[i] = true
46
- hash
47
- end
48
- @exclude_keys = except && except.split(/\s*,\s*/).inject({}) do |hash, i|
49
- hash[i] = true
50
- hash
51
- end
34
+ def filter_record(tag, time, record)
35
+ record = @extractor.add_query_params_field(record)
36
+ super(tag, time, record)
52
37
  end
53
38
 
54
39
  def emit(tag, es, chain)
@@ -60,64 +45,5 @@ module Fluent
60
45
 
61
46
  chain.next
62
47
  end
63
-
64
- def filter_record(tag, time, record)
65
- if record[key]
66
- begin
67
- url = begin
68
- URI.parse(record[key])
69
- rescue URI::InvalidURIError => e
70
- URI.parse(WEBrick::HTTPUtils.escape(record[key]))
71
- end
72
-
73
- if @add_url_scheme
74
- url_scheme_key = 'url_scheme'
75
- url_scheme_key = @add_field_prefix + url_scheme_key if @add_field_prefix
76
- record[url_scheme_key] = url.scheme || ''
77
- end
78
-
79
- if @add_url_host
80
- url_host_key = 'url_host'
81
- url_host_key = @add_field_prefix + url_host_key if @add_field_prefix
82
- record[url_host_key] = url.host || ''
83
- end
84
-
85
- if @add_url_port
86
- url_port_key = 'url_port'
87
- url_port_key = @add_field_prefix + url_port_key if @add_field_prefix
88
- record[url_port_key] = url.port || ''
89
- end
90
-
91
- if @add_url_path
92
- url_path_key = 'url_path'
93
- url_path_key = @add_field_prefix + url_path_key if @add_field_prefix
94
- record[url_path_key] = url.path || ''
95
- end
96
-
97
- unless url.query.nil?
98
- url.query.split('&').each do |pair|
99
- key, value = pair.split('=', 2).map { |i| CGI.unescape(i) }
100
- next if (key.nil? || key.empty?) && (!@permit_blank_key || value.nil? || value.empty?)
101
- key ||= ''
102
- value ||= ''
103
-
104
- key = @add_field_prefix + key if @add_field_prefix
105
- if only
106
- record[key] = value if @include_keys.has_key?(key)
107
- elsif except
108
- record[key] = value if !@exclude_keys.has_key?(key)
109
- else
110
- record[key] = value
111
- end
112
- end
113
- end
114
- record.delete key if discard_key
115
- rescue URI::InvalidURIError => error
116
- $log.warn("out_extract_query_params: #{error.message}")
117
- end
118
- end
119
-
120
- super(tag, time, record)
121
- end
122
48
  end
123
49
  end
@@ -0,0 +1,132 @@
1
+ require 'uri'
2
+ require 'cgi/util'
3
+ require 'webrick'
4
+
5
+ module Fluent
6
+ class QueryParamsExtractor
7
+
8
+ attr_reader :log
9
+
10
+ def initialize(plugin, conf)
11
+ @log = plugin.log
12
+
13
+ if plugin.instance_of?(Fluent::ExtractQueryParamsOutput)
14
+ unless have_tag_option?(plugin)
15
+ raise ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
16
+ end
17
+ end
18
+
19
+ @key = plugin.key
20
+ @only = plugin.only
21
+ @except = plugin.except
22
+ @discard_key = plugin.discard_key
23
+ @add_field_prefix = plugin.add_field_prefix
24
+ @permit_blank_key = plugin.permit_blank_key
25
+
26
+ @add_url_scheme = plugin.add_url_scheme
27
+ @add_url_host = plugin.add_url_host
28
+ @add_url_port = plugin.add_url_port
29
+ @add_url_path = plugin.add_url_path
30
+
31
+ if @only
32
+ @include_keys = @only.split(/\s*,\s*/).inject({}) do |hash, i|
33
+ hash[i] = true
34
+ hash
35
+ end
36
+ end
37
+
38
+ if @except
39
+ @exclude_keys = @except.split(/\s*,\s*/).inject({}) do |hash, i|
40
+ hash[i] = true
41
+ hash
42
+ end
43
+ end
44
+ end
45
+
46
+ def add_query_params_field(record)
47
+ return record unless record[@key]
48
+ url = parse_url(record[@key])
49
+ add_url_scheme(url, record)
50
+ add_url_host(url, record)
51
+ add_url_port(url, record)
52
+ add_url_path(url, record)
53
+ add_query_params(url, record)
54
+ record.delete(@key) if @discard_key
55
+ record
56
+ end
57
+
58
+ private
59
+
60
+ def have_tag_option?(plugin)
61
+ plugin.remove_tag_prefix ||
62
+ plugin.remove_tag_suffix ||
63
+ plugin.add_tag_prefix ||
64
+ plugin.add_tag_suffix
65
+ end
66
+
67
+ def parse_url(url_string)
68
+ URI.parse(url_string)
69
+ rescue URI::InvalidURIError
70
+ URI.parse(WEBrick::HTTPUtils.escape(url_string))
71
+ end
72
+
73
+ def create_field_key(field_key)
74
+ if add_field_prefix?
75
+ "#{@add_field_prefix}#{field_key}"
76
+ else
77
+ field_key
78
+ end
79
+ end
80
+
81
+ def add_url_scheme(url, record)
82
+ return unless @add_url_scheme
83
+ url_scheme_key = create_field_key('url_scheme')
84
+ record[url_scheme_key] = url.scheme || ''
85
+ end
86
+
87
+ def add_url_host(url, record)
88
+ return unless @add_url_host
89
+ url_host_key = create_field_key('url_host')
90
+ record[url_host_key] = url.host || ''
91
+ end
92
+
93
+ def add_url_port(url, record)
94
+ return unless @add_url_port
95
+ url_port_key = create_field_key('url_port')
96
+ record[url_port_key] = url.port || ''
97
+ end
98
+
99
+ def add_url_path(url, record)
100
+ return unless @add_url_path
101
+ url_path_key = create_field_key('url_path')
102
+ record[url_path_key] = url.path || ''
103
+ end
104
+
105
+ def add_field_prefix?
106
+ !!@add_field_prefix
107
+ end
108
+
109
+ def permit_blank_key?
110
+ @permit_blank_key
111
+ end
112
+
113
+ def add_query_params(url, record)
114
+ return if url.query.nil?
115
+ url.query.split('&').each do |pair|
116
+ key, value = pair.split('=', 2).map { |i| CGI.unescape(i) }
117
+ next if (key.nil? || key.empty?) && (!permit_blank_key? || value.nil? || value.empty?)
118
+ key ||= ''
119
+ value ||= ''
120
+
121
+ key = create_field_key(key)
122
+ if @only
123
+ record[key] = value if @include_keys.has_key?(key)
124
+ elsif @except
125
+ record[key] = value if !@exclude_keys.has_key?(key)
126
+ else
127
+ record[key] = value
128
+ end
129
+ end
130
+ end
131
+ end
132
+ end
@@ -0,0 +1,332 @@
1
+ require 'test_helper'
2
+
3
+ if Gem::Version.new(Fluent::VERSION) > Gem::Version.new('0.12')
4
+ class ExtractQueryParamsFilterTest < Test::Unit::TestCase
5
+ URL = 'http://example.com:80/?foo=bar&baz=qux&%E3%83%A2%E3%83%AA%E3%82%B9=%E3%81%99%E3%81%9F%E3%81%98%E3%81%8A'
6
+ QUERY_ONLY = '?foo=bar&baz=qux&%E3%83%A2%E3%83%AA%E3%82%B9=%E3%81%99%E3%81%9F%E3%81%98%E3%81%8A'
7
+
8
+ def setup
9
+ Fluent::Test.setup
10
+ @time = Fluent::Engine.now
11
+ end
12
+
13
+ def create_driver(conf, tag = 'test')
14
+ Fluent::Test::FilterTestDriver.new(
15
+ Fluent::ExtractQueryParamsFilter, tag
16
+ ).configure(conf)
17
+ end
18
+
19
+ def filter(config, messages)
20
+ d = create_driver(config, 'test')
21
+ d.run {
22
+ messages.each {|message|
23
+ d.filter(message, @time)
24
+ }
25
+ }
26
+ filtered = d.filtered_as_array
27
+ filtered.map {|m| m[2] }
28
+ end
29
+
30
+ def test_configure
31
+ d = create_driver(%[
32
+ key url
33
+ only foo, baz
34
+ ])
35
+
36
+ assert_equal 'url', d.instance.key
37
+ assert_equal 'foo, baz', d.instance.only
38
+ end
39
+
40
+ def test_filter
41
+ config = %[
42
+ key url
43
+ ]
44
+
45
+ record = {
46
+ 'url' => URL,
47
+ }
48
+ expected = {
49
+ 'url' => URL,
50
+ 'foo' => 'bar',
51
+ 'baz' => 'qux',
52
+ 'モリス' => 'すたじお'
53
+ }
54
+ filtered = filter(config, [record])
55
+ assert_equal(expected, filtered[0])
56
+ end
57
+
58
+ def test_filter_with_field_prefix
59
+ config = %[
60
+ key url
61
+ add_field_prefix query_
62
+ ]
63
+
64
+ record = {
65
+ 'url' => URL,
66
+ }
67
+ expected = {
68
+ 'url' => URL,
69
+ 'query_foo' => 'bar',
70
+ 'query_baz' => 'qux',
71
+ 'query_モリス' => 'すたじお'
72
+ }
73
+ filtered = filter(config, [record])
74
+ assert_equal(expected, filtered[0])
75
+ end
76
+
77
+ def test_filter_with_only
78
+ config = %[
79
+ key url
80
+ only foo, baz
81
+ ]
82
+
83
+ record = { 'url' => URL }
84
+ expected = {
85
+ 'url' => URL,
86
+ 'foo' => 'bar',
87
+ 'baz' => 'qux',
88
+ }
89
+ filtered = filter(config, [record])
90
+ assert_equal(expected, filtered[0])
91
+ end
92
+
93
+ def test_filter_with_except
94
+ config = %[
95
+ key url
96
+ except baz, モリス
97
+ ]
98
+
99
+ record = { 'url' => URL }
100
+ expected = {
101
+ 'url' => URL,
102
+ 'foo' => 'bar',
103
+ }
104
+ filtered = filter(config, [record])
105
+ assert_equal(expected, filtered[0])
106
+ end
107
+
108
+ def test_filter_with_discard
109
+ config = %[
110
+ key url
111
+ discard_key true
112
+ ]
113
+
114
+ record = { 'url' => URL }
115
+ expected = {
116
+ 'foo' => 'bar',
117
+ 'baz' => 'qux',
118
+ 'モリス' => 'すたじお'
119
+ }
120
+ filtered = filter(config, [record])
121
+ assert_equal(expected, filtered[0])
122
+ end
123
+
124
+ def test_filter_multi_records
125
+ config = %[
126
+ key url
127
+ only foo, baz
128
+ ]
129
+ records = [
130
+ { 'url' => URL },
131
+ { 'url' => URL },
132
+ { 'url' => URL }
133
+ ]
134
+ expected = [
135
+ { 'url' => URL, 'foo' => 'bar', 'baz' => 'qux' },
136
+ { 'url' => URL, 'foo' => 'bar', 'baz' => 'qux' },
137
+ { 'url' => URL, 'foo' => 'bar', 'baz' => 'qux' }
138
+ ]
139
+ filtered = filter(config, records)
140
+ assert_equal(expected, filtered)
141
+ end
142
+
143
+ def test_emit_without_match_key
144
+ config = %[
145
+ key no_such_key
146
+ only foo, baz
147
+ ]
148
+ record = { 'url' => URL }
149
+ filtered = filter(config, [record])
150
+ assert_equal(record, filtered[0])
151
+ end
152
+
153
+ def test_emit_with_invalid_url
154
+ config = %[
155
+ key url
156
+ ]
157
+ record = { 'url' => URL }
158
+ filtered = filter(config, [record])
159
+ assert_equal([record], filtered)
160
+ end
161
+
162
+ DIRTY_PATH_BLANK_1 = '/dummy?&baz=qux'
163
+ DIRTY_PATH_BLANK_2 = '/dummy?foo=bar&'
164
+ DIRTY_PATH_BLANK_3 = '/dummy?foo=bar&&baz=qux'
165
+ DIRTY_PATH_BLANK_4 = '/dummy?=&baz=qux'
166
+ DIRTY_PATH_KEY_ONLY_1 = '/dummy?foo=&baz=qux'
167
+ DIRTY_PATH_KEY_ONLY_2 = '/dummy?foo&baz=qux'
168
+ DIRTY_PATH_KEY_ONLY_3 = '/dummy?baz=qux&foo'
169
+ DIRTY_PATH_VALUE_ONLY_1 = '/dummy?=bar&baz=qux'
170
+ DIRTY_PATH_VALUE_ONLY_2 = '/dummy?baz=qux&=bar'
171
+ DIRTY_PATH_BASE64_1 = '/dummy?foo=ZXh0cmE=&baz=qux'
172
+ DIRTY_PATH_BASE64_2 = '/dummy?baz=qux&foo=ZXh0cmE='
173
+ DIRTY_PATH_BASE64_3 = '/dummy?foo=cGFkZGluZw==&baz=qux'
174
+ DIRTY_PATH_BASE64_4 = '/dummy?baz=qux&foo=cGFkZGluZw=='
175
+
176
+ def test_emit_with_dirty_paths
177
+ config = %[
178
+ key path
179
+ ]
180
+ records = [
181
+ { 'path' => DIRTY_PATH_BLANK_1 },
182
+ { 'path' => DIRTY_PATH_BLANK_2 },
183
+ { 'path' => DIRTY_PATH_BLANK_3 },
184
+ { 'path' => DIRTY_PATH_BLANK_4 },
185
+ { 'path' => DIRTY_PATH_KEY_ONLY_1 },
186
+ { 'path' => DIRTY_PATH_KEY_ONLY_2 },
187
+ { 'path' => DIRTY_PATH_KEY_ONLY_3 },
188
+ { 'path' => DIRTY_PATH_VALUE_ONLY_1 },
189
+ { 'path' => DIRTY_PATH_VALUE_ONLY_2 },
190
+ { 'path' => DIRTY_PATH_BASE64_1 },
191
+ { 'path' => DIRTY_PATH_BASE64_2 },
192
+ { 'path' => DIRTY_PATH_BASE64_3 },
193
+ { 'path' => DIRTY_PATH_BASE64_4 }
194
+ ]
195
+ expected = [
196
+ { 'path' => DIRTY_PATH_BLANK_1, 'baz' => 'qux' },
197
+ { 'path' => DIRTY_PATH_BLANK_2, 'foo' => 'bar' },
198
+ { 'path' => DIRTY_PATH_BLANK_3, 'foo' => 'bar', 'baz' => 'qux' },
199
+ { 'path' => DIRTY_PATH_BLANK_4, 'baz' => 'qux' },
200
+ { 'path' => DIRTY_PATH_KEY_ONLY_1, 'foo' => '', 'baz' => 'qux' },
201
+ { 'path' => DIRTY_PATH_KEY_ONLY_2, 'foo' => '', 'baz' => 'qux' },
202
+ { 'path' => DIRTY_PATH_KEY_ONLY_3, 'foo' => '', 'baz' => 'qux' },
203
+ { 'path' => DIRTY_PATH_VALUE_ONLY_1, 'baz' => 'qux' },
204
+ { 'path' => DIRTY_PATH_VALUE_ONLY_2, 'baz' => 'qux' },
205
+ { 'path' => DIRTY_PATH_BASE64_1, 'baz' => 'qux', 'foo' => 'ZXh0cmE=' },
206
+ { 'path' => DIRTY_PATH_BASE64_2, 'baz' => 'qux', 'foo' => 'ZXh0cmE=' },
207
+ { 'path' => DIRTY_PATH_BASE64_3, 'baz' => 'qux', 'foo' => 'cGFkZGluZw==' },
208
+ { 'path' => DIRTY_PATH_BASE64_4, 'baz' => 'qux', 'foo' => 'cGFkZGluZw==' }
209
+ ]
210
+ filtered = filter(config, records)
211
+ assert_equal(expected, filtered)
212
+ end
213
+
214
+ def test_emit_with_permit_blank_key
215
+ config = %[
216
+ key path
217
+ permit_blank_key yes
218
+ ]
219
+ records = [
220
+ { 'path' => DIRTY_PATH_VALUE_ONLY_1 },
221
+ { 'path' => DIRTY_PATH_VALUE_ONLY_2 }
222
+ ]
223
+ expected = [
224
+ { 'path' => DIRTY_PATH_VALUE_ONLY_1, '' => 'bar', 'baz' => 'qux' },
225
+ { 'path' => DIRTY_PATH_VALUE_ONLY_2, '' => 'bar', 'baz' => 'qux' }
226
+ ]
227
+ filtered = filter(config, records)
228
+ assert_equal(expected, filtered)
229
+ end
230
+
231
+ def test_raw_multibyte_chars
232
+ config = %[
233
+ key path
234
+ permit_blank_key yes
235
+ ]
236
+
237
+ raw_multibytes_src = '/path/to/ほげぽす/x?a=b'
238
+
239
+ records = [
240
+ { 'path' => raw_multibytes_src.dup.encode('sjis').force_encoding('ascii-8bit') },
241
+ { 'path' => raw_multibytes_src.dup.encode('eucjp').force_encoding('ascii-8bit') }
242
+ ]
243
+ expected = [
244
+ { 'path' => raw_multibytes_src.dup.encode('sjis').force_encoding('ascii-8bit'), 'a' => 'b' },
245
+ { 'path' => raw_multibytes_src.dup.encode('eucjp').force_encoding('ascii-8bit'), 'a' => 'b' }
246
+ ]
247
+ filtered = filter(config, records)
248
+ assert_equal(expected, filtered)
249
+ end
250
+
251
+ def test_filter_with_url_scheme_host_port_path
252
+ config = %[
253
+ key url
254
+
255
+ add_url_scheme true
256
+ add_url_host true
257
+ add_url_port true
258
+ add_url_path true
259
+ ]
260
+
261
+ record = {
262
+ 'url' => URL,
263
+ }
264
+ expected = {
265
+ 'url' => URL,
266
+ 'foo' => 'bar',
267
+ 'baz' => 'qux',
268
+ 'モリス' => 'すたじお',
269
+ 'url_scheme' => 'http',
270
+ 'url_host' => 'example.com',
271
+ 'url_port' => 80,
272
+ 'url_path' => '/'
273
+ }
274
+ filtered = filter(config, [record])
275
+ assert_equal(expected, filtered[0])
276
+ end
277
+
278
+ def test_filter_with_field_prefix_and_url_scheme_host_port_path
279
+ config = %[
280
+ key url
281
+ add_field_prefix query_
282
+
283
+ add_url_scheme true
284
+ add_url_host true
285
+ add_url_port true
286
+ add_url_path true
287
+ ]
288
+
289
+ record = {
290
+ 'url' => URL,
291
+ }
292
+ expected = {
293
+ 'url' => URL,
294
+ 'query_foo' => 'bar',
295
+ 'query_baz' => 'qux',
296
+ 'query_モリス' => 'すたじお',
297
+ 'query_url_scheme' => 'http',
298
+ 'query_url_host' => 'example.com',
299
+ 'query_url_port' => 80,
300
+ 'query_url_path' => '/'
301
+ }
302
+ filtered = filter(config, [record])
303
+ assert_equal(expected, filtered[0])
304
+ end
305
+
306
+ def test_filter_from_query_only_url_with_url_scheme_host_port_path
307
+ config = %[
308
+ key url
309
+
310
+ add_url_scheme true
311
+ add_url_host true
312
+ add_url_port true
313
+ add_url_path true
314
+ ]
315
+ record = {
316
+ 'url' => QUERY_ONLY,
317
+ }
318
+ expected = {
319
+ 'url' => QUERY_ONLY,
320
+ 'foo' => 'bar',
321
+ 'baz' => 'qux',
322
+ 'モリス' => 'すたじお',
323
+ 'url_scheme' => '',
324
+ 'url_host' => '',
325
+ 'url_port' => '',
326
+ 'url_path' => ''
327
+ }
328
+ filtered = filter(config, [record])
329
+ assert_equal(expected, filtered[0])
330
+ end
331
+ end
332
+ end
@@ -17,5 +17,9 @@ end
17
17
 
18
18
  require 'fluent/plugin/out_extract_query_params'
19
19
 
20
+ if Gem::Version.new(Fluent::VERSION) > Gem::Version.new('0.12')
21
+ require 'fluent/plugin/filter_extract_query_params'
22
+ end
23
+
20
24
  class Test::Unit::TestCase
21
25
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-extract_query_params
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.10
4
+ version: 0.0.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kentaro Kuribayashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-09-04 00:00:00.000000000 Z
11
+ date: 2015-09-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: test-unit
@@ -52,6 +52,20 @@ dependencies:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: appraisal
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
55
69
  description: Fluentd plugin to extract key/values from URL query parameters.
56
70
  email:
57
71
  - kentarok@gmail.com
@@ -61,12 +75,18 @@ extra_rdoc_files: []
61
75
  files:
62
76
  - ".gitignore"
63
77
  - ".travis.yml"
78
+ - Appraisals
64
79
  - Gemfile
65
80
  - LICENSE
66
81
  - README.md
67
82
  - Rakefile
68
83
  - fluent-plugin-extract_query_params.gemspec
84
+ - gemfiles/fluentd_0.10.gemfile
85
+ - gemfiles/fluentd_0.12.gemfile
86
+ - lib/fluent/plugin/filter_extract_query_params.rb
69
87
  - lib/fluent/plugin/out_extract_query_params.rb
88
+ - lib/fluent/plugin/query_params_extractor.rb
89
+ - test/plugin/test_filter_extract_query_params.rb
70
90
  - test/plugin/test_out_extract_query_params.rb
71
91
  - test/test_helper.rb
72
92
  homepage: http://github.com/kentaro/fluent-plugin-extract_query_params
@@ -94,5 +114,6 @@ signing_key:
94
114
  specification_version: 4
95
115
  summary: Fluentd plugin to extract key/values from URL query parameters
96
116
  test_files:
117
+ - test/plugin/test_filter_extract_query_params.rb
97
118
  - test/plugin/test_out_extract_query_params.rb
98
119
  - test/test_helper.rb