fluent-plugin-extract_query_params 0.0.10 → 0.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9364cbf126771270192d4ccb7e08ab1212233e97
4
- data.tar.gz: 968525fc0c61795056f3b864a4895e275afd33de
3
+ metadata.gz: ac87462a28ad9c0a04464033080272f321863f1d
4
+ data.tar.gz: b4304e2e55078d5d9554b788f4ad95ee1d2d1016
5
5
  SHA512:
6
- metadata.gz: c5645d2cad890c7db0962e862e356c726b0001fe258e65a5b8adef7497557330c7e660b7624b5f321d0d9842c20860e8b766da55f16c7f148a3a43e701cbf0b8
7
- data.tar.gz: 795e13c0383a603ef2efc627e227c674da64d4ca027eecd9f3a5ec43d3406d921a0037ee956e0de0c1bb59d1eb1b5d413af2832df07091912ee080f90512bfd3
6
+ metadata.gz: 2247eb4e072f07d11bb16477de324bae075fcd8cd455401463b007b7503f354374e816ba02d7693a08052c997cee4c5fd9c647d0d88833c5358db1cf4eaf0a66
7
+ data.tar.gz: e90bcefe1aba7baaaa115a79e4d7e91d53b9d2dd2f7f93050ef147bc275f30219148070141bb5ee01d4d5ad90cfae396520a4f163e6f21b188a2c199eb3cbb37
data/.gitignore CHANGED
@@ -3,7 +3,7 @@
3
3
  .bundle
4
4
  .config
5
5
  .yardoc
6
- Gemfile.lock
6
+ *.lock
7
7
  InstalledFiles
8
8
  _yardoc
9
9
  coverage
@@ -3,13 +3,9 @@ language: ruby
3
3
  rvm:
4
4
  - 2.1
5
5
  - 2.2
6
- - rbx
7
6
 
8
7
  gemfile:
9
- - Gemfile
8
+ - gemfiles/fluentd_0.10.gemfile
9
+ - gemfiles/fluentd_0.12.gemfile
10
10
 
11
- script: bundle exec rake test TESTOPTS="-v"
12
-
13
- matrix:
14
- allow_failures:
15
- - rvm: rbx
11
+ script: bundle exec rake test
@@ -0,0 +1,7 @@
1
+ appraise 'fluentd-0.10' do
2
+ gem 'fluentd', '0.10.61' # the last version of 0.10.x
3
+ end
4
+
5
+ appraise 'fluentd-0.12' do
6
+ gem 'rails', '~> 0.12.0'
7
+ end
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # fluent-plugin-extract_query_params, a plugin for [Fluentd](http://fluentd.org)
1
+ # fluent-plugin-extract_query_params, a plugin for [Fluentd](http://fluentd.org) [![](https://travis-ci.org/kentaro/fluent-plugin-extract_query_params.svg)](https://travis-ci.org/kentaro/fluent-plugin-extract_query_params)
2
2
 
3
3
  ## Component
4
4
 
@@ -10,6 +10,8 @@ Fluentd plugin to extract key/values from URL query parameters.
10
10
 
11
11
  Imagin you have a config as below:
12
12
 
13
+ fluentd `< 0.12`:
14
+
13
15
  ```
14
16
  <match test.**>
15
17
  type extract_query_params
@@ -20,6 +22,18 @@ Imagin you have a config as below:
20
22
  </match>
21
23
  ```
22
24
 
25
+ fluentd `>= 0.12`:
26
+
27
+ ```
28
+ <filter test.**>
29
+ type extract_query_params
30
+
31
+ key url
32
+ add_tag_prefix extracted.
33
+ only foo, baz
34
+ </match>
35
+ ```
36
+
23
37
  And you feed such a value into fluentd:
24
38
 
25
39
  ```
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |gem|
2
2
  gem.name = 'fluent-plugin-extract_query_params'
3
- gem.version = '0.0.10'
3
+ gem.version = '0.0.11'
4
4
  gem.authors = ['Kentaro Kuribayashi']
5
5
  gem.email = ['kentarok@gmail.com']
6
6
  gem.homepage = 'http://github.com/kentaro/fluent-plugin-extract_query_params'
@@ -19,5 +19,5 @@ Gem::Specification.new do |gem|
19
19
 
20
20
  gem.add_development_dependency 'rake'
21
21
  gem.add_runtime_dependency 'fluentd'
22
+ gem.add_runtime_dependency 'appraisal'
22
23
  end
23
-
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "fluentd", "0.10.61"
6
+
7
+ gemspec :path => "../"
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "rails", "~> 0.12.0"
6
+
7
+ gemspec :path => "../"
@@ -0,0 +1,32 @@
1
+ module Fluent
2
+ class ExtractQueryParamsFilter < Filter
3
+
4
+ Fluent::Plugin.register_filter('extract_query_params', self)
5
+
6
+ config_param :key, :string
7
+ config_param :only, :string, :default => nil
8
+ config_param :except, :string, :default => nil
9
+ config_param :discard_key, :bool, :default => false
10
+ config_param :add_field_prefix, :string, :default => nil
11
+ config_param :permit_blank_key, :bool, :default => false
12
+
13
+ config_param :add_url_scheme, :bool, :default => false
14
+ config_param :add_url_host, :bool, :default => false
15
+ config_param :add_url_port, :bool, :default => false
16
+ config_param :add_url_path, :bool, :default => false
17
+
18
+ def initialize
19
+ super
20
+ require 'fluent/plugin/query_params_extractor'
21
+ end
22
+
23
+ def configure(conf)
24
+ super
25
+ @extractor = QueryParamsExtractor.new(self, conf)
26
+ end
27
+
28
+ def filter(tag, time, record)
29
+ @extractor.add_query_params_field(record)
30
+ end
31
+ end
32
+ end
@@ -1,6 +1,3 @@
1
- require 'uri'
2
- require 'cgi/util'
3
-
4
1
  module Fluent
5
2
  class ExtractQueryParamsOutput < Output
6
3
  include Fluent::HandleTagNameMixin
@@ -25,30 +22,18 @@ module Fluent
25
22
  config_param :add_url_path, :bool, :default => false
26
23
 
27
24
  def initialize
25
+ require 'fluent/plugin/query_params_extractor'
28
26
  super
29
- require 'webrick'
30
27
  end
31
28
 
32
29
  def configure(conf)
33
30
  super
31
+ @extractor = QueryParamsExtractor.new(self, conf)
32
+ end
34
33
 
35
- if (
36
- !remove_tag_prefix &&
37
- !remove_tag_suffix &&
38
- !add_tag_prefix &&
39
- !add_tag_suffix
40
- )
41
- raise ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
42
- end
43
-
44
- @include_keys = only && only.split(/\s*,\s*/).inject({}) do |hash, i|
45
- hash[i] = true
46
- hash
47
- end
48
- @exclude_keys = except && except.split(/\s*,\s*/).inject({}) do |hash, i|
49
- hash[i] = true
50
- hash
51
- end
34
+ def filter_record(tag, time, record)
35
+ record = @extractor.add_query_params_field(record)
36
+ super(tag, time, record)
52
37
  end
53
38
 
54
39
  def emit(tag, es, chain)
@@ -60,64 +45,5 @@ module Fluent
60
45
 
61
46
  chain.next
62
47
  end
63
-
64
- def filter_record(tag, time, record)
65
- if record[key]
66
- begin
67
- url = begin
68
- URI.parse(record[key])
69
- rescue URI::InvalidURIError => e
70
- URI.parse(WEBrick::HTTPUtils.escape(record[key]))
71
- end
72
-
73
- if @add_url_scheme
74
- url_scheme_key = 'url_scheme'
75
- url_scheme_key = @add_field_prefix + url_scheme_key if @add_field_prefix
76
- record[url_scheme_key] = url.scheme || ''
77
- end
78
-
79
- if @add_url_host
80
- url_host_key = 'url_host'
81
- url_host_key = @add_field_prefix + url_host_key if @add_field_prefix
82
- record[url_host_key] = url.host || ''
83
- end
84
-
85
- if @add_url_port
86
- url_port_key = 'url_port'
87
- url_port_key = @add_field_prefix + url_port_key if @add_field_prefix
88
- record[url_port_key] = url.port || ''
89
- end
90
-
91
- if @add_url_path
92
- url_path_key = 'url_path'
93
- url_path_key = @add_field_prefix + url_path_key if @add_field_prefix
94
- record[url_path_key] = url.path || ''
95
- end
96
-
97
- unless url.query.nil?
98
- url.query.split('&').each do |pair|
99
- key, value = pair.split('=', 2).map { |i| CGI.unescape(i) }
100
- next if (key.nil? || key.empty?) && (!@permit_blank_key || value.nil? || value.empty?)
101
- key ||= ''
102
- value ||= ''
103
-
104
- key = @add_field_prefix + key if @add_field_prefix
105
- if only
106
- record[key] = value if @include_keys.has_key?(key)
107
- elsif except
108
- record[key] = value if !@exclude_keys.has_key?(key)
109
- else
110
- record[key] = value
111
- end
112
- end
113
- end
114
- record.delete key if discard_key
115
- rescue URI::InvalidURIError => error
116
- $log.warn("out_extract_query_params: #{error.message}")
117
- end
118
- end
119
-
120
- super(tag, time, record)
121
- end
122
48
  end
123
49
  end
@@ -0,0 +1,132 @@
1
+ require 'uri'
2
+ require 'cgi/util'
3
+ require 'webrick'
4
+
5
+ module Fluent
6
+ class QueryParamsExtractor
7
+
8
+ attr_reader :log
9
+
10
+ def initialize(plugin, conf)
11
+ @log = plugin.log
12
+
13
+ if plugin.instance_of?(Fluent::ExtractQueryParamsOutput)
14
+ unless have_tag_option?(plugin)
15
+ raise ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
16
+ end
17
+ end
18
+
19
+ @key = plugin.key
20
+ @only = plugin.only
21
+ @except = plugin.except
22
+ @discard_key = plugin.discard_key
23
+ @add_field_prefix = plugin.add_field_prefix
24
+ @permit_blank_key = plugin.permit_blank_key
25
+
26
+ @add_url_scheme = plugin.add_url_scheme
27
+ @add_url_host = plugin.add_url_host
28
+ @add_url_port = plugin.add_url_port
29
+ @add_url_path = plugin.add_url_path
30
+
31
+ if @only
32
+ @include_keys = @only.split(/\s*,\s*/).inject({}) do |hash, i|
33
+ hash[i] = true
34
+ hash
35
+ end
36
+ end
37
+
38
+ if @except
39
+ @exclude_keys = @except.split(/\s*,\s*/).inject({}) do |hash, i|
40
+ hash[i] = true
41
+ hash
42
+ end
43
+ end
44
+ end
45
+
46
+ def add_query_params_field(record)
47
+ return record unless record[@key]
48
+ url = parse_url(record[@key])
49
+ add_url_scheme(url, record)
50
+ add_url_host(url, record)
51
+ add_url_port(url, record)
52
+ add_url_path(url, record)
53
+ add_query_params(url, record)
54
+ record.delete(@key) if @discard_key
55
+ record
56
+ end
57
+
58
+ private
59
+
60
+ def have_tag_option?(plugin)
61
+ plugin.remove_tag_prefix ||
62
+ plugin.remove_tag_suffix ||
63
+ plugin.add_tag_prefix ||
64
+ plugin.add_tag_suffix
65
+ end
66
+
67
+ def parse_url(url_string)
68
+ URI.parse(url_string)
69
+ rescue URI::InvalidURIError
70
+ URI.parse(WEBrick::HTTPUtils.escape(url_string))
71
+ end
72
+
73
+ def create_field_key(field_key)
74
+ if add_field_prefix?
75
+ "#{@add_field_prefix}#{field_key}"
76
+ else
77
+ field_key
78
+ end
79
+ end
80
+
81
+ def add_url_scheme(url, record)
82
+ return unless @add_url_scheme
83
+ url_scheme_key = create_field_key('url_scheme')
84
+ record[url_scheme_key] = url.scheme || ''
85
+ end
86
+
87
+ def add_url_host(url, record)
88
+ return unless @add_url_host
89
+ url_host_key = create_field_key('url_host')
90
+ record[url_host_key] = url.host || ''
91
+ end
92
+
93
+ def add_url_port(url, record)
94
+ return unless @add_url_port
95
+ url_port_key = create_field_key('url_port')
96
+ record[url_port_key] = url.port || ''
97
+ end
98
+
99
+ def add_url_path(url, record)
100
+ return unless @add_url_path
101
+ url_path_key = create_field_key('url_path')
102
+ record[url_path_key] = url.path || ''
103
+ end
104
+
105
+ def add_field_prefix?
106
+ !!@add_field_prefix
107
+ end
108
+
109
+ def permit_blank_key?
110
+ @permit_blank_key
111
+ end
112
+
113
+ def add_query_params(url, record)
114
+ return if url.query.nil?
115
+ url.query.split('&').each do |pair|
116
+ key, value = pair.split('=', 2).map { |i| CGI.unescape(i) }
117
+ next if (key.nil? || key.empty?) && (!permit_blank_key? || value.nil? || value.empty?)
118
+ key ||= ''
119
+ value ||= ''
120
+
121
+ key = create_field_key(key)
122
+ if @only
123
+ record[key] = value if @include_keys.has_key?(key)
124
+ elsif @except
125
+ record[key] = value if !@exclude_keys.has_key?(key)
126
+ else
127
+ record[key] = value
128
+ end
129
+ end
130
+ end
131
+ end
132
+ end
@@ -0,0 +1,332 @@
1
+ require 'test_helper'
2
+
3
+ if Gem::Version.new(Fluent::VERSION) > Gem::Version.new('0.12')
4
+ class ExtractQueryParamsFilterTest < Test::Unit::TestCase
5
+ URL = 'http://example.com:80/?foo=bar&baz=qux&%E3%83%A2%E3%83%AA%E3%82%B9=%E3%81%99%E3%81%9F%E3%81%98%E3%81%8A'
6
+ QUERY_ONLY = '?foo=bar&baz=qux&%E3%83%A2%E3%83%AA%E3%82%B9=%E3%81%99%E3%81%9F%E3%81%98%E3%81%8A'
7
+
8
+ def setup
9
+ Fluent::Test.setup
10
+ @time = Fluent::Engine.now
11
+ end
12
+
13
+ def create_driver(conf, tag = 'test')
14
+ Fluent::Test::FilterTestDriver.new(
15
+ Fluent::ExtractQueryParamsFilter, tag
16
+ ).configure(conf)
17
+ end
18
+
19
+ def filter(config, messages)
20
+ d = create_driver(config, 'test')
21
+ d.run {
22
+ messages.each {|message|
23
+ d.filter(message, @time)
24
+ }
25
+ }
26
+ filtered = d.filtered_as_array
27
+ filtered.map {|m| m[2] }
28
+ end
29
+
30
+ def test_configure
31
+ d = create_driver(%[
32
+ key url
33
+ only foo, baz
34
+ ])
35
+
36
+ assert_equal 'url', d.instance.key
37
+ assert_equal 'foo, baz', d.instance.only
38
+ end
39
+
40
+ def test_filter
41
+ config = %[
42
+ key url
43
+ ]
44
+
45
+ record = {
46
+ 'url' => URL,
47
+ }
48
+ expected = {
49
+ 'url' => URL,
50
+ 'foo' => 'bar',
51
+ 'baz' => 'qux',
52
+ 'モリス' => 'すたじお'
53
+ }
54
+ filtered = filter(config, [record])
55
+ assert_equal(expected, filtered[0])
56
+ end
57
+
58
+ def test_filter_with_field_prefix
59
+ config = %[
60
+ key url
61
+ add_field_prefix query_
62
+ ]
63
+
64
+ record = {
65
+ 'url' => URL,
66
+ }
67
+ expected = {
68
+ 'url' => URL,
69
+ 'query_foo' => 'bar',
70
+ 'query_baz' => 'qux',
71
+ 'query_モリス' => 'すたじお'
72
+ }
73
+ filtered = filter(config, [record])
74
+ assert_equal(expected, filtered[0])
75
+ end
76
+
77
+ def test_filter_with_only
78
+ config = %[
79
+ key url
80
+ only foo, baz
81
+ ]
82
+
83
+ record = { 'url' => URL }
84
+ expected = {
85
+ 'url' => URL,
86
+ 'foo' => 'bar',
87
+ 'baz' => 'qux',
88
+ }
89
+ filtered = filter(config, [record])
90
+ assert_equal(expected, filtered[0])
91
+ end
92
+
93
+ def test_filter_with_except
94
+ config = %[
95
+ key url
96
+ except baz, モリス
97
+ ]
98
+
99
+ record = { 'url' => URL }
100
+ expected = {
101
+ 'url' => URL,
102
+ 'foo' => 'bar',
103
+ }
104
+ filtered = filter(config, [record])
105
+ assert_equal(expected, filtered[0])
106
+ end
107
+
108
+ def test_filter_with_discard
109
+ config = %[
110
+ key url
111
+ discard_key true
112
+ ]
113
+
114
+ record = { 'url' => URL }
115
+ expected = {
116
+ 'foo' => 'bar',
117
+ 'baz' => 'qux',
118
+ 'モリス' => 'すたじお'
119
+ }
120
+ filtered = filter(config, [record])
121
+ assert_equal(expected, filtered[0])
122
+ end
123
+
124
+ def test_filter_multi_records
125
+ config = %[
126
+ key url
127
+ only foo, baz
128
+ ]
129
+ records = [
130
+ { 'url' => URL },
131
+ { 'url' => URL },
132
+ { 'url' => URL }
133
+ ]
134
+ expected = [
135
+ { 'url' => URL, 'foo' => 'bar', 'baz' => 'qux' },
136
+ { 'url' => URL, 'foo' => 'bar', 'baz' => 'qux' },
137
+ { 'url' => URL, 'foo' => 'bar', 'baz' => 'qux' }
138
+ ]
139
+ filtered = filter(config, records)
140
+ assert_equal(expected, filtered)
141
+ end
142
+
143
+ def test_emit_without_match_key
144
+ config = %[
145
+ key no_such_key
146
+ only foo, baz
147
+ ]
148
+ record = { 'url' => URL }
149
+ filtered = filter(config, [record])
150
+ assert_equal(record, filtered[0])
151
+ end
152
+
153
+ def test_emit_with_invalid_url
154
+ config = %[
155
+ key url
156
+ ]
157
+ record = { 'url' => URL }
158
+ filtered = filter(config, [record])
159
+ assert_equal([record], filtered)
160
+ end
161
+
162
+ DIRTY_PATH_BLANK_1 = '/dummy?&baz=qux'
163
+ DIRTY_PATH_BLANK_2 = '/dummy?foo=bar&'
164
+ DIRTY_PATH_BLANK_3 = '/dummy?foo=bar&&baz=qux'
165
+ DIRTY_PATH_BLANK_4 = '/dummy?=&baz=qux'
166
+ DIRTY_PATH_KEY_ONLY_1 = '/dummy?foo=&baz=qux'
167
+ DIRTY_PATH_KEY_ONLY_2 = '/dummy?foo&baz=qux'
168
+ DIRTY_PATH_KEY_ONLY_3 = '/dummy?baz=qux&foo'
169
+ DIRTY_PATH_VALUE_ONLY_1 = '/dummy?=bar&baz=qux'
170
+ DIRTY_PATH_VALUE_ONLY_2 = '/dummy?baz=qux&=bar'
171
+ DIRTY_PATH_BASE64_1 = '/dummy?foo=ZXh0cmE=&baz=qux'
172
+ DIRTY_PATH_BASE64_2 = '/dummy?baz=qux&foo=ZXh0cmE='
173
+ DIRTY_PATH_BASE64_3 = '/dummy?foo=cGFkZGluZw==&baz=qux'
174
+ DIRTY_PATH_BASE64_4 = '/dummy?baz=qux&foo=cGFkZGluZw=='
175
+
176
+ def test_emit_with_dirty_paths
177
+ config = %[
178
+ key path
179
+ ]
180
+ records = [
181
+ { 'path' => DIRTY_PATH_BLANK_1 },
182
+ { 'path' => DIRTY_PATH_BLANK_2 },
183
+ { 'path' => DIRTY_PATH_BLANK_3 },
184
+ { 'path' => DIRTY_PATH_BLANK_4 },
185
+ { 'path' => DIRTY_PATH_KEY_ONLY_1 },
186
+ { 'path' => DIRTY_PATH_KEY_ONLY_2 },
187
+ { 'path' => DIRTY_PATH_KEY_ONLY_3 },
188
+ { 'path' => DIRTY_PATH_VALUE_ONLY_1 },
189
+ { 'path' => DIRTY_PATH_VALUE_ONLY_2 },
190
+ { 'path' => DIRTY_PATH_BASE64_1 },
191
+ { 'path' => DIRTY_PATH_BASE64_2 },
192
+ { 'path' => DIRTY_PATH_BASE64_3 },
193
+ { 'path' => DIRTY_PATH_BASE64_4 }
194
+ ]
195
+ expected = [
196
+ { 'path' => DIRTY_PATH_BLANK_1, 'baz' => 'qux' },
197
+ { 'path' => DIRTY_PATH_BLANK_2, 'foo' => 'bar' },
198
+ { 'path' => DIRTY_PATH_BLANK_3, 'foo' => 'bar', 'baz' => 'qux' },
199
+ { 'path' => DIRTY_PATH_BLANK_4, 'baz' => 'qux' },
200
+ { 'path' => DIRTY_PATH_KEY_ONLY_1, 'foo' => '', 'baz' => 'qux' },
201
+ { 'path' => DIRTY_PATH_KEY_ONLY_2, 'foo' => '', 'baz' => 'qux' },
202
+ { 'path' => DIRTY_PATH_KEY_ONLY_3, 'foo' => '', 'baz' => 'qux' },
203
+ { 'path' => DIRTY_PATH_VALUE_ONLY_1, 'baz' => 'qux' },
204
+ { 'path' => DIRTY_PATH_VALUE_ONLY_2, 'baz' => 'qux' },
205
+ { 'path' => DIRTY_PATH_BASE64_1, 'baz' => 'qux', 'foo' => 'ZXh0cmE=' },
206
+ { 'path' => DIRTY_PATH_BASE64_2, 'baz' => 'qux', 'foo' => 'ZXh0cmE=' },
207
+ { 'path' => DIRTY_PATH_BASE64_3, 'baz' => 'qux', 'foo' => 'cGFkZGluZw==' },
208
+ { 'path' => DIRTY_PATH_BASE64_4, 'baz' => 'qux', 'foo' => 'cGFkZGluZw==' }
209
+ ]
210
+ filtered = filter(config, records)
211
+ assert_equal(expected, filtered)
212
+ end
213
+
214
+ def test_emit_with_permit_blank_key
215
+ config = %[
216
+ key path
217
+ permit_blank_key yes
218
+ ]
219
+ records = [
220
+ { 'path' => DIRTY_PATH_VALUE_ONLY_1 },
221
+ { 'path' => DIRTY_PATH_VALUE_ONLY_2 }
222
+ ]
223
+ expected = [
224
+ { 'path' => DIRTY_PATH_VALUE_ONLY_1, '' => 'bar', 'baz' => 'qux' },
225
+ { 'path' => DIRTY_PATH_VALUE_ONLY_2, '' => 'bar', 'baz' => 'qux' }
226
+ ]
227
+ filtered = filter(config, records)
228
+ assert_equal(expected, filtered)
229
+ end
230
+
231
+ def test_raw_multibyte_chars
232
+ config = %[
233
+ key path
234
+ permit_blank_key yes
235
+ ]
236
+
237
+ raw_multibytes_src = '/path/to/ほげぽす/x?a=b'
238
+
239
+ records = [
240
+ { 'path' => raw_multibytes_src.dup.encode('sjis').force_encoding('ascii-8bit') },
241
+ { 'path' => raw_multibytes_src.dup.encode('eucjp').force_encoding('ascii-8bit') }
242
+ ]
243
+ expected = [
244
+ { 'path' => raw_multibytes_src.dup.encode('sjis').force_encoding('ascii-8bit'), 'a' => 'b' },
245
+ { 'path' => raw_multibytes_src.dup.encode('eucjp').force_encoding('ascii-8bit'), 'a' => 'b' }
246
+ ]
247
+ filtered = filter(config, records)
248
+ assert_equal(expected, filtered)
249
+ end
250
+
251
+ def test_filter_with_url_scheme_host_port_path
252
+ config = %[
253
+ key url
254
+
255
+ add_url_scheme true
256
+ add_url_host true
257
+ add_url_port true
258
+ add_url_path true
259
+ ]
260
+
261
+ record = {
262
+ 'url' => URL,
263
+ }
264
+ expected = {
265
+ 'url' => URL,
266
+ 'foo' => 'bar',
267
+ 'baz' => 'qux',
268
+ 'モリス' => 'すたじお',
269
+ 'url_scheme' => 'http',
270
+ 'url_host' => 'example.com',
271
+ 'url_port' => 80,
272
+ 'url_path' => '/'
273
+ }
274
+ filtered = filter(config, [record])
275
+ assert_equal(expected, filtered[0])
276
+ end
277
+
278
+ def test_filter_with_field_prefix_and_url_scheme_host_port_path
279
+ config = %[
280
+ key url
281
+ add_field_prefix query_
282
+
283
+ add_url_scheme true
284
+ add_url_host true
285
+ add_url_port true
286
+ add_url_path true
287
+ ]
288
+
289
+ record = {
290
+ 'url' => URL,
291
+ }
292
+ expected = {
293
+ 'url' => URL,
294
+ 'query_foo' => 'bar',
295
+ 'query_baz' => 'qux',
296
+ 'query_モリス' => 'すたじお',
297
+ 'query_url_scheme' => 'http',
298
+ 'query_url_host' => 'example.com',
299
+ 'query_url_port' => 80,
300
+ 'query_url_path' => '/'
301
+ }
302
+ filtered = filter(config, [record])
303
+ assert_equal(expected, filtered[0])
304
+ end
305
+
306
+ def test_filter_from_query_only_url_with_url_scheme_host_port_path
307
+ config = %[
308
+ key url
309
+
310
+ add_url_scheme true
311
+ add_url_host true
312
+ add_url_port true
313
+ add_url_path true
314
+ ]
315
+ record = {
316
+ 'url' => QUERY_ONLY,
317
+ }
318
+ expected = {
319
+ 'url' => QUERY_ONLY,
320
+ 'foo' => 'bar',
321
+ 'baz' => 'qux',
322
+ 'モリス' => 'すたじお',
323
+ 'url_scheme' => '',
324
+ 'url_host' => '',
325
+ 'url_port' => '',
326
+ 'url_path' => ''
327
+ }
328
+ filtered = filter(config, [record])
329
+ assert_equal(expected, filtered[0])
330
+ end
331
+ end
332
+ end
@@ -17,5 +17,9 @@ end
17
17
 
18
18
  require 'fluent/plugin/out_extract_query_params'
19
19
 
20
+ if Gem::Version.new(Fluent::VERSION) > Gem::Version.new('0.12')
21
+ require 'fluent/plugin/filter_extract_query_params'
22
+ end
23
+
20
24
  class Test::Unit::TestCase
21
25
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-extract_query_params
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.10
4
+ version: 0.0.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kentaro Kuribayashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-09-04 00:00:00.000000000 Z
11
+ date: 2015-09-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: test-unit
@@ -52,6 +52,20 @@ dependencies:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: appraisal
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
55
69
  description: Fluentd plugin to extract key/values from URL query parameters.
56
70
  email:
57
71
  - kentarok@gmail.com
@@ -61,12 +75,18 @@ extra_rdoc_files: []
61
75
  files:
62
76
  - ".gitignore"
63
77
  - ".travis.yml"
78
+ - Appraisals
64
79
  - Gemfile
65
80
  - LICENSE
66
81
  - README.md
67
82
  - Rakefile
68
83
  - fluent-plugin-extract_query_params.gemspec
84
+ - gemfiles/fluentd_0.10.gemfile
85
+ - gemfiles/fluentd_0.12.gemfile
86
+ - lib/fluent/plugin/filter_extract_query_params.rb
69
87
  - lib/fluent/plugin/out_extract_query_params.rb
88
+ - lib/fluent/plugin/query_params_extractor.rb
89
+ - test/plugin/test_filter_extract_query_params.rb
70
90
  - test/plugin/test_out_extract_query_params.rb
71
91
  - test/test_helper.rb
72
92
  homepage: http://github.com/kentaro/fluent-plugin-extract_query_params
@@ -94,5 +114,6 @@ signing_key:
94
114
  specification_version: 4
95
115
  summary: Fluentd plugin to extract key/values from URL query parameters
96
116
  test_files:
117
+ - test/plugin/test_filter_extract_query_params.rb
97
118
  - test/plugin/test_out_extract_query_params.rb
98
119
  - test/test_helper.rb