fluent-plugin-referer-parser 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -2
- data/.travis.yml +3 -2
- data/Gemfile +4 -0
- data/README.md +7 -1
- data/Rakefile +0 -1
- data/fluent-plugin-referer-parser.gemspec +6 -5
- data/lib/fluent/plugin/filter_referer_parser.rb +17 -13
- data/test/.rubocop.yml +3 -2
- data/test/plugin/test_filter_referer_parser.rb +7 -7
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 34c336f141c0deecd78a239f5b86c479c70e8f29
|
4
|
+
data.tar.gz: 8a98fe0e6956a991caabacd9ff25c59adef2c4d3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: db4006faa16724acb943d940c0c5b00359017c6c518395e924c06bd7131a9f91e8bc4f2fb41be252ea4b820531e07cad2398f195de06bdc951c7ce876462c41e
|
7
|
+
data.tar.gz: 6632b372a5d26a530a5f91ec40fa56f8f05bdfe716f133adb24c39924b71c74b55de76df733320e62ef3d8fcb380021aeb46be3ada078a064aa355adce6cce79
|
data/.rubocop.yml
CHANGED
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# fluent-plugin-referer-parser, a plugin for [Fluentd](http://fluentd.org)
|
2
2
|
|
3
|
+
[](https://travis-ci.org/haruyama/fluent-plugin-referer-parser)
|
4
|
+
|
3
5
|
## RefererParserFilter
|
4
6
|
|
5
7
|
'fluent-plugin-referer-parser' is a Fluentd plugin to parse Referer strings, based on [tagomoris/fluent-plugin-woothee](https://github.com/tagomoris/fluent-plugin-woothee).
|
@@ -9,9 +11,13 @@
|
|
9
11
|
|
10
12
|
| fluent-plugin-referer-parser | fluentd | ruby |
|
11
13
|
|------------------------------|-------------|--------|
|
12
|
-
| >= 0.1.0 | >= v0.14.15 | >= 2.
|
14
|
+
| >= 0.1.0 | >= v0.14.15 | >= 2.4 |
|
13
15
|
| < 0.1.0 | >= v0.12.0 | >= 1.9 |
|
14
16
|
|
17
|
+
fluent-plugin-referer-parser >= 0.1.0 works on ruby >= 2.1,
|
18
|
+
but referer-parser(0.3.0) does not work properly for non-UTF-8 search terms with ruby < 2.4.
|
19
|
+
You use fluent-plugin-referer-parser >= 0.1.0, so we recommend to use ruby 2.4 or later.
|
20
|
+
|
15
21
|
## Configuration
|
16
22
|
|
17
23
|
To add referer-parser result into matched messages:
|
data/Rakefile
CHANGED
@@ -1,19 +1,20 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
|
+
|
2
3
|
require 'English'
|
3
4
|
|
4
5
|
Gem::Specification.new do |gem|
|
5
6
|
gem.name = 'fluent-plugin-referer-parser'
|
6
|
-
gem.version = '0.1.
|
7
|
+
gem.version = '0.1.1'
|
7
8
|
gem.authors = ['HARUYAMA Seigo']
|
8
9
|
gem.email = ['haruyama@unixuser.org']
|
9
|
-
gem.description =
|
10
|
-
gem.summary =
|
10
|
+
gem.description = 'parsing by referer-parser. See: https://github.com/snowplow/referer-parser'
|
11
|
+
gem.summary = 'Fluentd plugin to parse Referer strings'
|
11
12
|
gem.homepage = 'https://github.com/haruyama/fluent-plugin-referer-parser'
|
12
13
|
gem.license = 'Apache-2.0'
|
13
14
|
|
14
15
|
gem.files = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR)
|
15
|
-
gem.executables = gem.files.grep(
|
16
|
-
gem.test_files = gem.files.grep(
|
16
|
+
gem.executables = gem.files.grep(%r{\Abin/}).map { |f| File.basename(f) }
|
17
|
+
gem.test_files = gem.files.grep(%r{\A(test|spec|features)/})
|
17
18
|
gem.require_paths = ['lib']
|
18
19
|
|
19
20
|
gem.add_development_dependency 'rake'
|
@@ -4,6 +4,7 @@ require 'referer-parser'
|
|
4
4
|
|
5
5
|
require 'fluent/plugin/input'
|
6
6
|
|
7
|
+
# Fluent::Plugin::RefererParserFilter filters Referer strings
|
7
8
|
class Fluent::Plugin::RefererParserFilter < Fluent::Plugin::Filter
|
8
9
|
Fluent::Plugin.register_filter('referer_parser', self)
|
9
10
|
|
@@ -25,32 +26,35 @@ class Fluent::Plugin::RefererParserFilter < Fluent::Plugin::Filter
|
|
25
26
|
RefererParser::Parser.new
|
26
27
|
end
|
27
28
|
|
28
|
-
if @encodings_yaml
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
29
|
+
@encodings = if @encodings_yaml
|
30
|
+
YAML.load_file(@encodings_yaml)
|
31
|
+
else
|
32
|
+
{}
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def get_search_term(search_term, uri)
|
37
|
+
parameters = CGI.parse(uri.query)
|
38
|
+
input_encoding = @encodings[uri.host] || parameters['ie'][0] || parameters['ei'][0]
|
39
|
+
return search_term.force_encoding(input_encoding).encode('utf-8') if input_encoding && /\Autf-?8\z/i !~ input_encoding
|
40
|
+
search_term
|
33
41
|
end
|
34
42
|
|
35
|
-
def filter(
|
43
|
+
def filter(_tag, _time, record)
|
36
44
|
begin
|
37
45
|
parsed = @referer_parser.parse(record[@key_name])
|
38
46
|
record[@out_key_known] = parsed[:known]
|
39
47
|
if parsed[:known]
|
40
|
-
search_term = parsed[:term]
|
41
48
|
uri = URI.parse(parsed[:uri])
|
42
|
-
host = uri.host
|
43
|
-
parameters = CGI.parse(uri.query)
|
44
|
-
input_encoding = @encodings[host] || parameters['ie'][0] || parameters['ei'][0]
|
45
49
|
begin
|
46
|
-
search_term =
|
50
|
+
search_term = get_search_term(parsed[:term], uri)
|
47
51
|
rescue
|
52
|
+
search_term = parsed[:term]
|
48
53
|
log.error('invalid referer: ' + uri.to_s)
|
49
54
|
end
|
50
55
|
record.merge!(
|
51
|
-
@out_key_known => true,
|
52
56
|
@out_key_referer => parsed[:source],
|
53
|
-
@out_key_host => host,
|
57
|
+
@out_key_host => uri.host,
|
54
58
|
@out_key_search_term => search_term
|
55
59
|
)
|
56
60
|
end
|
data/test/.rubocop.yml
CHANGED
@@ -8,7 +8,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
|
|
8
8
|
key_name referer
|
9
9
|
remove_prefix test
|
10
10
|
add_prefix merged
|
11
|
-
)
|
11
|
+
).freeze
|
12
12
|
|
13
13
|
CONFIG2 = %(
|
14
14
|
key_name ref
|
@@ -16,14 +16,14 @@ class RefererParserFilterTest < Test::Unit::TestCase
|
|
16
16
|
out_key_referer ref_referer
|
17
17
|
out_key_host ref_host
|
18
18
|
out_key_search_term ref_search_term
|
19
|
-
)
|
19
|
+
).freeze
|
20
20
|
|
21
21
|
CONFIG3 = %(
|
22
22
|
type referer_parser
|
23
23
|
key_name ref
|
24
24
|
referers_yaml test/data/referers.yaml
|
25
25
|
encodings_yaml test/data/encodings.yaml
|
26
|
-
)
|
26
|
+
).freeze
|
27
27
|
|
28
28
|
def setup
|
29
29
|
Fluent::Test.setup
|
@@ -56,7 +56,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
|
|
56
56
|
|
57
57
|
test 'filter & merge' do
|
58
58
|
d = create_driver CONFIG2
|
59
|
-
assert_equal 'ref',
|
59
|
+
assert_equal 'ref', d.instance.key_name
|
60
60
|
|
61
61
|
assert_equal 'ref_known', d.instance.out_key_known
|
62
62
|
assert_equal 'ref_referer', d.instance.out_key_referer
|
@@ -72,7 +72,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
|
|
72
72
|
{ 'value' => 2, 'referer' => 'http://www.unixuser.org/' },
|
73
73
|
{ 'value' => 3, 'referer' => 'http://www.google.co.jp/search?hl=ja&ie=Shift_JIS&c2coff=1&q=%83%7D%83%8B%83%60%83L%83%83%83X%83g%81@%8Aw%8Em%98_%95%B6&lr=' },
|
74
74
|
{ 'value' => 4, 'referer' => 'http://www.google.co.jp/search?hl=ja&ie=Shift_J&c2coff=1&q=%83%7D%83%8B%83%60%83L%83%83%83X%83g%81@%8Aw%8Em%98_%95%B6&lr=' },
|
75
|
-
{ 'value' => 5, 'referer' => 'http://search.yahoo.co.jp/search?p=%E3%81%BB%E3%81%92&aq=-1&oq=&ei=UTF-8&fr=sfp_as&x=wrt' }
|
75
|
+
{ 'value' => 5, 'referer' => 'http://search.yahoo.co.jp/search?p=%E3%81%BB%E3%81%92&aq=-1&oq=&ei=UTF-8&fr=sfp_as&x=wrt' }
|
76
76
|
]
|
77
77
|
expected = [
|
78
78
|
{
|
@@ -107,7 +107,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
|
|
107
107
|
'referer_known' => true,
|
108
108
|
'referer_referer' => 'Google',
|
109
109
|
'referer_host' => 'www.google.co.jp',
|
110
|
-
'referer_search_term' => 'マルチキャスト 学士論文'.encode(
|
110
|
+
'referer_search_term' => 'マルチキャスト 学士論文'.encode('Shift_JIS').force_encoding('US-ASCII')
|
111
111
|
},
|
112
112
|
{
|
113
113
|
'value' => 5,
|
@@ -145,7 +145,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
|
|
145
145
|
{
|
146
146
|
'value' => 2,
|
147
147
|
'ref' => 'http://www.unixuser.org/',
|
148
|
-
'ref_known' => false
|
148
|
+
'ref_known' => false
|
149
149
|
},
|
150
150
|
{
|
151
151
|
'value' => 3,
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-referer-parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- HARUYAMA Seigo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-06-
|
11
|
+
date: 2017-06-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -116,7 +116,7 @@ rubyforge_project:
|
|
116
116
|
rubygems_version: 2.6.11
|
117
117
|
signing_key:
|
118
118
|
specification_version: 4
|
119
|
-
summary: Fluentd plugin to parse
|
119
|
+
summary: Fluentd plugin to parse Referer strings
|
120
120
|
test_files:
|
121
121
|
- test/.rubocop.yml
|
122
122
|
- test/data/encodings.yaml
|