fluent-plugin-referer-parser 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7e2b1770c1c3d1a3fc00efce8fc3b2fd88496891
4
- data.tar.gz: ee51c634a8b0cbf538e310c858d6bea6d512007e
3
+ metadata.gz: 34c336f141c0deecd78a239f5b86c479c70e8f29
4
+ data.tar.gz: 8a98fe0e6956a991caabacd9ff25c59adef2c4d3
5
5
  SHA512:
6
- metadata.gz: eebfc2c7ceca2a7107d6b10a08f966ac7aa10826f25552b2b79073f970100d8045e9c6140b46c790004621ab19e9a8f91a966f6f70ee54bb6e73feafc3b52dcb
7
- data.tar.gz: 1b283b06e7d82231b84b6ebab648c7f342150e562f325c456f503709e290cf01aed39d24dea99e225e1b4e0d2782db94904068dfb2af4204bf6c41c66ee466da
6
+ metadata.gz: db4006faa16724acb943d940c0c5b00359017c6c518395e924c06bd7131a9f91e8bc4f2fb41be252ea4b820531e07cad2398f195de06bdc951c7ce876462c41e
7
+ data.tar.gz: 6632b372a5d26a530a5f91ec40fa56f8f05bdfe716f133adb24c39924b71c74b55de76df733320e62ef3d8fcb380021aeb46be3ada078a064aa355adce6cce79
data/.rubocop.yml CHANGED
@@ -19,5 +19,6 @@ ClassLength:
19
19
  CountComments: false
20
20
  Max: 100
21
21
 
22
- GlobalVars:
23
- AllowedVariables: [$log]
22
+ Metrics/AbcSize:
23
+ Enabled: true
24
+ Max: 20
data/.travis.yml CHANGED
@@ -1,5 +1,6 @@
1
1
  language: ruby
2
2
  rvm:
3
- - 2.2
4
- - 2.3.4
5
3
  - 2.4.1
4
+ script:
5
+ - bundle exec rake
6
+ - bundle exec rubocop
data/Gemfile CHANGED
@@ -1,3 +1,7 @@
1
1
  source 'https://rubygems.org'
2
2
 
3
3
  gemspec
4
+
5
+ group :test do
6
+ gem 'rubocop'
7
+ end
data/README.md CHANGED
@@ -1,5 +1,7 @@
1
1
  # fluent-plugin-referer-parser, a plugin for [Fluentd](http://fluentd.org)
2
2
 
3
+ [![Build Status](https://travis-ci.org/haruyama/fluent-plugin-referer-parser.svg?branch=master)](https://travis-ci.org/haruyama/fluent-plugin-referer-parser)
4
+
3
5
  ## RefererParserFilter
4
6
 
5
7
  'fluent-plugin-referer-parser' is a Fluentd plugin to parse Referer strings, based on [tagomoris/fluent-plugin-woothee](https://github.com/tagomoris/fluent-plugin-woothee).
@@ -9,9 +11,13 @@
9
11
 
10
12
  | fluent-plugin-referer-parser | fluentd | ruby |
11
13
  |------------------------------|-------------|--------|
12
- | >= 0.1.0 | >= v0.14.15 | >= 2.1 |
14
+ | >= 0.1.0 | >= v0.14.15 | >= 2.4 |
13
15
  | < 0.1.0 | >= v0.12.0 | >= 1.9 |
14
16
 
17
+ fluent-plugin-referer-parser >= 0.1.0 works on ruby >= 2.1,
18
+ but referer-parser(0.3.0) does not work properly for non-UTF-8 search terms with ruby < 2.4.
19
+ You use fluent-plugin-referer-parser >= 0.1.0, so we recommend to use ruby 2.4 or later.
20
+
15
21
  ## Configuration
16
22
 
17
23
  To add referer-parser result into matched messages:
data/Rakefile CHANGED
@@ -1,4 +1,3 @@
1
- #!/usr/bin/env rake
2
1
  require 'bundler/gem_tasks'
3
2
 
4
3
  require 'rake/testtask'
@@ -1,19 +1,20 @@
1
1
  # -*- encoding: utf-8 -*-
2
+
2
3
  require 'English'
3
4
 
4
5
  Gem::Specification.new do |gem|
5
6
  gem.name = 'fluent-plugin-referer-parser'
6
- gem.version = '0.1.0'
7
+ gem.version = '0.1.1'
7
8
  gem.authors = ['HARUYAMA Seigo']
8
9
  gem.email = ['haruyama@unixuser.org']
9
- gem.description = %q(parsing by referer-parser. See: https://github.com/snowplow/referer-parser)
10
- gem.summary = %q(Fluentd plugin to parse UserAgent strings)
10
+ gem.description = 'parsing by referer-parser. See: https://github.com/snowplow/referer-parser'
11
+ gem.summary = 'Fluentd plugin to parse Referer strings'
11
12
  gem.homepage = 'https://github.com/haruyama/fluent-plugin-referer-parser'
12
13
  gem.license = 'Apache-2.0'
13
14
 
14
15
  gem.files = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR)
15
- gem.executables = gem.files.grep(/\Abin\//).map { |f| File.basename(f) }
16
- gem.test_files = gem.files.grep(/\A(test|spec|features)\//)
16
+ gem.executables = gem.files.grep(%r{\Abin/}).map { |f| File.basename(f) }
17
+ gem.test_files = gem.files.grep(%r{\A(test|spec|features)/})
17
18
  gem.require_paths = ['lib']
18
19
 
19
20
  gem.add_development_dependency 'rake'
@@ -4,6 +4,7 @@ require 'referer-parser'
4
4
 
5
5
  require 'fluent/plugin/input'
6
6
 
7
+ # Fluent::Plugin::RefererParserFilter filters Referer strings
7
8
  class Fluent::Plugin::RefererParserFilter < Fluent::Plugin::Filter
8
9
  Fluent::Plugin.register_filter('referer_parser', self)
9
10
 
@@ -25,32 +26,35 @@ class Fluent::Plugin::RefererParserFilter < Fluent::Plugin::Filter
25
26
  RefererParser::Parser.new
26
27
  end
27
28
 
28
- if @encodings_yaml
29
- @encodings = YAML.load_file(@encodings_yaml)
30
- else
31
- @encodings = {}
32
- end
29
+ @encodings = if @encodings_yaml
30
+ YAML.load_file(@encodings_yaml)
31
+ else
32
+ {}
33
+ end
34
+ end
35
+
36
+ def get_search_term(search_term, uri)
37
+ parameters = CGI.parse(uri.query)
38
+ input_encoding = @encodings[uri.host] || parameters['ie'][0] || parameters['ei'][0]
39
+ return search_term.force_encoding(input_encoding).encode('utf-8') if input_encoding && /\Autf-?8\z/i !~ input_encoding
40
+ search_term
33
41
  end
34
42
 
35
- def filter(tag, time, record)
43
+ def filter(_tag, _time, record)
36
44
  begin
37
45
  parsed = @referer_parser.parse(record[@key_name])
38
46
  record[@out_key_known] = parsed[:known]
39
47
  if parsed[:known]
40
- search_term = parsed[:term]
41
48
  uri = URI.parse(parsed[:uri])
42
- host = uri.host
43
- parameters = CGI.parse(uri.query)
44
- input_encoding = @encodings[host] || parameters['ie'][0] || parameters['ei'][0]
45
49
  begin
46
- search_term = search_term.force_encoding(input_encoding).encode('utf-8') if input_encoding && /\Autf-?8\z/i !~ input_encoding
50
+ search_term = get_search_term(parsed[:term], uri)
47
51
  rescue
52
+ search_term = parsed[:term]
48
53
  log.error('invalid referer: ' + uri.to_s)
49
54
  end
50
55
  record.merge!(
51
- @out_key_known => true,
52
56
  @out_key_referer => parsed[:source],
53
- @out_key_host => host,
57
+ @out_key_host => uri.host,
54
58
  @out_key_search_term => search_term
55
59
  )
56
60
  end
data/test/.rubocop.yml CHANGED
@@ -19,5 +19,6 @@ ClassLength:
19
19
  CountComments: false
20
20
  Max: 200
21
21
 
22
- GlobalVars:
23
- AllowedVariables: [$log]
22
+ BlockLength:
23
+ Max: 200
24
+
@@ -8,7 +8,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
8
8
  key_name referer
9
9
  remove_prefix test
10
10
  add_prefix merged
11
- )
11
+ ).freeze
12
12
 
13
13
  CONFIG2 = %(
14
14
  key_name ref
@@ -16,14 +16,14 @@ class RefererParserFilterTest < Test::Unit::TestCase
16
16
  out_key_referer ref_referer
17
17
  out_key_host ref_host
18
18
  out_key_search_term ref_search_term
19
- )
19
+ ).freeze
20
20
 
21
21
  CONFIG3 = %(
22
22
  type referer_parser
23
23
  key_name ref
24
24
  referers_yaml test/data/referers.yaml
25
25
  encodings_yaml test/data/encodings.yaml
26
- )
26
+ ).freeze
27
27
 
28
28
  def setup
29
29
  Fluent::Test.setup
@@ -56,7 +56,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
56
56
 
57
57
  test 'filter & merge' do
58
58
  d = create_driver CONFIG2
59
- assert_equal 'ref', d.instance.key_name
59
+ assert_equal 'ref', d.instance.key_name
60
60
 
61
61
  assert_equal 'ref_known', d.instance.out_key_known
62
62
  assert_equal 'ref_referer', d.instance.out_key_referer
@@ -72,7 +72,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
72
72
  { 'value' => 2, 'referer' => 'http://www.unixuser.org/' },
73
73
  { 'value' => 3, 'referer' => 'http://www.google.co.jp/search?hl=ja&ie=Shift_JIS&c2coff=1&q=%83%7D%83%8B%83%60%83L%83%83%83X%83g%81@%8Aw%8Em%98_%95%B6&lr=' },
74
74
  { 'value' => 4, 'referer' => 'http://www.google.co.jp/search?hl=ja&ie=Shift_J&c2coff=1&q=%83%7D%83%8B%83%60%83L%83%83%83X%83g%81@%8Aw%8Em%98_%95%B6&lr=' },
75
- { 'value' => 5, 'referer' => 'http://search.yahoo.co.jp/search?p=%E3%81%BB%E3%81%92&aq=-1&oq=&ei=UTF-8&fr=sfp_as&x=wrt' },
75
+ { 'value' => 5, 'referer' => 'http://search.yahoo.co.jp/search?p=%E3%81%BB%E3%81%92&aq=-1&oq=&ei=UTF-8&fr=sfp_as&x=wrt' }
76
76
  ]
77
77
  expected = [
78
78
  {
@@ -107,7 +107,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
107
107
  'referer_known' => true,
108
108
  'referer_referer' => 'Google',
109
109
  'referer_host' => 'www.google.co.jp',
110
- 'referer_search_term' => 'マルチキャスト 学士論文'.encode("Shift_JIS").force_encoding("US-ASCII")
110
+ 'referer_search_term' => 'マルチキャスト 学士論文'.encode('Shift_JIS').force_encoding('US-ASCII')
111
111
  },
112
112
  {
113
113
  'value' => 5,
@@ -145,7 +145,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
145
145
  {
146
146
  'value' => 2,
147
147
  'ref' => 'http://www.unixuser.org/',
148
- 'ref_known' => false,
148
+ 'ref_known' => false
149
149
  },
150
150
  {
151
151
  'value' => 3,
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-referer-parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - HARUYAMA Seigo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-06-02 00:00:00.000000000 Z
11
+ date: 2017-06-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -116,7 +116,7 @@ rubyforge_project:
116
116
  rubygems_version: 2.6.11
117
117
  signing_key:
118
118
  specification_version: 4
119
- summary: Fluentd plugin to parse UserAgent strings
119
+ summary: Fluentd plugin to parse Referer strings
120
120
  test_files:
121
121
  - test/.rubocop.yml
122
122
  - test/data/encodings.yaml