fluent-plugin-referer-parser 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7e2b1770c1c3d1a3fc00efce8fc3b2fd88496891
-  data.tar.gz: ee51c634a8b0cbf538e310c858d6bea6d512007e
+  metadata.gz: 34c336f141c0deecd78a239f5b86c479c70e8f29
+  data.tar.gz: 8a98fe0e6956a991caabacd9ff25c59adef2c4d3
 SHA512:
-  metadata.gz: eebfc2c7ceca2a7107d6b10a08f966ac7aa10826f25552b2b79073f970100d8045e9c6140b46c790004621ab19e9a8f91a966f6f70ee54bb6e73feafc3b52dcb
-  data.tar.gz: 1b283b06e7d82231b84b6ebab648c7f342150e562f325c456f503709e290cf01aed39d24dea99e225e1b4e0d2782db94904068dfb2af4204bf6c41c66ee466da
+  metadata.gz: db4006faa16724acb943d940c0c5b00359017c6c518395e924c06bd7131a9f91e8bc4f2fb41be252ea4b820531e07cad2398f195de06bdc951c7ce876462c41e
+  data.tar.gz: 6632b372a5d26a530a5f91ec40fa56f8f05bdfe716f133adb24c39924b71c74b55de76df733320e62ef3d8fcb380021aeb46be3ada078a064aa355adce6cce79

data/.rubocop.yml

@@ -19,5 +19,6 @@ ClassLength:
   CountComments: false
   Max: 100
 
-GlobalVars:
-  AllowedVariables: [$log]
+Metrics/AbcSize:
+  Enabled: true
+  Max: 20

data/.travis.yml

@@ -1,5 +1,6 @@
 language: ruby
 rvm:
-  - 2.2
-  - 2.3.4
   - 2.4.1
+script:
+  - bundle exec rake
+  - bundle exec rubocop

data/Gemfile

@@ -1,3 +1,7 @@
 source 'https://rubygems.org'
 
 gemspec
+
+group :test do
+  gem 'rubocop'
+end

data/README.md

@@ -1,5 +1,7 @@
 # fluent-plugin-referer-parser, a plugin for [Fluentd](http://fluentd.org)
 
+[![Build Status](https://travis-ci.org/haruyama/fluent-plugin-referer-parser.svg?branch=master)](https://travis-ci.org/haruyama/fluent-plugin-referer-parser)
+
 ## RefererParserFilter
 
 'fluent-plugin-referer-parser' is a Fluentd plugin to parse Referer strings, based on [tagomoris/fluent-plugin-woothee](https://github.com/tagomoris/fluent-plugin-woothee).
@@ -9,9 +11,13 @@
 
 | fluent-plugin-referer-parser | fluentd     | ruby   |
 |------------------------------|-------------|--------|
-| >= 0.1.0                     | >= v0.14.15 | >= 2.1 |
+| >= 0.1.0                     | >= v0.14.15 | >= 2.4 |
 | < 0.1.0                      | >= v0.12.0  | >= 1.9 |
 
+fluent-plugin-referer-parser >= 0.1.0 works on ruby >= 2.1,
+but referer-parser(0.3.0) does not work properly for non-UTF-8 search terms with ruby < 2.4.
+You use fluent-plugin-referer-parser >= 0.1.0, so we recommend to use ruby 2.4 or later.
+
 ## Configuration
 
 To add referer-parser result into matched messages:

data/Rakefile

@@ -1,4 +1,3 @@
-#!/usr/bin/env rake
 require 'bundler/gem_tasks'
 
 require 'rake/testtask'

data/fluent-plugin-referer-parser.gemspec

@@ -1,19 +1,20 @@
 # -*- encoding: utf-8 -*-
+
 require 'English'
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-referer-parser'
-  gem.version       = '0.1.0'
+  gem.version       = '0.1.1'
   gem.authors       = ['HARUYAMA Seigo']
   gem.email         = ['haruyama@unixuser.org']
-  gem.description   = %q(parsing by referer-parser. See: https://github.com/snowplow/referer-parser)
-  gem.summary       = %q(Fluentd plugin to parse UserAgent strings)
+  gem.description   = 'parsing by referer-parser. See: https://github.com/snowplow/referer-parser'
+  gem.summary       = 'Fluentd plugin to parse Referer strings'
   gem.homepage      = 'https://github.com/haruyama/fluent-plugin-referer-parser'
   gem.license       = 'Apache-2.0'
 
   gem.files         = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR)
-  gem.executables   = gem.files.grep(/\Abin\//).map { |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(/\A(test|spec|features)\//)
+  gem.executables   = gem.files.grep(%r{\Abin/}).map { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{\A(test|spec|features)/})
   gem.require_paths = ['lib']
 
   gem.add_development_dependency 'rake'

data/lib/fluent/plugin/filter_referer_parser.rb

@@ -4,6 +4,7 @@ require 'referer-parser'
 
 require 'fluent/plugin/input'
 
+# Fluent::Plugin::RefererParserFilter filters Referer strings
 class Fluent::Plugin::RefererParserFilter < Fluent::Plugin::Filter
   Fluent::Plugin.register_filter('referer_parser', self)
 
@@ -25,32 +26,35 @@ class Fluent::Plugin::RefererParserFilter < Fluent::Plugin::Filter
                         RefererParser::Parser.new
                       end
 
-    if @encodings_yaml
-      @encodings = YAML.load_file(@encodings_yaml)
-    else
-      @encodings = {}
-    end
+    @encodings = if @encodings_yaml
+                   YAML.load_file(@encodings_yaml)
+                 else
+                   {}
+                 end
+  end
+
+  def get_search_term(search_term, uri)
+    parameters = CGI.parse(uri.query)
+    input_encoding = @encodings[uri.host] || parameters['ie'][0] || parameters['ei'][0]
+    return search_term.force_encoding(input_encoding).encode('utf-8') if input_encoding && /\Autf-?8\z/i !~ input_encoding
+    search_term
   end
 
-  def filter(tag, time, record)
+  def filter(_tag, _time, record)
     begin
       parsed = @referer_parser.parse(record[@key_name])
       record[@out_key_known] = parsed[:known]
       if parsed[:known]
-        search_term = parsed[:term]
         uri = URI.parse(parsed[:uri])
-        host = uri.host
-        parameters = CGI.parse(uri.query)
-        input_encoding = @encodings[host] || parameters['ie'][0] || parameters['ei'][0]
         begin
-          search_term = search_term.force_encoding(input_encoding).encode('utf-8') if input_encoding && /\Autf-?8\z/i !~ input_encoding
+          search_term = get_search_term(parsed[:term], uri)
         rescue
+          search_term = parsed[:term]
           log.error('invalid referer: ' + uri.to_s)
         end
         record.merge!(
-          @out_key_known       => true,
           @out_key_referer     => parsed[:source],
-          @out_key_host        => host,
+          @out_key_host        => uri.host,
           @out_key_search_term => search_term
         )
       end

data/test/.rubocop.yml

@@ -19,5 +19,6 @@ ClassLength:
   CountComments: false
   Max: 200
 
-GlobalVars:
-  AllowedVariables: [$log]
+BlockLength:
+  Max: 200
+

data/test/plugin/test_filter_referer_parser.rb

@@ -8,7 +8,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
     key_name referer
     remove_prefix test
     add_prefix merged
-  )
+  ).freeze
 
   CONFIG2 = %(
     key_name ref
@@ -16,14 +16,14 @@ class RefererParserFilterTest < Test::Unit::TestCase
     out_key_referer      ref_referer
     out_key_host         ref_host
     out_key_search_term  ref_search_term
-  )
+  ).freeze
 
   CONFIG3 = %(
     type referer_parser
     key_name ref
     referers_yaml test/data/referers.yaml
     encodings_yaml test/data/encodings.yaml
-  )
+  ).freeze
 
   def setup
     Fluent::Test.setup
@@ -56,7 +56,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
 
     test 'filter & merge' do
       d = create_driver CONFIG2
-      assert_equal 'ref',    d.instance.key_name
+      assert_equal 'ref', d.instance.key_name
 
       assert_equal 'ref_known',       d.instance.out_key_known
       assert_equal 'ref_referer',     d.instance.out_key_referer
@@ -72,7 +72,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
         { 'value' => 2, 'referer' => 'http://www.unixuser.org/' },
         { 'value' => 3, 'referer' => 'http://www.google.co.jp/search?hl=ja&ie=Shift_JIS&c2coff=1&q=%83%7D%83%8B%83%60%83L%83%83%83X%83g%81@%8Aw%8Em%98_%95%B6&lr=' },
         { 'value' => 4, 'referer' => 'http://www.google.co.jp/search?hl=ja&ie=Shift_J&c2coff=1&q=%83%7D%83%8B%83%60%83L%83%83%83X%83g%81@%8Aw%8Em%98_%95%B6&lr=' },
-        { 'value' => 5, 'referer' => 'http://search.yahoo.co.jp/search?p=%E3%81%BB%E3%81%92&aq=-1&oq=&ei=UTF-8&fr=sfp_as&x=wrt' },
+        { 'value' => 5, 'referer' => 'http://search.yahoo.co.jp/search?p=%E3%81%BB%E3%81%92&aq=-1&oq=&ei=UTF-8&fr=sfp_as&x=wrt' }
       ]
       expected = [
         {
@@ -107,7 +107,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
           'referer_known' => true,
           'referer_referer' => 'Google',
           'referer_host' => 'www.google.co.jp',
-          'referer_search_term' => 'マルチキャスト　学士論文'.encode("Shift_JIS").force_encoding("US-ASCII")
+          'referer_search_term' => 'マルチキャスト　学士論文'.encode('Shift_JIS').force_encoding('US-ASCII')
         },
         {
           'value' => 5,
@@ -145,7 +145,7 @@ class RefererParserFilterTest < Test::Unit::TestCase
         {
           'value' => 2,
           'ref' => 'http://www.unixuser.org/',
-          'ref_known' => false,
+          'ref_known' => false
         },
         {
           'value' => 3,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-referer-parser
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - HARUYAMA Seigo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-02 00:00:00.000000000 Z
+date: 2017-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -116,7 +116,7 @@ rubyforge_project:
 rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
-summary: Fluentd plugin to parse UserAgent strings
+summary: Fluentd plugin to parse Referer strings
 test_files:
 - test/.rubocop.yml
 - test/data/encodings.yaml