fluent-plugin-referer-parser 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dd83b311f03254625e7b916003dec4f6035cc61a
-  data.tar.gz: ece5d27318aa190985fef7e3cc10d85214210884
+  metadata.gz: b2e28d942761a6c6393cd106b2a64ff5be802e4a
+  data.tar.gz: c03bf0ed0ebbe8bf97b4368d609f7ad1f0866186
 SHA512:
-  metadata.gz: 451ebb2c50be591dc417bcfbffd33bd10f5dd919707f510ce4dfbff4dfde3befd5c39f44b2628f1369f094f8033f622be3fa1b7477832b5fb1bda0aa0adbe291
-  data.tar.gz: 6df8bb5c8542c145dc5517724a06e6bae3d443092e87f96370c6eac331a02a263f76090d15440054b5d7c349dc3830536b342754e3bfa71f235a36227ffb7d62
+  metadata.gz: 137841368528697ede2578449cb931f0ff92f4dcb63202dbd3680312403e80c124be3d9ec3fabbd885d3277c05747ebb1cf09536bfd42562dc8f5d47c6f670be
+  data.tar.gz: 5a2841b7f3160bb25dd032c39c7ec484f34e8fb773fd079976c18d3913656448415a15856a4b4f6a9618fe918063897bf098d442e718a276e6cb83e7fbec3bd3

data/README.md

@@ -17,7 +17,7 @@ To add referer-parser result into matched messages:
       add_prefix merged
     </match>
 
-Output messages with tag 'merged.**' has 'referer\_known', 'referer\_referer' and 'referer\_search\_term' attributes. If you want to change attribute names, write configurations as below:
+Output messages with tag 'merged.**' has 'referer_known', 'referer_referer' and 'referer_search_term' attributes. If you want to change attribute names, write configurations as below:
 
     <match input.**>
       type referer_parser
@@ -26,9 +26,14 @@ Output messages with tag 'merged.**' has 'referer\_known', 'referer\_referer' an
       add_prefix merged
       out_key_known        ref_known
       out_key_referer      ref_referer
+      out_key_host         ref_host
       out_key_search_term  ref_search_term
     </match>
 
+If you want to use your own referers definition, you can use 'referers_yaml' attribute.
+'referers_yaml' should be referers.yaml format of [snowplow/referer-parser](https://github.com/snowplow/referer-parser).
+
+* [Sample](test/data/referers.yaml)
 
 ## Copyright
 

data/fluent-plugin-referer-parser.gemspec

@@ -2,9 +2,9 @@
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-referer-parser'
-  gem.version       = '0.0.5'
+  gem.version       = '0.0.6'
   gem.authors       = ['TAGOMORI Satoshi', 'HARUYAMA Seigo']
-  gem.email         = ['tagomoris@gmail.com', 'haruyama@unixuser.org']
+  gem.email         = ['haruyama@unixuser.org']
   gem.description   = %q{parsing by referer-parser. See: https://github.com/snowplow/referer-parser}
   gem.summary       = %q{Fluentd plugin to parse UserAgent strings}
   gem.homepage      = 'https://github.com/haruyama/fluent-plugin-referer-parser'

data/lib/fluent/plugin/out_referer_parser.rb

@@ -1,5 +1,3 @@
-require 'referer-parser'
-
 # referer parser output
 class Fluent::RefererParserOutput < Fluent::Output
   Fluent::Plugin.register_output('referer_parser', self)
@@ -8,22 +6,33 @@ class Fluent::RefererParserOutput < Fluent::Output
   config_param :remove_prefix, :string, default: nil
   config_param :add_prefix,    :string, default: nil
 
-  config_param :key_name, :string
+  config_param :key_name,       :string
+  config_param :referers_yaml,  :string, default: nil
+  config_param :encodings_yaml, :string, default: nil
 
   config_param :out_key_known,       :string, default: 'referer_known'
   config_param :out_key_referer,     :string, default: 'referer_referer'
+  config_param :out_key_host,        :string, default: 'referer_host'
   config_param :out_key_search_term, :string, default: 'referer_search_term'
 
-  REFERER = RefererParser::Referer.new('http://example.org/')
-
   def initialize
     super
     require 'cgi'
+    require 'yaml'
+    require 'referer-parser'
   end
 
   def configure(conf)
     super
 
+    @referer_parser = RefererParser::Referer.new('http://example.org/', @referers_yaml)
+
+    if @encodings_yaml
+      @encodings = YAML.load_file(@encodings_yaml)
+    else
+      @encodings = {}
+    end
+
     if !@tag && !@remove_prefix && !@add_prefix
       fail Fluent::ConfigError, 'missing both of remove_prefix and add_prefix'
     end
@@ -59,24 +68,26 @@ class Fluent::RefererParserOutput < Fluent::Output
   def emit(tag, es, chain)
     tag = tag_mangle(tag)
     es.each do |time, record|
-      valid = true
+      is_valid = true
       begin
-        REFERER.parse(record[@key_name])
+        @referer_parser.parse(record[@key_name])
       rescue
-        valid = false
+        is_valid = false
       end
-      if valid && REFERER.known?
-        search_term = REFERER.search_term
-        parameters = CGI.parse(REFERER.uri.query)
-        input_encoding = parameters['ie'][0] || parameters['ei'][0]
+      if is_valid && @referer_parser.known?
+        search_term = @referer_parser.search_term
+        host = @referer_parser.uri.host
+        parameters = CGI.parse(@referer_parser.uri.query)
+        input_encoding = @encodings[host] || parameters['ie'][0] || parameters['ei'][0]
         begin
           search_term = search_term.force_encoding(input_encoding).encode('utf-8') if input_encoding && /\Autf-?8\z/i !~ input_encoding
         rescue
-          $log.error('invalid referer: ' + REFERER.uri.to_s)
+          $log.error('invalid referer: ' + @referer_parser.uri.to_s)
         end
         record.merge!(
           @out_key_known       => true,
-          @out_key_referer     => REFERER.referer,
+          @out_key_referer     => @referer_parser.referer,
+          @out_key_host        => host,
           @out_key_search_term => search_term,
         )
       else

data/test/data/encodings.yaml

@@ -0,0 +1,2 @@
+---
+ezsch.ezweb.ne.jp: Shift_JIS

data/test/data/referers.yaml

@@ -0,0 +1,16 @@
+search:
+
+  Google:
+    parameters:
+      - q
+    domains:
+      - www.google.com
+      - www.google.co.jp
+
+  Ezweb:
+    parameters:
+      - query
+    domains:
+      - ezsch.ezweb.ne.jp
+
+

data/test/plugin/test_out_referer_parser.rb

@@ -17,7 +17,17 @@ remove_prefix test
 add_prefix merged
 out_key_known        ref_known
 out_key_referer      ref_referer
+out_key_host         ref_host
 out_key_search_term  ref_search_term
+]
+
+  CONFIG3 = %[
+type referer_parser
+key_name ref
+remove_prefix test
+add_prefix merged
+referers_yaml test/data/referers.yaml
+encodings_yaml test/data/encodings.yaml
 ]
 
   def create_driver(conf = CONFIG1, tag = 'test')
@@ -82,8 +92,9 @@ out_key_search_term  ref_search_term
     assert_equal 1,                                  m['value']
     assert_equal true,                               m['referer_known']
     assert_equal 'Google',                           m['referer_referer']
+    assert_equal 'www.google.com',                   m['referer_host']
     assert_equal 'gateway oracle cards denise linn', m['referer_search_term']
-    assert_equal 5,         m.keys.size
+    assert_equal 6,         m.keys.size
 
     m = emits[2][2]
     assert_equal 2,         m['value']
@@ -96,22 +107,25 @@ out_key_search_term  ref_search_term
     assert_equal 3,                          m['value']
     assert_equal true,                       m['referer_known']
     assert_equal 'Google',                   m['referer_referer']
+    assert_equal 'www.google.co.jp',         m['referer_host']
     assert_equal 'マルチキャスト　学士論文', m['referer_search_term']
-    assert_equal 5,         m.keys.size
+    assert_equal 6,         m.keys.size
 
     # invalid input_encoding
     m = emits[4][2]
     assert_equal 4,        m['value']
     assert_equal true,     m['referer_known']
     assert_equal 'Google', m['referer_referer']
-    assert_equal 5,        m.keys.size
+    assert_equal 'www.google.co.jp',         m['referer_host']
+    assert_equal 6,        m.keys.size
 
     m = emits[5][2]
-    assert_equal 5,       m['value']
-    assert_equal true,    m['referer_known']
-    assert_equal 'Yahoo!', m['referer_referer']
-    assert_equal 'ほげ',  m['referer_search_term']
-    assert_equal 5,       m.keys.size
+    assert_equal 5,                    m['value']
+    assert_equal true,                 m['referer_known']
+    assert_equal 'Yahoo!',             m['referer_referer']
+    assert_equal 'search.yahoo.co.jp', m['referer_host']
+    assert_equal 'ほげ',               m['referer_search_term']
+    assert_equal 6,                    m.keys.size
   end
 
   def test_emit2
@@ -132,27 +146,77 @@ out_key_search_term  ref_search_term
     m = emits[0][2]
     assert_equal 0,         m['value']
     assert_equal false,     m['ref_known']
-    assert_nil   m['referer_referer']
-    assert_nil   m['referer_search_term']
+    assert_nil   m['ref_referer']
+    assert_nil   m['ref_search_term']
     assert_equal 2,         m.keys.size
 
     m = emits[1][2]
     assert_equal 1,                                  m['value']
     assert_equal true,                               m['ref_known']
     assert_equal 'Google',                           m['ref_referer']
+    assert_equal 'www.google.com',                   m['ref_host']
     assert_equal 'gateway oracle cards denise linn', m['ref_search_term']
-    assert_equal 5,         m.keys.size
+    assert_equal 6,         m.keys.size
 
     m = emits[2][2]
     assert_equal 2,         m['value']
     assert_equal false,     m['ref_known']
-    assert_nil   m['referer_referer']
-    assert_nil   m['referer_search_term']
+    assert_nil   m['ref_referer']
+    assert_nil   m['ref_host']
+    assert_nil   m['ref_search_term']
 
     m = emits[3][2]
-    assert_equal 3,        m['value']
-    assert_equal true,     m['ref_known']
-    assert_equal 'Google', m['ref_referer']
-    assert_equal 'ほげ',   m['ref_search_term']
+    assert_equal 3,                m['value']
+    assert_equal true,             m['ref_known']
+    assert_equal 'Google',         m['ref_referer']
+    assert_equal 'www.google.com', m['ref_host']
+    assert_equal 'ほげ',           m['ref_search_term']
+  end
+
+  def test_emit3
+    d = create_driver(CONFIG3, 'test.message')
+    time = Time.parse('2012-07-20 16:40:30').to_i
+    d.run do
+      d.emit({ 'value' => 0 }, time)
+      d.emit({ 'value' => 1, 'ref' => 'http://ezsch.ezweb.ne.jp/search/?sr=0101&query=aiueo%20%95a%93I' }, time)
+      d.emit({ 'value' => 2, 'ref' => 'http://ezsch.ezweb.ne.jp/search/ezGoogleMain.php?query=%83%8D' }, time)
+      d.emit({ 'value' => 3, 'ref' => 'http://www.google.co.jp/search?hl=ja&ie=Shift_JIS&c2coff=1&q=%83%7D%83%8B%83%60%83L%83%83%83X%83g%81@%8Aw%8Em%98_%95%B6&lr=' }, time)
+    end
+
+    emits = d.emits
+    assert_equal 4,                emits.size
+    assert_equal 'merged.message', emits[0][0]
+    assert_equal time,             emits[0][1]
+
+    m = emits[0][2]
+    assert_equal 0,     m['value']
+    assert_equal false, m['referer_known']
+    assert_nil          m['referer_referer']
+    assert_nil          m['referer_search_term']
+    assert_equal 2,     m.keys.size
+
+    m = emits[1][2]
+    assert_equal 1,                   m['value']
+    assert_equal true,                m['referer_known']
+    assert_equal 'Ezweb',             m['referer_referer']
+    assert_equal 'ezsch.ezweb.ne.jp', m['referer_host']
+    assert_equal 'aiueo 病的',        m['referer_search_term']
+    assert_equal 6,                   m.keys.size
+
+    m = emits[2][2]
+    assert_equal 2,                   m['value']
+    assert_equal true,                m['referer_known']
+    assert_equal 'Ezweb',             m['referer_referer']
+    assert_equal 'ezsch.ezweb.ne.jp', m['referer_host']
+    assert_equal 'ロ',                m['referer_search_term']
+    assert_equal 6,                   m.keys.size
+
+    m = emits[3][2]
+    assert_equal 3,                          m['value']
+    assert_equal true,                       m['referer_known']
+    assert_equal 'Google',                   m['referer_referer']
+    assert_equal 'www.google.co.jp',         m['referer_host']
+    assert_equal 'マルチキャスト　学士論文', m['referer_search_term']
+    assert_equal 6,         m.keys.size
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-referer-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - TAGOMORI Satoshi
@@ -9,60 +9,59 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-07 00:00:00.000000000 Z
+date: 2014-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: referer-parser
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.0
 description: 'parsing by referer-parser. See: https://github.com/snowplow/referer-parser'
 email:
-- tagomoris@gmail.com
 - haruyama@unixuser.org
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -70,6 +69,8 @@ files:
 - fluent-plugin-referer-parser.gemspec
 - lib/fluent/plugin/.rubocop.yml
 - lib/fluent/plugin/out_referer_parser.rb
+- test/data/encodings.yaml
+- test/data/referers.yaml
 - test/helper.rb
 - test/plugin/.rubocop.yml
 - test/plugin/test_out_referer_parser.rb
@@ -83,12 +84,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
@@ -98,6 +99,8 @@ signing_key:
 specification_version: 4
 summary: Fluentd plugin to parse UserAgent strings
 test_files:
+- test/data/encodings.yaml
+- test/data/referers.yaml
 - test/helper.rb
 - test/plugin/.rubocop.yml
 - test/plugin/test_out_referer_parser.rb