fluent-plugin-referer-parser 0.0.10 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c5c18c51dd917efdde31a1b3569afc85f60186c7
-  data.tar.gz: 67bf6e6d8771b84fdf83d13f14e1fa81a2c5389d
+  metadata.gz: 7e2b1770c1c3d1a3fc00efce8fc3b2fd88496891
+  data.tar.gz: ee51c634a8b0cbf538e310c858d6bea6d512007e
 SHA512:
-  metadata.gz: e9ea39d3e47f6490fbed05d20a26ebf024e484c86b910328b45eed881886a8f2a3f405eb21e8647a690b3f2307a18f4328b97ed42868fb2ba832cc0fc2b8b505
-  data.tar.gz: 3883ba94f1f12970e9edf3ba8c6dd73c17fb980e137dfcf534bfa73d4162527b1b99e37081d84302a00db9dafb2d9577b53ac03ac1affde475e2ec03784f18ec
+  metadata.gz: eebfc2c7ceca2a7107d6b10a08f966ac7aa10826f25552b2b79073f970100d8045e9c6140b46c790004621ab19e9a8f91a966f6f70ee54bb6e73feafc3b52dcb
+  data.tar.gz: 1b283b06e7d82231b84b6ebab648c7f342150e562f325c456f503709e290cf01aed39d24dea99e225e1b4e0d2782db94904068dfb2af4204bf6c41c66ee466da

data/README.md

@@ -1,29 +1,31 @@
 # fluent-plugin-referer-parser, a plugin for [Fluentd](http://fluentd.org)
 
-## RefererParserOutput
+## RefererParserFilter
 
 'fluent-plugin-referer-parser' is a Fluentd plugin to parse Referer strings, based on [tagomoris/fluent-plugin-woothee](https://github.com/tagomoris/fluent-plugin-woothee).
 'fluent-plugin-referer-parser' uses [snowplow/referer-parser](https://github.com/snowplow/referer-parser).
 
+## Requirements
+
+| fluent-plugin-referer-parser | fluentd     | ruby   |
+|------------------------------|-------------|--------|
+| >= 0.1.0                     | >= v0.14.15 | >= 2.1 |
+| < 0.1.0                      | >= v0.12.0  | >= 1.9 |
 
 ## Configuration
 
 To add referer-parser result into matched messages:
 
-    <match input.**>
+    <filter>
       @type referer_parser
       key_name referer
-      remove_prefix input
-      add_prefix merged
     </match>
 
 Output messages with tag 'merged.**' has 'referer_known', 'referer_referer' and 'referer_search_term' attributes. If you want to change attribute names, write configurations as below:
 
-    <match input.**>
+    <filter>
       @type referer_parser
       key_name ref
-      remove_prefix input
-      add_prefix merged
       out_key_known        ref_known
       out_key_referer      ref_referer
       out_key_host         ref_host

data/fluent-plugin-referer-parser.gemspec

@@ -3,8 +3,8 @@ require 'English'
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-referer-parser'
-  gem.version       = '0.0.10'
-  gem.authors       = ['TAGOMORI Satoshi', 'HARUYAMA Seigo']
+  gem.version       = '0.1.0'
+  gem.authors       = ['HARUYAMA Seigo']
   gem.email         = ['haruyama@unixuser.org']
   gem.description   = %q(parsing by referer-parser. See: https://github.com/snowplow/referer-parser)
   gem.summary       = %q(Fluentd plugin to parse UserAgent strings)
@@ -18,6 +18,6 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'test-unit', '>= 3.2'
-  gem.add_runtime_dependency 'fluentd', '~> 0.12.0'
+  gem.add_runtime_dependency 'fluentd', '>= 0.14.15', '< 2'
   gem.add_runtime_dependency 'referer-parser', '~> 0.3.0'
 end

data/lib/fluent/plugin/filter_referer_parser.rb

@@ -2,7 +2,9 @@ require 'cgi'
 require 'yaml'
 require 'referer-parser'
 
-class Fluent::RefererParserFilter < Fluent::Filter
+require 'fluent/plugin/input'
+
+class Fluent::Plugin::RefererParserFilter < Fluent::Plugin::Filter
   Fluent::Plugin.register_filter('referer_parser', self)
 
   config_param :key_name,       :string

data/test/helper.rb

@@ -1,26 +1,6 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts 'Run `bundle install` to install missing gems'
-  exit e.status_code
-end
+require 'bundler/setup'
 require 'test/unit'
 
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
+$LOAD_PATH.unshift(__dir__)
 require 'fluent/test'
-unless ENV.key?('VERBOSE')
-  nulllogger = Object.new
-  nulllogger.instance_eval do |obj|
-    def method_missing(method, *args)
-      # pass
-    end
-  end
-  $log = nulllogger
-end
-
-class Test::Unit::TestCase
-end

data/test/plugin/test_filter_referer_parser.rb

@@ -1,7 +1,8 @@
 require 'helper'
+require 'fluent/test/driver/filter'
 require 'fluent/plugin/filter_referer_parser'
 
-class Fluent::RefererParserFilterTest < Test::Unit::TestCase
+class RefererParserFilterTest < Test::Unit::TestCase
   # through & merge
   CONFIG1 = %(
     key_name referer
@@ -28,20 +29,19 @@ class Fluent::RefererParserFilterTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
-  def create_driver(conf = CONFIG1, tag = 'test')
-    Fluent::Test::FilterTestDriver.new(Fluent::RefererParserFilter, tag).configure(conf)
+  def create_driver(conf = CONFIG1)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::RefererParserFilter).configure(conf)
   end
 
   def filter(config, messages)
     d = create_driver(config)
     time = Time.parse('2012-07-20 16:40:30').to_i
-    d.run do
+    d.run(default_tag: 'test') do
       messages.each do |message|
-        d.emit(message, time)
+        d.feed(time, message)
       end
     end
-    filtered = d.filtered_as_array
-    filtered.map {|m| m[2] }
+    d.filtered_records
   end
 
   sub_test_case 'configure' do

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-referer-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.1.0
 platform: ruby
 authors:
-- TAGOMORI Satoshi
 - HARUYAMA Seigo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-30 00:00:00.000000000 Z
+date: 2017-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -43,16 +42,22 @@ dependencies:
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.15
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.15
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: referer-parser
   requirement: !ruby/object:Gem::Requirement
@@ -83,13 +88,11 @@ files:
 - Rakefile
 - fluent-plugin-referer-parser.gemspec
 - lib/fluent/plugin/filter_referer_parser.rb
-- lib/fluent/plugin/out_referer_parser.rb
 - test/.rubocop.yml
 - test/data/encodings.yaml
 - test/data/referers.yaml
 - test/helper.rb
 - test/plugin/test_filter_referer_parser.rb
-- test/plugin/test_out_referer_parser.rb
 homepage: https://github.com/haruyama/fluent-plugin-referer-parser
 licenses:
 - Apache-2.0
@@ -120,4 +123,3 @@ test_files:
 - test/data/referers.yaml
 - test/helper.rb
 - test/plugin/test_filter_referer_parser.rb
-- test/plugin/test_out_referer_parser.rb

data/lib/fluent/plugin/out_referer_parser.rb

@@ -1,106 +0,0 @@
-# referer parser output
-class Fluent::RefererParserOutput < Fluent::Output
-  Fluent::Plugin.register_output('referer_parser', self)
-
-  config_param :tag,           :string, default: nil
-  config_param :remove_prefix, :string, default: nil
-  config_param :add_prefix,    :string, default: nil
-
-  config_param :key_name,       :string
-  config_param :referers_yaml,  :string, default: nil
-  config_param :encodings_yaml, :string, default: nil
-
-  config_param :out_key_known,       :string, default: 'referer_known'
-  config_param :out_key_referer,     :string, default: 'referer_referer'
-  config_param :out_key_host,        :string, default: 'referer_host'
-  config_param :out_key_search_term, :string, default: 'referer_search_term'
-
-  def initialize
-    super
-    require 'cgi'
-    require 'yaml'
-    require 'referer-parser'
-  end
-
-  def configure(conf)
-    super
-
-    @referer_parser = if @referers_yaml
-                        RefererParser::Parser.new(@referers_yaml)
-                      else
-                        RefererParser::Parser.new
-                      end
-
-    if @encodings_yaml
-      @encodings = YAML.load_file(@encodings_yaml)
-    else
-      @encodings = {}
-    end
-
-    if !@tag && !@remove_prefix && !@add_prefix
-      fail Fluent::ConfigError, 'missing both of remove_prefix and add_prefix'
-    end
-    if @tag && (@remove_prefix || @add_prefix)
-      fail Fluent::ConfigError, 'both of tag and remove_prefix/add_prefix must not be specified'
-    end
-    if @remove_prefix
-      @removed_prefix_string = @remove_prefix + '.'
-      @removed_length = @removed_prefix_string.length
-    end
-    @added_prefix_string = @add_prefix + '.' if @add_prefix
-  end
-
-  def tag_mangle(tag)
-    if @tag
-      @tag
-    else
-      if @remove_prefix &&
-          ( (tag.start_with?(@removed_prefix_string) && tag.length > @removed_length) || tag == @remove_prefix)
-        tag = tag[@removed_length..-1]
-      end
-      if @add_prefix
-        tag = if tag && tag.length > 0
-                @added_prefix_string + tag
-              else
-                @add_prefix
-              end
-      end
-      tag
-    end
-  end
-
-  def emit(tag, es, chain)
-    tag = tag_mangle(tag)
-    es.each do |time, record|
-      is_valid = true
-      parsed = begin
-                 @referer_parser.parse(record[@key_name])
-               rescue
-                 is_valid = false
-                 {}
-               end
-      if is_valid && parsed[:known]
-        search_term = parsed[:term]
-        uri = URI.parse(parsed[:uri])
-        host = uri.host
-        parameters = CGI.parse(uri.query)
-        input_encoding = @encodings[host] || parameters['ie'][0] || parameters['ei'][0]
-        begin
-          search_term = search_term.force_encoding(input_encoding).encode('utf-8') if input_encoding && /\Autf-?8\z/i !~ input_encoding
-        rescue
-          log.error('invalid referer: ' + uri.to_s)
-        end
-        record.merge!(
-          @out_key_known       => true,
-          @out_key_referer     => parsed[:source],
-          @out_key_host        => host,
-          @out_key_search_term => search_term
-        )
-      else
-        record[@out_key_known] = false
-      end
-      router.emit(tag, time, record)
-    end
-    chain.next
-  end
-end

data/test/plugin/test_out_referer_parser.rb

@@ -1,227 +0,0 @@
-require 'helper'
-require 'fluent/plugin/out_referer_parser'
-
-# RefererParserOutput test
-class Fluent::RefererParserOutputTest < Test::Unit::TestCase
-  # through & merge
-  CONFIG1 = %(
-type referer_parser
-key_name referer
-remove_prefix test
-add_prefix merged
-)
-
-  CONFIG2 = %(
-type referer_parser
-key_name ref
-remove_prefix test
-add_prefix merged
-out_key_known        ref_known
-out_key_referer      ref_referer
-out_key_host         ref_host
-out_key_search_term  ref_search_term
-)
-
-  CONFIG3 = %(
-type referer_parser
-key_name ref
-remove_prefix test
-add_prefix merged
-referers_yaml test/data/referers.yaml
-encodings_yaml test/data/encodings.yaml
-)
-
-  def setup
-    Fluent::Test.setup
-  end
-
-  def create_driver(conf = CONFIG1, tag = 'test')
-    Fluent::Test::OutputTestDriver.new(Fluent::RefererParserOutput, tag).configure(conf)
-  end
-
-  def test_configure
-    # through & merge
-    d = create_driver CONFIG1
-    assert_equal 'referer', d.instance.key_name
-    assert_equal 'test',    d.instance.remove_prefix
-    assert_equal 'merged',  d.instance.add_prefix
-
-    assert_equal 'referer_known',       d.instance.out_key_known
-    assert_equal 'referer_referer',     d.instance.out_key_referer
-    assert_equal 'referer_search_term', d.instance.out_key_search_term
-
-    # filter & merge
-    d = create_driver CONFIG2
-    assert_equal 'ref',    d.instance.key_name
-    assert_equal 'test',   d.instance.remove_prefix
-    assert_equal 'merged', d.instance.add_prefix
-
-    assert_equal 'ref_known',       d.instance.out_key_known
-    assert_equal 'ref_referer',     d.instance.out_key_referer
-    assert_equal 'ref_search_term', d.instance.out_key_search_term
-  end
-
-  def test_tag_mangle
-    p = create_driver(CONFIG1).instance
-    assert_equal 'merged.data', p.tag_mangle('data')
-    assert_equal 'merged.data', p.tag_mangle('test.data')
-    assert_equal 'merged.test.data', p.tag_mangle('test.test.data')
-    assert_equal 'merged', p.tag_mangle('test')
-  end
-
-  def test_emit1
-    d = create_driver(CONFIG1, 'test.message')
-    time = Time.parse('2012-07-20 16:40:30').to_i
-    d.run do
-      d.emit({ 'value' => 0 }, time)
-      d.emit({ 'value' => 1, 'referer' => 'http://www.google.com/search?q=gateway+oracle+cards+denise+linn&hl=en&client=safari' }, time)
-      d.emit({ 'value' => 2, 'referer' => 'http://www.unixuser.org/' }, time)
-      d.emit({ 'value' => 3, 'referer' => 'http://www.google.co.jp/search?hl=ja&ie=Shift_JIS&c2coff=1&q=%83%7D%83%8B%83%60%83L%83%83%83X%83g%81@%8Aw%8Em%98_%95%B6&lr=' }, time)
-      d.emit({ 'value' => 4, 'referer' => 'http://www.google.co.jp/search?hl=ja&ie=Shift_J&c2coff=1&q=%83%7D%83%8B%83%60%83L%83%83%83X%83g%81@%8Aw%8Em%98_%95%B6&lr=' }, time)
-      d.emit({ 'value' => 5, 'referer' => 'http://search.yahoo.co.jp/search?p=%E3%81%BB%E3%81%92&aq=-1&oq=&ei=UTF-8&fr=sfp_as&x=wrt' }, time)
-    end
-
-    emits = d.emits
-    assert_equal 6,                emits.size
-    assert_equal 'merged.message', emits[0][0]
-    assert_equal time,             emits[0][1]
-
-    m = emits[0][2]
-    assert_equal 0,         m['value']
-    assert_equal false,     m['referer_known']
-    assert_nil m['referer_referer']
-    assert_nil m['referer_search_term']
-    assert_equal 2,         m.keys.size
-
-    m = emits[1][2]
-    assert_equal 1,                                  m['value']
-    assert_equal true,                               m['referer_known']
-    assert_equal 'Google',                           m['referer_referer']
-    assert_equal 'www.google.com',                   m['referer_host']
-    assert_equal 'gateway oracle cards denise linn', m['referer_search_term']
-    assert_equal 6,         m.keys.size
-
-    m = emits[2][2]
-    assert_equal 2,         m['value']
-    assert_equal false,     m['referer_known']
-    assert_nil m['referer_referer']
-    assert_nil m['referer_search_term']
-    assert_equal 3,         m.keys.size
-
-    m = emits[3][2]
-    assert_equal 3,                          m['value']
-    assert_equal true,                       m['referer_known']
-    assert_equal 'Google',                   m['referer_referer']
-    assert_equal 'www.google.co.jp',         m['referer_host']
-    assert_equal 'マルチキャスト　学士論文', m['referer_search_term']
-    assert_equal 6,         m.keys.size
-
-    # invalid input_encoding
-    m = emits[4][2]
-    assert_equal 4,        m['value']
-    assert_equal true,     m['referer_known']
-    assert_equal 'Google', m['referer_referer']
-    assert_equal 'www.google.co.jp',         m['referer_host']
-    assert_equal 6,        m.keys.size
-
-    m = emits[5][2]
-    assert_equal 5,                    m['value']
-    assert_equal true,                 m['referer_known']
-    assert_equal 'Yahoo!',             m['referer_referer']
-    assert_equal 'search.yahoo.co.jp', m['referer_host']
-    assert_equal 'ほげ',               m['referer_search_term']
-    assert_equal 6,                    m.keys.size
-  end
-
-  def test_emit2
-    d = create_driver(CONFIG2, 'test.message')
-    time = Time.parse('2012-07-20 16:40:30').to_i
-    d.run do
-      d.emit({ 'value' => 0 }, time)
-      d.emit({ 'value' => 1, 'ref' => 'http://www.google.com/search?q=gateway+oracle+cards+denise+linn&hl=en&client=safari' }, time)
-      d.emit({ 'value' => 2, 'ref' => 'http://www.unixuser.org/' }, time)
-      d.emit({ 'value' => 3, 'ref' => 'https://www.google.com/search?q=%E3%81%BB%E3%81%92&ie=utf-8&oe=utf-8' }, time)
-    end
-
-    emits = d.emits
-    assert_equal 4,                emits.size
-    assert_equal 'merged.message', emits[0][0]
-    assert_equal time,             emits[0][1]
-
-    m = emits[0][2]
-    assert_equal 0,         m['value']
-    assert_equal false,     m['ref_known']
-    assert_nil m['ref_referer']
-    assert_nil m['ref_search_term']
-    assert_equal 2,         m.keys.size
-
-    m = emits[1][2]
-    assert_equal 1,                                  m['value']
-    assert_equal true,                               m['ref_known']
-    assert_equal 'Google',                           m['ref_referer']
-    assert_equal 'www.google.com',                   m['ref_host']
-    assert_equal 'gateway oracle cards denise linn', m['ref_search_term']
-    assert_equal 6,         m.keys.size
-
-    m = emits[2][2]
-    assert_equal 2,         m['value']
-    assert_equal false,     m['ref_known']
-    assert_nil m['ref_referer']
-    assert_nil m['ref_host']
-    assert_nil m['ref_search_term']
-
-    m = emits[3][2]
-    assert_equal 3,                m['value']
-    assert_equal true,             m['ref_known']
-    assert_equal 'Google',         m['ref_referer']
-    assert_equal 'www.google.com', m['ref_host']
-    assert_equal 'ほげ',           m['ref_search_term']
-  end
-
-  def test_emit3
-    d = create_driver(CONFIG3, 'test.message')
-    time = Time.parse('2012-07-20 16:40:30').to_i
-    d.run do
-      d.emit({ 'value' => 0 }, time)
-      d.emit({ 'value' => 1, 'ref' => 'http://ezsch.ezweb.ne.jp/search/?sr=0101&query=aiueo%20%95a%93I' }, time)
-      d.emit({ 'value' => 2, 'ref' => 'http://ezsch.ezweb.ne.jp/search/ezGoogleMain.php?query=%83%8D' }, time)
-      d.emit({ 'value' => 3, 'ref' => 'http://www.google.co.jp/search?hl=ja&ie=Shift_JIS&c2coff=1&q=%83%7D%83%8B%83%60%83L%83%83%83X%83g%81@%8Aw%8Em%98_%95%B6&lr=' }, time)
-    end
-
-    emits = d.emits
-    assert_equal 4,                emits.size
-    assert_equal 'merged.message', emits[0][0]
-    assert_equal time,             emits[0][1]
-
-    m = emits[0][2]
-    assert_equal 0,     m['value']
-    assert_equal false, m['referer_known']
-    assert_nil m['referer_referer']
-    assert_nil m['referer_search_term']
-    assert_equal 2,     m.keys.size
-
-    m = emits[1][2]
-    assert_equal 1,                   m['value']
-    assert_equal true,                m['referer_known']
-    assert_equal 'Ezweb',             m['referer_referer']
-    assert_equal 'ezsch.ezweb.ne.jp', m['referer_host']
-    assert_equal 'aiueo 病的',        m['referer_search_term']
-    assert_equal 6,                   m.keys.size
-
-    m = emits[2][2]
-    assert_equal 2,                   m['value']
-    assert_equal true,                m['referer_known']
-    assert_equal 'Ezweb',             m['referer_referer']
-    assert_equal 'ezsch.ezweb.ne.jp', m['referer_host']
-    assert_equal 'ロ',                m['referer_search_term']
-    assert_equal 6,                   m.keys.size
-
-    m = emits[3][2]
-    assert_equal 3,                          m['value']
-    assert_equal true,                       m['referer_known']
-    assert_equal 'Google',                   m['referer_referer']
-    assert_equal 'www.google.co.jp',         m['referer_host']
-    assert_equal 'マルチキャスト　学士論文', m['referer_search_term']
-    assert_equal 6,         m.keys.size
-  end
-end