fluent-plugin-woothee 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-woothee.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2012- TAGOMORI Satoshi
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,79 @@
+# fluent-plugin-woothee
+
+## WootheeOutput
+
+'fluent-plugin-woothee' is a Fluentd plugin to parse UserAgent strings and to filter/drop specified categories of user terminals (like 'pc', 'smartphone' and so on).
+
+'woothee' is multi-language user-agent strings parser project. See: https://github.com/tagomoris/woothee
+
+## Configuration
+
+To add woothee parser result into matched messages:
+
+    <match input.**>
+      type woothee
+      key_name agent
+      remove_prefix input
+      add_prefix merged
+      merge_agent_info yes
+    </match>
+
+Output messages with tag 'merged.**' has attributes like 'agent\_name', 'agent\_category' and 'agent\_os' from woothee parser result. If you want to change attribute names, or want to merge more attributes of browser vendor and its version, write configurations as below:
+
+    <match input.**>
+      type woothee
+      key_name agent
+      remove_prefix input
+      add_prefix merged
+      merge_agent_info yes
+      out_key_name ua_name
+      out_key_category ua_category
+      out_key_os ua_os
+      out_key_version ua_version
+      out_key_vendor ua_vendor
+    </match>
+
+To re-emit messages with specified user-agent categories (and merge woothee parser result), configure like this:
+
+    <match input.**>
+      type woothee
+      key_name agent
+      filter_categories pc,smartphone,mobilephone,appliance
+      remove_prefix input
+      add_prefix merged
+      merge_agent_info yes
+    </match>
+
+Or, you can specify categories to drop (and not to merge woothee result):
+
+    <match input.**>
+      type woothee
+      key_name agent
+      drop_categories crawler
+      remove_prefix input
+      add_prefix merged
+      merge_agent_info false # default
+    </match>
+
+### Fast Crawler Filter
+
+If you want to drop __almost__ all of messages with crawler's user-agent, and not to merge woothee result, you just specify plugin type:
+
+    <match input.**>
+      type woothee_fast_crawler_filter
+      key_name useragent
+      tag filtered
+    </match>
+
+'fluent-plugin-woothee' uses 'Woothee.is_crawler' of woothee with this configuration, fast and incomplete method to judge user-agent is crawler or not.
+If you want to drop all of crawlers completely, specify 'type woothee' and 'drop_categories crawler'.
+
+## TODO
+
+* patches welcome!
+
+## Copyright
+
+* Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
+* License
+  * Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-woothee.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-woothee"
+  gem.version       = "0.0.1"
+  gem.authors       = ["TAGOMORI Satoshi"]
+  gem.email         = ["tagomoris@gmail.com"]
+  gem.description   = %q{parsing by Project Woothee. See https://github.com/tagomoris/woothee }
+  gem.summary       = %q{Fluentd plugin to parse UserAgent strings}
+  gem.homepage      = "https://github.com/tagomoris/fluent-plugin-woothee"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "fluentd"
+  gem.add_development_dependency "woothee", ">= 0.2.4"
+  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "woothee", ">= 0.2.4"
+end

data/lib/fluent/plugin/out_woothee.rb

@@ -0,0 +1,142 @@
+class Fluent::WootheeOutput < Fluent::Output
+  Fluent::Plugin.register_output('woothee', self)
+  Fluent::Plugin.register_output('woothee_fast_crawler_filter', self)
+
+  config_param :tag, :string, :default => nil
+  config_param :remove_prefix, :string, :default => nil
+  config_param :add_prefix, :string, :default => nil
+
+  config_param :fast_crawler_filter_mode, :bool, :default => false
+
+  config_param :key_name, :string
+
+  config_param :filter_categories, :default => [] do |val|
+    val.split(',').map(&:to_sym)
+  end
+  config_param :drop_categories, :default => [] do |val|
+    val.split(',').map(&:to_sym)
+  end
+  attr_accessor :mode
+
+  config_param :merge_agent_info, :bool, :default => false
+  config_param :out_key_name, :string, :default => 'agent_name'
+  config_param :out_key_category, :string, :default => 'agent_category'
+  config_param :out_key_os, :string, :default => 'agent_os'
+  config_param :out_key_version, :string, :default => nil # supress output
+  config_param :out_key_vendor, :string, :default => nil # supress output
+
+  def initialize
+    super
+    require 'woothee'
+  end
+
+  def configure(conf)
+    super
+
+    # tag ->
+    if not @tag and not @remove_prefix and not @add_prefix
+      raise Fluent::ConfigError, "missing both of remove_prefix and add_prefix"
+    end
+    if @tag and (@remove_prefix or @add_prefix)
+      raise Fluent::ConfigError, "both of tag and remove_prefix/add_prefix must not be specified"
+    end
+    if @remove_prefix
+      @removed_prefix_string = @remove_prefix + '.'
+      @removed_length = @removed_prefix_string.length
+    end
+    if @add_prefix
+      @added_prefix_string = @add_prefix + '.'
+    end
+    # <- tag
+
+    if conf['type'] == 'woothee_fast_crawler_filter' or @fast_crawler_filter_mode
+      @fast_crawler_filter_mode = true
+
+      if @filter_categories.size > 0 or @drop_categories.size > 0 or @merge_agent_info
+        raise Fluent::ConfigError, "fast_crawler_filter cannot be specified with filter/drop/merge options"
+      end
+      
+      return
+    end
+
+    if @filter_categories.size > 0 and @drop_categories.size > 0
+      raise Fluent::ConfigError, "both of 'filter' and 'drop' categories specified"
+    elsif @filter_categories.size > 0
+      unless @filter_categories.reduce(true){|r,i| r and Woothee::CATEGORY_LIST.include?(i)}
+        raise Fluent::ConfigError, "filter_categories has invalid category name"
+      end
+      @mode = :filter
+    elsif @drop_categories.size > 0
+      unless @drop_categories.reduce(true){|r,i| r and Woothee::CATEGORY_LIST.include?(i)}
+        raise Fluent::ConfigError, "drop_categories has invalid category name"
+      end
+      @mode = :drop
+    else
+      @mode = :through
+    end
+
+    if @mode == :through and not @merge_agent_info
+      raise Fluent::ConfigError, "configured not to do nothing (not to do either filter/drop nor addition of parser result)"
+    end
+  end
+
+  def tag_mangle(tag)
+    if @tag
+      @tag
+    else
+      if @remove_prefix and
+          ( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @remove_prefix)
+        tag = tag[@removed_length..-1]
+      end 
+      if @add_prefix
+        tag = if tag and tag.length > 0
+                @added_prefix_string + tag
+              else
+                @add_prefix
+              end
+      end
+      tag
+    end
+  end
+
+  def fast_crawler_filter_emit(tag, es)
+    es.each do |time,record|
+      unless Woothee.is_crawler(record[@key_name] || '')
+        Fluent::Engine.emit(tag, time, record)
+      end
+    end
+  end
+
+  def normal_emit(tag, es)
+    es.each do |time,record|
+      parsed = Woothee.parse(record[@key_name] || '')
+
+      category = parsed[Woothee::ATTRIBUTE_CATEGORY]
+      next if @mode == :filter and not @filter_categories.include?(category)
+      next if @mode == :drop and @drop_categories.include?(category)
+
+      if @merge_agent_info
+        record = record.merge({
+            @out_key_name => parsed[Woothee::ATTRIBUTE_NAME],
+            @out_key_category => parsed[Woothee::ATTRIBUTE_CATEGORY].to_s,
+            @out_key_os => parsed[Woothee::ATTRIBUTE_OS]
+          })
+        record[@out_key_version] = parsed[Woothee::ATTRIBUTE_VERSION] if @out_key_version
+        record[@out_key_vendor] = parsed[Woothee::ATTRIBUTE_VENDOR] if @out_key_vendor
+      end
+      Fluent::Engine.emit(tag, time, record)
+    end
+  end
+
+  def emit(tag, es, chain)
+    tag = tag_mangle(tag)
+
+    if @fast_crawler_filter_mode
+      fast_crawler_filter_emit(tag, es)
+    else
+      normal_emit(tag, es)
+    end
+
+    chain.next
+  end
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_woothee'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_woothee.rb

@@ -0,0 +1,352 @@
+require 'helper'
+
+class Fluent::WootheeOutputTest < Test::Unit::TestCase
+  # fast crawler filter
+  CONFIG0 = %[
+type woothee_fast_crawler_filter
+key_name useragent
+tag filtered
+]
+
+  # through & merge
+  CONFIG1 = %[
+type woothee
+key_name agent
+remove_prefix test
+add_prefix merged
+merge_agent_info yes
+]
+
+  # filter & merge
+  CONFIG2 = %[
+type woothee
+key_name agent
+filter_categories pc,smartphone,mobilephone,appliance
+remove_prefix test
+add_prefix merged
+merge_agent_info yes
+out_key_name ua_name
+out_key_category ua_category
+out_key_os ua_os
+out_key_version ua_version
+out_key_vendor ua_vendor
+]
+
+  # drop & non-merge
+  CONFIG3 = %[
+type woothee
+key_name user_agent
+drop_categories crawler,misc
+tag selected
+]
+
+  def create_driver(conf=CONFIG1,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::WootheeOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    # fast_crawler_filter
+    d = create_driver CONFIG0
+    assert_equal true, d.instance.fast_crawler_filter_mode
+    assert_equal 'useragent', d.instance.key_name
+    assert_equal 'filtered', d.instance.tag
+
+    # through & merge
+    d = create_driver CONFIG1
+    assert_equal false, d.instance.fast_crawler_filter_mode
+    assert_equal 'agent', d.instance.key_name
+    assert_equal 'test', d.instance.remove_prefix
+    assert_equal 'merged', d.instance.add_prefix
+
+    assert_equal 0, d.instance.filter_categories.size
+    assert_equal 0, d.instance.drop_categories.size
+    assert_equal :through, d.instance.mode
+
+    assert_equal true, d.instance.merge_agent_info
+    assert_equal 'agent_name', d.instance.out_key_name
+    assert_equal 'agent_category', d.instance.out_key_category
+    assert_equal 'agent_os', d.instance.out_key_os
+    assert_nil d.instance.out_key_version
+    assert_nil d.instance.out_key_vendor
+
+    # filter & merge
+    d = create_driver CONFIG2
+    assert_equal false, d.instance.fast_crawler_filter_mode
+    assert_equal 'agent', d.instance.key_name
+    assert_equal 'test', d.instance.remove_prefix
+    assert_equal 'merged', d.instance.add_prefix
+
+    assert_equal 4, d.instance.filter_categories.size
+    assert_equal [:pc,:smartphone,:mobilephone,:appliance], d.instance.filter_categories
+    assert_equal 0, d.instance.drop_categories.size
+    assert_equal :filter, d.instance.mode
+
+    assert_equal true, d.instance.merge_agent_info
+    assert_equal 'ua_name', d.instance.out_key_name
+    assert_equal 'ua_category', d.instance.out_key_category
+    assert_equal 'ua_os', d.instance.out_key_os
+    assert_equal 'ua_version', d.instance.out_key_version
+    assert_equal 'ua_vendor', d.instance.out_key_vendor
+
+    # drop & non-merge
+    d = create_driver CONFIG3
+    assert_equal false, d.instance.fast_crawler_filter_mode
+    assert_equal 'user_agent', d.instance.key_name
+    assert_equal 'selected', d.instance.tag
+
+    assert_equal 0, d.instance.filter_categories.size
+    assert_equal 2, d.instance.drop_categories.size
+    assert_equal [:crawler,:misc], d.instance.drop_categories
+    assert_equal :drop, d.instance.mode
+
+    assert_equal false, d.instance.merge_agent_info
+  end
+
+  def test_tag_mangle
+    p = create_driver(CONFIG0).instance
+    assert_equal 'filtered', p.tag_mangle('data')
+    assert_equal 'filtered', p.tag_mangle('test.data')
+    assert_equal 'filtered', p.tag_mangle('test.test.data')
+    assert_equal 'filtered', p.tag_mangle('test')
+
+    p = create_driver(CONFIG1).instance
+    assert_equal 'merged.data', p.tag_mangle('data')
+    assert_equal 'merged.data', p.tag_mangle('test.data')
+    assert_equal 'merged.test.data', p.tag_mangle('test.test.data')
+    assert_equal 'merged', p.tag_mangle('test')
+
+    p = create_driver(CONFIG3).instance
+    assert_equal 'selected', p.tag_mangle('data')
+    assert_equal 'selected', p.tag_mangle('test.data')
+    assert_equal 'selected', p.tag_mangle('test.test.data')
+    assert_equal 'selected', p.tag_mangle('test')
+  end
+
+  def test_emit_fast_crawler_filter
+    d = create_driver CONFIG0
+    time = Time.parse('2012-07-20 16:19:00').to_i
+    d.run do
+      d.emit({'useragent' => 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', 'value' => 1}, time)
+      d.emit({'useragent' => 'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)', 'value' => 2}, time)
+      d.emit({'useragent' => 'Mozilla/5.0 (iPad; U; CPU OS 4_3_2 like Mac OS X; ja-jp) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5', 'value' => 3}, time)
+      d.emit({'useragent' => 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)', 'value' => 4}, time)
+      d.emit({'useragent' => 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)', 'value' => 5}, time)
+      d.emit({'useragent' => 'Mozilla/5.0 (compatible; Rakutenbot/1.0; +http://dynamic.rakuten.co.jp/bot.html)', 'value' => 6}, time)
+      d.emit({'useragent' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_4; ja-jp) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.20.1', 'value' => 7}, time)
+      d.emit({'useragent' => 'Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/)', 'value' => 8}, time)
+    end
+
+    emits = d.emits
+    assert_equal 4, emits.size
+
+    assert_equal 'filtered', emits[0][0]
+    assert_equal time, emits[0][1]
+    assert_equal 'Mozilla/5.0 (iPad; U; CPU OS 4_3_2 like Mac OS X; ja-jp) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5', emits[0][2]['useragent']
+    assert_equal 3, emits[0][2]['value']
+    assert_equal 2, emits[0][2].keys.size
+
+    assert_equal 4, emits[1][2]['value']
+    assert_equal 6, emits[2][2]['value']
+    assert_equal 7, emits[3][2]['value']
+  end
+
+#   # through & merge
+  def test_emit_through
+    d = create_driver(CONFIG1, 'test.message')
+    time = Time.parse('2012-07-20 16:40:30').to_i
+    d.run do
+      d.emit({'value' => 0, 'agent' => 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)'}, time)
+      d.emit({'value' => 1, 'agent' => 'Mozilla/5.0 (Windows NT 6.0; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'}, time)
+      d.emit({'value' => 2, 'agent' => 'Mozilla/5.0 (Ubuntu; X11; Linux i686; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'}, time)
+      d.emit({'value' => 3, 'agent' => 'Mozilla/5.0 (Linux; U; Android 3.1; ja-jp; L-06C Build/HMJ37) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13'}, time)
+      d.emit({'value' => 4, 'agent' => 'DoCoMo/1.0/N505i/c20/TB/W24H12'}, time)
+      d.emit({'value' => 5, 'agent' => 'Mozilla/5.0 (PlayStation Vita 1.51) AppleWebKit/531.22.8 (KHTML, like Gecko) Silk/3.2'}, time)
+      d.emit({'value' => 6, 'agent' => 'Mozilla/5.0 (compatible; Google Desktop/5.9.1005.12335; http://desktop.google.com/)'}, time)
+      d.emit({'value' => 7, 'agent' => 'msnbot/1.1 (+http://search.msn.com/msnbot.htm)'}, time)
+    end
+
+    emits = d.emits
+    assert_equal 8, emits.size
+    assert_equal 'merged.message', emits[0][0]
+    assert_equal time, emits[0][1]
+
+    # 'agent' => 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)'
+    m = emits[0][2]
+    assert_equal 0, m['value']
+    assert_equal 'Internet Explorer', m['agent_name']
+    assert_equal 'pc', m['agent_category']
+    assert_equal 'Windows 8', m['agent_os']
+    assert_equal 5, m.keys.size
+
+    # 'agent' => 'Mozilla/5.0 (Windows NT 6.0; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'
+    m = emits[1][2]
+    assert_equal 1, m['value']
+    assert_equal 'Firefox', m['agent_name']
+    assert_equal 'pc', m['agent_category']
+    assert_equal 'Windows Vista', m['agent_os']
+
+    # 'agent' => 'Mozilla/5.0 (Ubuntu; X11; Linux i686; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'
+    m = emits[2][2]
+    assert_equal 2, m['value']
+    assert_equal 'Firefox', m['agent_name']
+    assert_equal 'pc', m['agent_category']
+    assert_equal 'Linux', m['agent_os']
+
+    # 'agent' => 'Mozilla/5.0 (Linux; U; Android 3.1; ja-jp; L-06C Build/HMJ37) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13'
+    m = emits[3][2]
+    assert_equal 3, m['value']
+    assert_equal 'Safari', m['agent_name']
+    assert_equal 'smartphone', m['agent_category']
+    assert_equal 'Android', m['agent_os']
+
+    # 'agent' => 'DoCoMo/1.0/N505i/c20/TB/W24H12'
+    m = emits[4][2]
+    assert_equal 4, m['value']
+    assert_equal 'docomo', m['agent_name']
+    assert_equal 'mobilephone', m['agent_category']
+    assert_equal 'docomo', m['agent_os']
+
+    # 'agent' => 'Mozilla/5.0 (PlayStation Vita 1.51) AppleWebKit/531.22.8 (KHTML, like Gecko) Silk/3.2'
+    m = emits[5][2]
+    assert_equal 5, m['value']
+    assert_equal 'PlayStation Vita', m['agent_name']
+    assert_equal 'appliance', m['agent_category']
+    assert_equal 'PlayStation Vita', m['agent_os']
+
+    # 'agent' => 'Mozilla/5.0 (compatible; Google Desktop/5.9.1005.12335; http://desktop.google.com/)'
+    m = emits[6][2]
+    assert_equal 6, m['value']
+    assert_equal 'Google Desktop', m['agent_name']
+    assert_equal 'misc', m['agent_category']
+    assert_equal 'UNKNOWN', m['agent_os']
+
+    # 'agent' => 'msnbot/1.1 (+http://search.msn.com/msnbot.htm)'
+    m = emits[7][2]
+    assert_equal 7, m['value']
+    assert_equal 'msnbot', m['agent_name']
+    assert_equal 'crawler', m['agent_category']
+    assert_equal 'UNKNOWN', m['agent_os']
+  end
+
+#   # filter & merge
+  def test_emit_filter
+    d = create_driver(CONFIG2, 'test.message')
+    time = Time.parse('2012-07-20 16:40:30').to_i
+    d.run do
+      d.emit({'value' => 0, 'agent' => 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)'}, time)
+      d.emit({'value' => 1, 'agent' => 'Mozilla/5.0 (Windows NT 6.0; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'}, time)
+      d.emit({'value' => 2, 'agent' => 'Mozilla/5.0 (Ubuntu; X11; Linux i686; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'}, time)
+      d.emit({'value' => 3, 'agent' => 'Mozilla/5.0 (Linux; U; Android 3.1; ja-jp; L-06C Build/HMJ37) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13'}, time)
+      d.emit({'value' => 4, 'agent' => 'DoCoMo/1.0/N505i/c20/TB/W24H12'}, time)
+      d.emit({'value' => 5, 'agent' => 'Mozilla/5.0 (PlayStation Vita 1.51) AppleWebKit/531.22.8 (KHTML, like Gecko) Silk/3.2'}, time)
+      d.emit({'value' => 6, 'agent' => 'Mozilla/5.0 (compatible; Google Desktop/5.9.1005.12335; http://desktop.google.com/)'}, time)
+      d.emit({'value' => 7, 'agent' => 'msnbot/1.1 (+http://search.msn.com/msnbot.htm)'}, time)
+    end
+
+    emits = d.emits
+    assert_equal 6, emits.size
+    assert_equal 'merged.message', emits[0][0]
+    assert_equal time, emits[0][1]
+
+    # 'agent' => 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)'
+    m = emits[0][2]
+    assert_equal 7, m.keys.size
+    assert_equal 0, m['value']
+    assert_equal 'Internet Explorer', m['ua_name']
+    assert_equal 'pc', m['ua_category']
+    assert_equal 'Windows 8', m['ua_os']
+    assert_equal 'Microsoft', m['ua_vendor']
+    assert_equal '10.0', m['ua_version']
+
+    # 'agent' => 'Mozilla/5.0 (Windows NT 6.0; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'
+    m = emits[1][2]
+    assert_equal 1, m['value']
+    assert_equal 'Firefox', m['ua_name']
+    assert_equal 'pc', m['ua_category']
+    assert_equal 'Windows Vista', m['ua_os']
+    assert_equal 'Mozilla', m['ua_vendor']
+    assert_equal '9.0.1', m['ua_version']
+
+    # 'agent' => 'Mozilla/5.0 (Ubuntu; X11; Linux i686; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'
+    m = emits[2][2]
+    assert_equal 2, m['value']
+    assert_equal 'Firefox', m['ua_name']
+    assert_equal 'pc', m['ua_category']
+    assert_equal 'Linux', m['ua_os']
+    assert_equal 'Mozilla', m['ua_vendor']
+    assert_equal '9.0.1', m['ua_version']
+
+    # 'agent' => 'Mozilla/5.0 (Linux; U; Android 3.1; ja-jp; L-06C Build/HMJ37) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13'
+    m = emits[3][2]
+    assert_equal 3, m['value']
+    assert_equal 'Safari', m['ua_name']
+    assert_equal 'smartphone', m['ua_category']
+    assert_equal 'Android', m['ua_os']
+    assert_equal 'Apple', m['ua_vendor']
+    assert_equal '4.0', m['ua_version']
+
+    # 'agent' => 'DoCoMo/1.0/N505i/c20/TB/W24H12'
+    m = emits[4][2]
+    assert_equal 4, m['value']
+    assert_equal 'docomo', m['ua_name']
+    assert_equal 'mobilephone', m['ua_category']
+    assert_equal 'docomo', m['ua_os']
+    assert_equal 'docomo', m['ua_vendor']
+    assert_equal 'N505i', m['ua_version']
+
+    # 'agent' => 'Mozilla/5.0 (PlayStation Vita 1.51) AppleWebKit/531.22.8 (KHTML, like Gecko) Silk/3.2'
+    m = emits[5][2]
+    assert_equal 5, m['value']
+    assert_equal 'PlayStation Vita', m['ua_name']
+    assert_equal 'appliance', m['ua_category']
+    assert_equal 'PlayStation Vita', m['ua_os']
+    assert_equal 'Sony', m['ua_vendor']
+    assert_equal 'UNKNOWN', m['ua_version']
+  end
+
+#   # drop & non-merge
+  def test_emit_drop
+    d = create_driver(CONFIG3, 'test.message')
+    time = Time.parse('2012-07-20 16:40:30').to_i
+    d.run do
+      d.emit({'value' => 0, 'user_agent' => 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)'}, time)
+      d.emit({'value' => 1, 'user_agent' => 'Mozilla/5.0 (Windows NT 6.0; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'}, time)
+      d.emit({'value' => 2, 'user_agent' => 'Mozilla/5.0 (Ubuntu; X11; Linux i686; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'}, time)
+      d.emit({'value' => 3, 'user_agent' => 'Mozilla/5.0 (Linux; U; Android 3.1; ja-jp; L-06C Build/HMJ37) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13'}, time)
+      d.emit({'value' => 4, 'user_agent' => 'DoCoMo/1.0/N505i/c20/TB/W24H12'}, time)
+      d.emit({'value' => 5, 'user_agent' => 'Mozilla/5.0 (PlayStation Vita 1.51) AppleWebKit/531.22.8 (KHTML, like Gecko) Silk/3.2'}, time)
+      d.emit({'value' => 6, 'user_agent' => 'Mozilla/5.0 (compatible; Google Desktop/5.9.1005.12335; http://desktop.google.com/)'}, time)
+      d.emit({'value' => 7, 'user_agent' => 'msnbot/1.1 (+http://search.msn.com/msnbot.htm)'}, time)
+    end
+
+    emits = d.emits
+    assert_equal 6, emits.size
+    assert_equal 'selected', emits[0][0]
+    assert_equal time, emits[0][1]
+
+    # 'agent' => 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)'
+    m = emits[0][2]
+    assert_equal 0, m['value']
+    assert_equal 2, m.keys.size
+
+    # 'agent' => 'Mozilla/5.0 (Windows NT 6.0; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'
+    m = emits[1][2]
+    assert_equal 1, m['value']
+
+    # 'agent' => 'Mozilla/5.0 (Ubuntu; X11; Linux i686; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'
+    m = emits[2][2]
+    assert_equal 2, m['value']
+
+    # 'agent' => 'Mozilla/5.0 (Linux; U; Android 3.1; ja-jp; L-06C Build/HMJ37) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13'
+    m = emits[3][2]
+    assert_equal 3, m['value']
+
+    # 'agent' => 'DoCoMo/1.0/N505i/c20/TB/W24H12'
+    m = emits[4][2]
+    assert_equal 4, m['value']
+
+    # 'agent' => 'Mozilla/5.0 (PlayStation Vita 1.51) AppleWebKit/531.22.8 (KHTML, like Gecko) Silk/3.2'
+    m = emits[5][2]
+    assert_equal 5, m['value']
+  end
+end

metadata

@@ -0,0 +1,136 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-woothee
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- TAGOMORI Satoshi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: woothee
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.2.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.2.4
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: woothee
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.2.4
+description: ! 'parsing by Project Woothee. See https://github.com/tagomoris/woothee '
+email:
+- tagomoris@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-woothee.gemspec
+- lib/fluent/plugin/out_woothee.rb
+- test/helper.rb
+- test/plugin/test_out_woothee.rb
+homepage: https://github.com/tagomoris/fluent-plugin-woothee
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.21
+signing_key: 
+specification_version: 3
+summary: Fluentd plugin to parse UserAgent strings
+test_files:
+- test/helper.rb
+- test/plugin/test_out_woothee.rb