fluent-plugin-geoip 0.5.1 → 0.6.0

data/.travis.yml

@@ -1,9 +1,20 @@
+sudo: required
 language: ruby
 
 rvm:
+  - 2.3.0
+  - 2.2
   - 2.1
   - 2.0.0
   - 1.9.3
 
 before_install:
+  - sudo apt-get update
   - sudo apt-get install libgeoip-dev
+  - gem update bundler
+
+gemfile:
+  - Gemfile
+  - gemfiles/fluentd_v0.10.gemfile
+  - gemfiles/fluentd_v0.12.gemfile
+

data/Appraisals

@@ -0,0 +1,8 @@
+appraise "fluentd v0.10" do
+  gem "fluentd", "~> 0.10.46"
+end
+
+appraise "fluentd v0.12" do
+  gem "fluentd", "~> 0.12.0"
+end
+

data/fluent-plugin-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-geoip"
-  spec.version       = "0.5.1"
+  spec.version       = "0.6.0"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}
@@ -18,12 +18,11 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
-
-  if defined?(RUBY_VERSION) && RUBY_VERSION > '2.2'
-    spec.add_development_dependency "test-unit", '~> 3'
-  end
+  spec.add_development_dependency "appraisal"
+  spec.add_development_dependency "test-unit", ">= 3.1.0"
 
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
   spec.add_runtime_dependency "geoip-c"
 end
+

data/gemfiles/fluentd_v0.10.gemfile

@@ -0,0 +1,8 @@
+# This file was generated by Appraisal
+
+source "http://rubygems.org"
+
+gem "fluentd", "~> 0.10.46"
+
+gemspec :path => "../"
+

data/gemfiles/fluentd_v0.12.gemfile

@@ -0,0 +1,8 @@
+# This file was generated by Appraisal
+
+source "http://rubygems.org"
+
+gem "fluentd", "~> 0.12.0"
+
+gemspec :path => "../"
+

data/lib/fluent/plugin/filter_geoip.rb

@@ -0,0 +1,32 @@
+module Fluent
+  class GeoipFilter < Filter
+    Plugin.register_filter('geoip', self)
+
+    config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
+    config_param :geoip_lookup_key, :string, :default => 'host'
+    config_param :skip_adding_null_record, :bool, :default => false
+
+    config_set_default :include_tag_key, false
+
+    config_param :hostname_command, :string, :default => 'hostname'
+
+    config_param :flush_interval, :time, :default => 0
+    config_param :log_level, :string, :default => 'warn'
+
+
+    def initialize
+      require 'fluent/plugin/geoip'
+
+      super
+    end
+
+    def configure(conf)
+      super
+      @geoip = Fluent::GeoIP.new(self, conf)
+    end
+
+    def filter(tag, time, record)
+      @geoip.add_geoip_field(record)
+    end
+  end
+end

data/lib/fluent/plugin/geoip.rb

@@ -0,0 +1,150 @@
+require 'geoip'
+require 'yajl'
+
+module Fluent
+  class GeoIP
+    REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<geoip_key>-?[^\[]+)\[['"](?<record_key>-?[^'"]+)['"]\]\}$/
+    REGEXP_PLACEHOLDER_SCAN = /['"]?(\$\{[^\}]+?\})['"]?/
+
+    GEOIP_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
+
+    attr_reader :log
+
+    def initialize(plugin, conf)
+      @map = {}
+      plugin.geoip_lookup_key = plugin.geoip_lookup_key.split(/\s*,\s*/)
+      @geoip_lookup_key = plugin.geoip_lookup_key
+      @skip_adding_null_record = plugin.skip_adding_null_record
+      @log = plugin.log
+
+      # enable_key_* format (legacy format)
+      conf.keys.select{|k| k =~ /^enable_key_/}.each do |key|
+        geoip_key = key.sub('enable_key_','')
+        raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
+        @geoip_lookup_key.zip(conf[key].split(/\s*,\s*/)).each do |lookup_field,record_key|
+          if record_key.nil?
+            raise Fluent::ConfigError, "geoip: missing value found at '#{key} #{lookup_field}'"
+          end
+          @map.store(record_key, "${#{geoip_key}['#{lookup_field}']}")
+        end
+      end
+      if conf.keys.select{|k| k =~ /^enable_key_/}.size > 0
+        log.warn "geoip: 'enable_key_*' config format is obsoleted. use <record></record> directive for now."
+        log.warn "geoip: for further details referable to https://github.com/y-ken/fluent-plugin-geoip"
+      end
+
+      # <record></record> directive
+      conf.elements.select { |element| element.name == 'record' }.each { |element|
+        element.each_pair { |k, v|
+          element.has_key?(k) # to suppress unread configuration warning
+          v = v[1..v.size-2] if quoted_value?(v)
+          @map[k] = v
+          validate_json = Proc.new {
+            begin
+              dummy_text = Yajl::Encoder.encode('dummy_text')
+              Yajl::Parser.parse(v.gsub(REGEXP_PLACEHOLDER_SCAN, dummy_text))
+            rescue Yajl::ParseError => e
+              raise Fluent::ConfigError, "geoip: failed to parse '#{v}' as json."
+            end
+          }
+          validate_json.call if json?(v.tr('\'"\\', ''))
+        }
+      }
+      @placeholder_keys = @map.values.join.scan(REGEXP_PLACEHOLDER_SCAN).map{ |placeholder| placeholder[0] }.uniq
+      @placeholder_keys.each do |key|
+        geoip_key = key.match(REGEXP_PLACEHOLDER_SINGLE)[:geoip_key]
+        raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
+      end
+
+      if plugin.is_a?(Fluent::Output)
+        @placeholder_expander = PlaceholderExpander.new
+        unless have_tag_option?(plugin)
+          raise Fluent::ConfigError, "geoip: required at least one option of 'tag', 'remove_tag_prefix', 'remove_tag_suffix', 'add_tag_prefix', 'add_tag_suffix'."
+        end
+      end
+
+      @geoip = ::GeoIP::City.new(plugin.geoip_database, :memory, false)
+    end
+
+    def add_geoip_field(record)
+      placeholder = create_placeholder(geolocate(get_address(record)))
+      return record if @skip_adding_null_record && placeholder.values.first.nil?
+      @map.each do |record_key, value|
+        if value.match(REGEXP_PLACEHOLDER_SINGLE)
+          rewrited = placeholder[value]
+        elsif json?(value)
+          rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN) {|match|
+            match = match[1..match.size-2] if quoted_value?(match)
+            Yajl::Encoder.encode(placeholder[match])
+          }
+          rewrited = parse_json(rewrited)
+        else
+          rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN, placeholder)
+        end
+        record.store(record_key, rewrited)
+      end
+      return record
+    end
+
+    private
+
+    def have_tag_option?(plugin)
+      plugin.tag ||
+        plugin.remove_tag_prefix || plugin.remove_tag_suffix ||
+        plugin.add_tag_prefix || plugin.add_tag_suffix
+    end
+
+    def json?(text)
+      text.match(/^\[.+\]$/) || text.match(/^\{.+\}$/)
+    end
+
+    def quoted_value?(text)
+      # to improbe compatibility with fluentd v1-config
+      trim_quote = text[1..text.size-2]
+      text.match(/(^'.+'$|^".+"$)/)
+    end
+
+    def parse_json(message)
+      begin
+        return Yajl::Parser.parse(message)
+      rescue Yajl::ParseError => e
+        log.info "geoip: failed to parse '#{message}' as json.", :error_class => e.class, :error => e.message
+        return nil
+      end
+    end
+
+    def get_address(record)
+      address = {}
+      @geoip_lookup_key.each do |field|
+        address.store(field, record[field]); next if not record[field].nil?
+        key = field.split('.')
+        obj = record
+        key.each {|k|
+          break obj = nil if not obj.has_key?(k)
+          obj = obj[k]
+        }
+        address.store(field, obj)
+      end
+      return address
+    end
+
+    def geolocate(addresses)
+      geodata = {}
+      addresses.each do |field, ip|
+        geo = ip.nil? ? nil : @geoip.look_up(ip)
+        geodata.store(field, geo)
+      end
+      return geodata
+    end
+
+    def create_placeholder(geodata)
+      placeholder = {}
+      @placeholder_keys.each do |placeholder_key|
+        position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
+        next if position.nil? or geodata[position[:record_key]].nil?
+        placeholder.store(placeholder_key, geodata[position[:record_key]][position[:geoip_key].to_sym])
+      end
+      return placeholder
+    end
+  end
+end

data/lib/fluent/plugin/out_geoip.rb

@@ -3,11 +3,6 @@ require 'fluent/mixin/rewrite_tag_name'
 class Fluent::GeoipOutput < Fluent::BufferedOutput
   Fluent::Plugin.register_output('geoip', self)
 
-  REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<geoip_key>-?[^\[]+)\[['"](?<record_key>-?[^'"]+)['"]\]\}$/
-  REGEXP_PLACEHOLDER_SCAN = /['"]?(\$\{[^\}]+?\})['"]?/
-
-  GEOIP_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
-
   config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
   config_param :geoip_lookup_key, :string, :default => 'host'
   config_param :tag, :string, :default => nil
@@ -28,64 +23,23 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
     define_method("log") { $log }
   end
 
+  # To support Fluentd v0.10.57 or earlier
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
   def initialize
-    require 'geoip'
-    require 'yajl'
+    require 'fluent/plugin/geoip'
 
     super
   end
 
   def configure(conf)
     super
-
-    @map = {}
-    @geoip_lookup_key = @geoip_lookup_key.split(/\s*,\s*/)
-
-    # enable_key_* format (legacy format)
-    conf.keys.select{|k| k =~ /^enable_key_/}.each do |key|
-      geoip_key = key.sub('enable_key_','')
-      raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
-      @geoip_lookup_key.zip(conf[key].split(/\s*,\s*/)).each do |lookup_field,record_key|
-        if record_key.nil?
-          raise Fluent::ConfigError, "geoip: missing value found at '#{key} #{lookup_field}'"
-        end
-        @map.store(record_key, "${#{geoip_key}['#{lookup_field}']}")
-      end
-    end
-    if conf.keys.select{|k| k =~ /^enable_key_/}.size > 0
-      log.warn "geoip: 'enable_key_*' config format is obsoleted. use <record></record> directive for now."
-      log.warn "geoip: for further details referable to https://github.com/y-ken/fluent-plugin-geoip"
-    end
-
-    # <record></record> directive
-    conf.elements.select { |element| element.name == 'record' }.each { |element|
-      element.each_pair { |k, v|
-        element.has_key?(k) # to suppress unread configuration warning
-        v = v[1..v.size-2] if quoted_value?(v)
-        @map[k] = v
-        validate_json = Proc.new { 
-          begin
-            dummy_text = Yajl::Encoder.encode('dummy_text')
-            Yajl::Parser.parse(v.gsub(REGEXP_PLACEHOLDER_SCAN, dummy_text))
-          rescue Yajl::ParseError => e
-            raise Fluent::ConfigError, "geoip: failed to parse '#{v}' as json."
-          end
-        }
-        validate_json.call if json?(v.tr('\'"\\', ''))
-      }
-    }
-    @placeholder_keys = @map.values.join.scan(REGEXP_PLACEHOLDER_SCAN).map{ |placeholder| placeholder[0] }.uniq
-    @placeholder_keys.each do |key|
-      geoip_key = key.match(REGEXP_PLACEHOLDER_SINGLE)[:geoip_key]
-      raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
-    end
-    @placeholder_expander = PlaceholderExpander.new
-
-    if ( !@tag && !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
-      raise Fluent::ConfigError, "geoip: required at least one option of 'tag', 'remove_tag_prefix', 'remove_tag_suffix', 'add_tag_prefix', 'add_tag_suffix'."
+    Fluent::GeoIP.class_eval do
+      include Fluent::Mixin::RewriteTagName
     end
-
-    @geoip = GeoIP::City.new(@geoip_database, :memory, false)
+    @geoip = Fluent::GeoIP.new(self, conf)
   end
 
   def start
@@ -102,82 +56,7 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
 
   def write(chunk)
     chunk.msgpack_each do |tag, time, record|
-      Fluent::Engine.emit(tag, time, add_geoip_field(record))
-    end
-  end
-
-  private
-
-  def json?(text)
-    text.match(/^\[.+\]$/) || text.match(/^\{.+\}$/)
-  end
-
-  def quoted_value?(text)
-    # to improbe compatibility with fluentd v1-config
-    trim_quote = text[1..text.size-2]
-    text.match(/(^'.+'$|^".+"$)/)
-  end
-
-  def add_geoip_field(record)
-    placeholder = create_placeholder(geolocate(get_address(record)))
-    return record if @skip_adding_null_record && placeholder.values.first.nil?
-    @map.each do |record_key, value|
-      if value.match(REGEXP_PLACEHOLDER_SINGLE)
-        rewrited = placeholder[value]
-      elsif json?(value)
-        rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN) {|match|
-          match = match[1..match.size-2] if quoted_value?(match)
-          Yajl::Encoder.encode(placeholder[match])
-        }
-        rewrited = parse_json(rewrited)
-      else
-        rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN, placeholder)
-      end
-      record.store(record_key, rewrited)
-    end
-    return record
-  end
-
-  def parse_json(message)
-    begin
-      return Yajl::Parser.parse(message)
-    rescue Yajl::ParseError => e
-      log.info "geoip: failed to parse '#{message}' as json.", :error_class => e.class, :error => e.message
-      return nil
-    end
-  end
-
-  def get_address(record)
-    address = {}
-    @geoip_lookup_key.each do |field|
-      address.store(field, record[field]); next if not record[field].nil?
-      key = field.split('.')
-      obj = record
-      key.each {|k|
-        break obj = nil if not obj.has_key?(k)
-        obj = obj[k]
-      }
-      address.store(field, obj)
-    end
-    return address
-  end
-
-  def geolocate(addresses)
-    geodata = {}
-    addresses.each do |field, ip|
-      geo = ip.nil? ? nil : @geoip.look_up(ip)
-      geodata.store(field, geo)
-    end
-    return geodata
-  end
-
-  def create_placeholder(geodata)
-    placeholder = {}
-    @placeholder_keys.each do |placeholder_key|
-      position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
-      next if position.nil? or geodata[position[:record_key]].nil?
-      placeholder.store(placeholder_key, geodata[position[:record_key]][position[:geoip_key].to_sym])
+      router.emit(tag, time, @geoip.add_geoip_field(record))
     end
-    return placeholder
   end
 end

data/test/helper.rb

@@ -23,6 +23,9 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/plugin/out_geoip'
+if Fluent.const_defined?(:Filter)
+  require 'fluent/plugin/filter_geoip'
+end
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_filter_geoip.rb

@@ -0,0 +1,436 @@
+require 'helper'
+
+class GeoipFilterTest < Test::Unit::TestCase
+  def setup
+    omit_unless(Fluent.const_defined?(:Filter))
+    Fluent::Test.setup
+    @time = Fluent::Engine.now
+  end
+
+  CONFIG = %[
+    geoip_lookup_key  host
+    enable_key_city   geoip_city
+    remove_tag_prefix input.
+    tag               geoip.${tag}
+  ]
+
+  def create_driver(conf=CONFIG, tag='test', use_v1=false)
+    Fluent::Test::FilterTestDriver.new(Fluent::GeoipFilter, tag).configure(conf, use_v1)
+  end
+
+  def filter(config, messages, use_v1=false)
+    d = create_driver(config, 'test', use_v1)
+    d.run {
+      messages.each {|message|
+        d.filter(message, @time)
+      }
+    }
+    filtered = d.filtered_as_array
+    filtered.map {|m| m[2] }
+  end
+
+  def test_configure
+    assert_nothing_raised {
+      create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      create_driver('enable_key_cities')
+    }
+    d = create_driver %[
+      enable_key_city   geoip_city
+      remove_tag_prefix input.
+      tag               geoip.${tag}
+    ]
+    assert_equal 'geoip_city', d.instance.config['enable_key_city']
+
+    # multiple key config
+    d = create_driver %[
+      geoip_lookup_key  from.ip, to.ip
+      enable_key_city   from_city, to_city
+      remove_tag_prefix input.
+      tag               geoip.${tag}
+    ]
+    assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
+
+    # multiple key config (bad configure)
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        geoip_lookup_key  from.ip, to.ip
+        enable_key_city   from_city
+        enable_key_region from_region
+        remove_tag_prefix input.
+        tag               geoip.${tag}
+      ]
+    }
+
+    # invalid json structure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        geoip_lookup_key  host
+        <record>
+          invalid_json    {"foo" => 123}
+        </record>
+        remove_tag_prefix input.
+        tag               geoip.${tag}
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        geoip_lookup_key  host
+        <record>
+          invalid_json    {"foo" : string, "bar" : 123}
+        </record>
+        remove_tag_prefix input.
+        tag               geoip.${tag}
+      ]
+    }
+  end
+
+  def test_filter
+    messages = [
+      {'host' => '66.102.3.80', 'message' => 'valid ip'},
+      {'message' => 'missing field'},
+    ]
+    expected = [
+      {'host' => '66.102.3.80', 'message' => 'valid ip', 'geoip_city' => 'Mountain View'},
+      {'message' => 'missing field', 'geoip_city' => nil},
+    ]
+    filtered = filter(CONFIG, messages)
+    assert_equal(expected, filtered)
+  end
+
+  def test_filter_with_dot_key
+    config = %[
+      geoip_lookup_key  ip.origin, ip.dest
+      <record>
+        origin_country  ${country_code['ip.origin']}
+        dest_country    ${country_code['ip.dest']}
+      </record>
+    ]
+    messages = [
+      {'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8'}
+    ]
+    expected = [
+      {'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8',
+       'origin_country' => 'US', 'dest_country' => 'US'}
+    ]
+    filtered = filter(config, messages)
+    assert_equal(expected, filtered)
+  end
+
+  def test_filter_nested_attr
+    config = %[
+      geoip_lookup_key  host.ip
+      enable_key_city   geoip_city
+    ]
+    messages = [
+      {'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip'},
+      {'message' => 'missing field'}
+    ]
+    expected = [
+      {'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip', 'geoip_city' => 'Mountain View'},
+      {'message' => 'missing field', 'geoip_city' => nil}
+    ]
+    filtered = filter(config, messages)
+    assert_equal(expected, filtered)
+  end
+
+  def test_filter_with_unknown_address
+    config = %[
+      geoip_lookup_key  host
+      <record>
+        geoip_city      ${city['host']}
+        geopoint        [${longitude['host']}, ${latitude['host']}]
+      </record>
+      skip_adding_null_record false
+    ]
+    # 203.0.113.1 is a test address described in RFC5737
+    messages = [
+      {'host' => '203.0.113.1', 'message' => 'invalid ip'},
+      {'host' => '0', 'message' => 'invalid ip'}
+    ]
+    expected = [
+      {'host' => '203.0.113.1', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]},
+      {'host' => '0', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]}
+    ]
+    filtered = filter(config, messages)
+    assert_equal(expected, filtered)
+  end
+
+  def test_filter_with_skip_unknown_address
+    config = %[
+      geoip_lookup_key  host
+      <record>
+        geoip_city      ${city['host']}
+        geopoint        [${longitude['host']}, ${latitude['host']}]
+      </record>
+      skip_adding_null_record true
+    ]
+    # 203.0.113.1 is a test address described in RFC5737
+    messages = [
+      {'host' => '203.0.113.1', 'message' => 'invalid ip'},
+      {'host' => '0', 'message' => 'invalid ip'},
+      {'host' => '8.8.8.8', 'message' => 'google public dns'}
+    ]
+    expected = [
+      {'host' => '203.0.113.1', 'message' => 'invalid ip'},
+      {'host' => '0', 'message' => 'invalid ip'},
+      {'host' => '8.8.8.8', 'message' => 'google public dns',
+       'geoip_city' => 'Mountain View', 'geopoint' => [-122.08380126953125, 37.38600158691406]}
+    ]
+    filtered = filter(config, messages)
+    assert_equal(expected, filtered)
+  end
+
+  def test_filter_multiple_key
+    config = %[
+      geoip_lookup_key  from.ip, to.ip
+      enable_key_city   from_city, to_city
+    ]
+    messages = [
+      {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
+      {'message' => 'missing field'}
+    ]
+    expected = [
+      {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'},
+       'from_city' => 'Mountain View', 'to_city' => 'Tokorozawa'},
+      {'message' => 'missing field', 'from_city' => nil, 'to_city' => nil}
+    ]
+    filtered = filter(config, messages)
+    assert_equal(expected, filtered)
+  end
+
+  def test_filter_multiple_key_multiple_record
+    config = %[
+      geoip_lookup_key  from.ip, to.ip
+      enable_key_city   from_city, to_city
+      enable_key_country_name from_country, to_country
+    ]
+    messages = [
+      {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
+      {'from' => {'ip' => '66.102.3.80'}},
+      {'message' => 'missing field'}
+    ]
+    expected = [
+      {
+        'from' => {'ip' => '66.102.3.80'},
+        'to' => {'ip' => '125.54.15.42'},
+        'from_city' => 'Mountain View',
+        'from_country' => 'United States',
+        'to_city' => 'Tokorozawa',
+        'to_country' => 'Japan'
+      },
+      {
+        'from' => {'ip' => '66.102.3.80'},
+        'from_city' => 'Mountain View',
+        'from_country' => 'United States',
+        'to_city' => nil,
+        'to_country' => nil
+      },
+      {
+        'message' => 'missing field',
+        'from_city' => nil,
+        'from_country' => nil,
+        'to_city' => nil,
+        'to_country' => nil
+      }
+    ]
+    filtered = filter(config, messages)
+    assert_equal(expected, filtered)
+  end
+
+  def test_filter_record_directive
+    config = %[
+      geoip_lookup_key  from.ip
+      <record>
+        from_city       ${city['from.ip']}
+        from_country    ${country_name['from.ip']}
+        latitude        ${latitude['from.ip']}
+        longitude       ${longitude['from.ip']}
+        float_concat    ${latitude['from.ip']},${longitude['from.ip']}
+        float_array     [${longitude['from.ip']}, ${latitude['from.ip']}]
+        float_nest      { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
+        string_concat   ${city['from.ip']},${country_name['from.ip']}
+        string_array    [${city['from.ip']}, ${country_name['from.ip']}]
+        string_nest     { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
+        unknown_city    ${city['unknown_key']}
+        undefined       ${city['undefined']}
+        broken_array1   [${longitude['from.ip']}, ${latitude['undefined']}]
+        broken_array2   [${longitude['undefined']}, ${latitude['undefined']}]
+      </record>
+    ]
+    messages = [
+      { 'from' => {'ip' => '66.102.3.80'} },
+      { 'message' => 'missing field' },
+    ]
+    expected = [
+      {
+        'from' => {'ip' => '66.102.3.80'},
+        'from_city' => 'Mountain View',
+        'from_country' => 'United States',
+        'latitude' => 37.4192008972168,
+        'longitude' => -122.05740356445312,
+        'float_concat' => '37.4192008972168,-122.05740356445312',
+        'float_array' => [-122.05740356445312, 37.4192008972168],
+        'float_nest' => { 'lat' => 37.4192008972168, 'lon' => -122.05740356445312 },
+        'string_concat' => 'Mountain View,United States',
+        'string_array' => ["Mountain View", "United States"],
+        'string_nest' => {"city" => "Mountain View", "country_name" => "United States"},
+        'unknown_city' => nil,
+        'undefined' => nil,
+        'broken_array1' => [-122.05740356445312, nil],
+        'broken_array2' => [nil, nil]
+      },
+      {
+        'message' => 'missing field',
+        'from_city' => nil,
+        'from_country' => nil,
+        'latitude' => nil,
+        'longitude' => nil,
+        'float_concat' => ',',
+        'float_array' => [nil, nil],
+        'float_nest' => { 'lat' => nil, 'lon' => nil },
+        'string_concat' => ',',
+        'string_array' => [nil, nil],
+        'string_nest' => { "city" => nil, "country_name" => nil },
+        'unknown_city' => nil,
+        'undefined' => nil,
+        'broken_array1' => [nil, nil],
+        'broken_array2' => [nil, nil]
+      },
+    ]
+    filtered = filter(config, messages)
+    # test-unit cannot calculate diff between large Array
+    assert_equal(expected[0], filtered[0])
+    assert_equal(expected[1], filtered[1])
+  end
+
+  def test_filter_record_directive_multiple_record
+    config = %[
+      geoip_lookup_key  from.ip, to.ip
+      <record>
+        from_city       ${city['from.ip']}
+        to_city         ${city['to.ip']}
+        from_country    ${country_name['from.ip']}
+        to_country      ${country_name['to.ip']}
+        string_array    [${country_name['from.ip']}, ${country_name['to.ip']}]
+      </record>
+    ]
+    messages = [
+      {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
+      {'message' => 'missing field'}
+    ]
+    expected = [
+      {
+        'from' => { 'ip' => '66.102.3.80' },
+        'to' => { 'ip' => '125.54.15.42' },
+        'from_city' => 'Mountain View',
+        'from_country' => 'United States',
+        'to_city' => 'Tokorozawa',
+        'to_country' => 'Japan',
+        'string_array' => ['United States', 'Japan']
+      },
+      {
+        'message' => 'missing field',
+        'from_city' => nil,
+        'from_country' => nil,
+        'to_city' => nil,
+        'to_country' => nil,
+        'string_array' => [nil, nil]
+      }
+    ]
+    filtered = filter(config, messages)
+    assert_equal(expected, filtered)
+  end
+
+  CONFIG_QUOTED_RECORD = %[
+    geoip_lookup_key  host
+    <record>
+      location_properties  '{ "country_code" : "${country_code["host"]}", "lat": ${latitude["host"]}, "lon": ${longitude["host"]} }'
+      location_string      ${latitude['host']},${longitude['host']}
+      location_string2     ${country_code["host"]}
+      location_array       "[${longitude['host']},${latitude['host']}]"
+      location_array2      '[${longitude["host"]},${latitude["host"]}]'
+      peculiar_pattern     '[GEOIP] message => {"lat":${latitude["host"]}, "lon":${longitude["host"]}}'
+    </record>
+    remove_tag_prefix input.
+    tag               geoip.${tag}
+  ]
+
+  def test_filter_quoted_record
+    messages = [
+      {'host' => '66.102.3.80', 'message' => 'valid ip'}
+    ]
+    expected = [
+      {
+        'host' => '66.102.3.80', 'message' => 'valid ip',
+        'location_properties' => {
+          'country_code' => 'US',
+          'lat' => 37.4192008972168,
+          'lon' => -122.05740356445312
+        },
+        'location_string' => '37.4192008972168,-122.05740356445312',
+        'location_string2' => 'US',
+        'location_array' => [-122.05740356445312, 37.4192008972168],
+        'location_array2' => [-122.05740356445312, 37.4192008972168],
+        'peculiar_pattern' => '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}'
+      }
+    ]
+    filtered = filter(CONFIG_QUOTED_RECORD, messages)
+    assert_equal(expected, filtered)
+  end
+
+  def test_filter_v1_config_compatibility
+    messages = [
+      {'host' => '66.102.3.80', 'message' => 'valid ip'}
+    ]
+    expected = [
+      {
+        'host' => '66.102.3.80', 'message' => 'valid ip',
+        'location_properties' => {
+          'country_code' => 'US',
+          'lat' => 37.4192008972168,
+          'lon' => -122.05740356445312
+        },
+        'location_string' => '37.4192008972168,-122.05740356445312',
+        'location_string2' => 'US',
+        'location_array' => [-122.05740356445312, 37.4192008972168],
+        'location_array2' => [-122.05740356445312, 37.4192008972168],
+        'peculiar_pattern' => '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}'
+      }
+    ]
+    filtered = filter(CONFIG_QUOTED_RECORD, messages, true)
+    assert_equal(expected, filtered)
+  end
+
+  def test_filter_multiline_v1_config
+    config = %[
+      geoip_lookup_key  host
+      <record>
+        location_properties  {
+          "city": "${city['host']}",
+          "country_code": "${country_code['host']}",
+          "latitude": "${latitude['host']}",
+          "longitude": "${longitude['host']}"
+        }
+      </record>
+    ]
+    messages = [
+      { 'host' => '66.102.3.80', 'message' => 'valid ip' }
+    ]
+    expected = [
+      {
+        'host' => '66.102.3.80', 'message' => 'valid ip',
+        "location_properties" => {
+          "city" => "Mountain View",
+          "country_code" => "US",
+          "latitude" => 37.4192008972168,
+          "longitude" => -122.05740356445312
+        }
+      }
+    ]
+    filtered = filter(config, messages, true)
+    assert_equal(expected, filtered)
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-geoip
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.6.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-28 00:00:00.000000000 Z
+date: 2016-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -43,6 +43,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +132,20 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .travis.yml
+- Appraisals
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - data/GeoLiteCity.dat
 - fluent-plugin-geoip.gemspec
+- gemfiles/fluentd_v0.10.gemfile
+- gemfiles/fluentd_v0.12.gemfile
+- lib/fluent/plugin/filter_geoip.rb
+- lib/fluent/plugin/geoip.rb
 - lib/fluent/plugin/out_geoip.rb
 - test/helper.rb
+- test/plugin/test_filter_geoip.rb
 - test/plugin/test_out_geoip.rb
 homepage: https://github.com/y-ken/fluent-plugin-geoip
 licenses:
@@ -137,4 +175,5 @@ summary: Fluentd Output plugin to add information about geographical location of
   addresses with Maxmind GeoIP databases.
 test_files:
 - test/helper.rb
+- test/plugin/test_filter_geoip.rb
 - test/plugin/test_out_geoip.rb