fluent-plugin-geoip 0.0.6 → 0.1.0

data/README.md

@@ -2,8 +2,8 @@
 
 Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.
 
-fluent-plugin-geoip has bundled cost-free [GeoLite City database](http://dev.maxmind.com/geoip/legacy/geolite/) by default.  
-Also you can use purchased [GeoIP City database](http://www.maxmind.com/en/city) ([lang:ja](http://www.maxmind.com/ja/city)) which costs starting from $50.  
+fluent-plugin-geoip has bundled cost-free [GeoLite City database](http://dev.maxmind.com/geoip/legacy/geolite/) by default.<br />
+Also you can use purchased [GeoIP City database](http://www.maxmind.com/en/city) ([lang:ja](http://www.maxmind.com/ja/city)) which costs starting from $50.
 
 The accuracy details for GeoLite City (free) and GeoIP City (purchased) has described at the page below.
 
@@ -40,31 +40,35 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
 <match access.apache>
   type geoip
 
-  # Specify geoip lookup field (default: host)
+  # Specify one or more geoip lookup field which has ip address (default: host)
   # in the case of accessing nested value, delimit keys by dot like 'host.ip'.
-  geoip_lookup_key         host
+  geoip_lookup_key  host
 
   # Specify geoip database (using bundled GeoLiteCity databse by default)
-  geoip_database           'data/GeoLiteCity.dat'
-
-  # Set adding field of geolocate results (more than one settings are required.)
-  enable_key_city          geoip_city
-  enable_key_latitude      geoip_lat
-  enable_key_longitude     geoip_lon
-  enable_key_country_code3 geoip_country3
-  enable_key_country_code  geoip_country
-  enable_key_country_name  geoip_country_name
-  enable_key_dma_code      geoip_dma
-  enable_key_area_code     geoip_area
-  enable_key_region        geoip_region
-
-  # Setting for tag
-  remove_tag_prefix        access.
-  add_tag_prefix           geoip.
-  include_tag_key          false
-
-  # Buffering time (default: 60s)
-  flush_interval           1s
+  geoip_database    'data/GeoLiteCity.dat'
+
+  # Set adding field with placeholder (more than one settings are required.)
+  <record>
+    city            ${city['host']}
+    latitude        ${latitude['host']}
+    longitude       ${longitude['host']}
+    country_code3   ${country_code3['host']}
+    country         ${country['host']}
+    country_name    ${country_name['host']}
+    dma             ${dma['host']}
+    area            ${area['host']}
+    region          ${region['host']}
+  </record>
+
+  # Settings for tag
+  remove_tag_prefix access.
+  tag               geoip.${tag}
+
+  # Set log_level for fluentd-v0.10.43 or earlier (default: warn)
+  log_level         info
+
+  # Set buffering time (default: 0s)
+  flush_interval    1s
 </match>
 ```
 
@@ -73,20 +77,39 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
 ```xml
 <match access.apache>
   type geoip
+  geoip_lookup_key  user1_host, user2_host
+  <record>
+    user1_city      ${city['user1_host']}
+    user2_city      ${city['user2_host']}
+  </record>
+  remove_tag_prefix access.
+  tag               geoip.${tag}
+</match>
+```
 
-  # Set ip address key to geolocate
-  geoip_lookup_key        user1_host, user2_host
-
-  # Set adding field of geolocate results
-  enable_key_city         user1_city, user2_city
-  enable_key_country_name user1_country, user2_country
+#### Advanced config samples
 
-  # Setting for tag
-  remove_tag_prefix       access.
-  add_tag_prefix          geoip.
+It is a sample to get friendly geo point recdords for elasticsearch with Yajl (JSON) parser.
 
-  # Buffering time
-  flush_interval          1s
+```
+<match input.access>
+  type                   geoip
+  geoip_lookup_key       host
+  <record>
+    # lat lon as properties
+    # ex. {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
+    location_properties  { "lat":${latitude['host']}, "lon":${longitude['host']}}
+  
+    # lat lon as string
+    # ex. "37.4192008972168,-122.05740356445312"
+    location_string      ${latitude['host']},${longitude['host']}
+    
+    # lat lon as array (it is useful for Kibana's bettermap.)
+    # ex. [-122.05740356445312, 37.4192008972168]
+    location_array       [${longitude['host']},${latitude['host']}]
+  </record>
+  remove_tag_prefix      access.
+  tag                    geoip.${tag}
 </match>
 ```
 
@@ -105,14 +128,15 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
     type stdout
   </store>
   <store>
-    type geoip
-    geoip_lookup_key     host
-    enable_key_city      city
-    enable_key_latitude  lat
-    enable_key_longitude lon
-    remove_tag_prefix    test.
-    add_tag_prefix       debug.
-    flush_interval       5s
+    type    geoip
+    geoip_lookup_key  host
+    <record>
+      city  ${city['host']}
+      lat   ${latitude['host']}
+      lon   ${longitude['host']}
+    </record>
+    remove_tag_prefix test.
+    tag     debug.${tag}
   </store>
 </match>
 
@@ -133,30 +157,69 @@ $ tail /var/log/td-agent/td-agent.log
 2013-08-04 16:21:32 +0900 debug.geoip: {"host":"66.102.9.80","message":"test","city":"Mountain View","lat":37.4192008972168,"lon":-122.05740356445312}
 ```
 
-For more details of geoip data format is described at the page below in section `GeoIP City Edition CSV Database Fields`.  
+For more details of geoip data format is described at the page below in section `GeoIP City Edition CSV Database Fields`.<br />
 http://dev.maxmind.com/geoip/legacy/csv/
 
+## Placeholders
+
+Provides these placeholders for adding field of geolocate results.
+
+* ${city}
+* ${latitude}
+* ${longitude}
+* ${country_code3}
+* ${country_code}
+* ${country_name}
+* ${dma_code}
+* ${area_code}
+* ${region}
+
+## Parameters
+
+* `include_tag_key` (default: false)
+* `tag_key`
+
+Add original tag name into filtered record using SetTagKeyMixin.<br />
+Further details are written at http://docs.fluentd.org/articles/in_exec
+
+* `remove_tag_prefix`
+* `remove_tag_suffix`
+* `add_tag_prefix`
+* `add_tag_suffix`
+
+Set one or more option are required unless using `tag` option for editing tag name. (HandleTagNameMixin feature)
+
+* `tag`
+
+On using this option with tag placeholder like `tag geoip.${tag}` (test code is available at [test_out_geoip.rb](https://github.com/y-ken/fluent-plugin-geoip/blob/master/test/plugin/test_out_geoip.rb)), it will be overwrite after these options affected. which are remove_tag_prefix, remove_tag_suffix, add_tag_prefix and add_tag_suffix.
+
+* `flush_interval` (default: 0 sec)
+
+Set buffering time to execute bulk lookup geoip.
+
 ## Articles
 
-* [IPアドレスを元に位置情報をリアルタイムに付与する fluent-plugin-geoip v0.0.1をリリースしました #fluentd - Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-has-released)  
+* [IPアドレスを元に位置情報をリアルタイムに付与する fluent-plugin-geoip v0.0.1をリリースしました #fluentd - Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-has-released)<br />
 http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-has-released
 
-* [初の安定版 fluent-plugin-geoip v0.0.3 をリリースしました #fluentd- Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.3)  
+* [初の安定版 fluent-plugin-geoip v0.0.3 をリリースしました #fluentd- Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.3)<br />
 http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.3
 
-* [fluent-plugin-geoip v0.0.4 をリリースしました。ElasticSearch＋Kibanaの世界地図に位置情報をプロットするために必要なFluentdの設定サンプルも紹介します- Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.4)  
+* [fluent-plugin-geoip v0.0.4 をリリースしました。ElasticSearch＋Kibanaの世界地図に位置情報をプロットするために必要なFluentdの設定サンプルも紹介します- Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.4)<br />
 http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.4
 
-* [Released GeoIP plugin to work together with ElasticSearch + Kibana v3](https://groups.google.com/d/topic/fluentd/OVIcH_SKBwM/discussion)  
+* [Released GeoIP plugin to work together with ElasticSearch + Kibana v3](https://groups.google.com/d/topic/fluentd/OVIcH_SKBwM/discussion)<br />
 https://groups.google.com/d/topic/fluentd/OVIcH_SKBwM/discussion
 
-* [Fluentd、Amazon RedshiftとTableauを用いたカジュアルなデータ可視化 | SmartNews開発者ブログ](http://developer.smartnews.be/blog/2013/10/03/easy-data-analysis-using-fluentd-redshift-and-tableau/)  
+* [Fluentd、Amazon RedshiftとTableauを用いたカジュアルなデータ可視化 | SmartNews開発者ブログ](http://developer.smartnews.be/blog/2013/10/03/easy-data-analysis-using-fluentd-redshift-and-tableau/)<br />
 http://developer.smartnews.be/blog/2013/10/03/easy-data-analysis-using-fluentd-redshift-and-tableau/
 
 ## TODO
 
 Pull requests are very welcome!!
 
+* support [GeoIP2](http://dev.maxmind.com/geoip/geoip2/whats-new-in-geoip2/)
+
 ## Contributing
 
 1. Fork it
@@ -167,7 +230,7 @@ Pull requests are very welcome!!
 
 ## Copyright
 
-Copyright (c) 2013- Kentaro Yoshida (@yoshi_ken)
+Copyright (c) 2013- Kentaro Yoshida ([@yoshi_ken](https://twitter.com/yoshi_ken))
 
 ## License
 

data/fluent-plugin-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-geoip"
-  spec.version       = "0.0.6"
+  spec.version       = "0.1.0"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}
@@ -19,5 +19,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
   spec.add_runtime_dependency "geoip-c"
 end

data/lib/fluent/plugin/out_geoip.rb

@@ -1,43 +1,75 @@
+require 'fluent/mixin/rewrite_tag_name'
+
 class Fluent::GeoipOutput < Fluent::BufferedOutput
   Fluent::Plugin.register_output('geoip', self)
 
+  REGEXP_PLACEHOLDER = /^\$\{(?<geoip_key>-?[^\[]+)\['(?<record_key>-?[^']+)'\]\}$/
   GEOIP_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
+
   config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
   config_param :geoip_lookup_key, :string, :default => 'host'
+  config_param :tag, :string, :default => nil
 
   include Fluent::HandleTagNameMixin
   include Fluent::SetTagKeyMixin
   config_set_default :include_tag_key, false
-  attr_reader :geoip_keys_map
+
+  include Fluent::Mixin::RewriteTagName
+  config_param :hostname_command, :string, :default => 'hostname'
+
+  config_param :flush_interval, :time, :default => 0
+  config_param :log_level, :string, :default => 'warn'
+
+  # Define `log` method for v0.10.42 or earlier
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
 
   def initialize
     require 'geoip'
+    require 'yajl'
+
     super
   end
 
   def configure(conf)
     super
 
-    @geoip_keys_map = Hash.new
+    @map = {}
+    @geoip_lookup_key = @geoip_lookup_key.split(/\s*,\s*/)
+
+    # enable_key_* format (legacy format)
     conf.keys.select{|k| k =~ /^enable_key_/}.each do |key|
-      geoip_key_name = key.sub('enable_key_','')
-      raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key_name}" unless GEOIP_KEYS.include?(geoip_key_name)
-      @geoip_keys_map.store(geoip_key_name, conf[key].split(/\s*,\s*/))
+      geoip_key = key.sub('enable_key_','')
+      raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
+      @geoip_lookup_key.zip(conf[key].split(/\s*,\s*/)).each do |lookup_field,record_key|
+        if record_key.nil?
+          raise Fluent::ConfigError, "geoip: missing value found at '#{key} #{lookup_field}'"
+        end
+        @map.store(record_key, "${#{geoip_key}['#{lookup_field}']}")
+      end
+    end
+    if conf.keys.select{|k| k =~ /^enable_key_/}.size > 0
+      log.warn "geoip: 'enable_key_*' config format is obsoleted. use <record></record> directive for now."
+      log.warn "geoip: for further details referable to https://github.com/y-ken/fluent-plugin-geoip"
     end
 
-    @geoip_lookup_key = @geoip_lookup_key.split(/\s*,\s*/).map {|lookupkey|
-      lookupkey.split(".")
-    }
-    if @geoip_lookup_key.size > 1
-      @geoip_keys_map.each{|name, key|
-        if key.size != @geoip_lookup_key.size
-          raise Fluent::ConfigError, "geoip: lookup key length is not match #{name}"
-        end
+    # <record></record> directive
+    conf.elements.select { |element| element.name == 'record' }.each { |element|
+      element.each_pair { |k, v|
+        element.has_key?(k) # to suppress unread configuration warning
+        @map[k] = v
       }
+    }
+    @placeholder_keys = @map.values.join.scan(/(\$\{[^}]+\})/).map{ |placeholder| placeholder[0] }.uniq
+    @placeholder_keys.each do |key|
+      geoip_key = key.match(REGEXP_PLACEHOLDER)[:geoip_key]
+      raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
     end
+    @placeholder_expander = PlaceholderExpander.new
 
-    if ( !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
-      raise Fluent::ConfigError, "geoip: missing remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix."
+    if ( !@tag && !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
+      raise Fluent::ConfigError, "geoip: required at least one option of 'tag', 'remove_tag_prefix', 'remove_tag_suffix', 'add_tag_prefix', 'add_tag_suffix'."
     end
 
     @geoip = GeoIP::City.new(@geoip_database, :memory, false)
@@ -61,27 +93,63 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
     end
   end
 
+
+  private
+  def add_geoip_field(record)
+    placeholder = create_placeholder(geolocate(get_address(record)))
+    @map.each do |record_key, value|
+      if value.match(REGEXP_PLACEHOLDER)
+        rewrited = placeholder[value]
+      else
+        rewrited = value.gsub(/\$\{[^\}]+?\}/, placeholder)
+        if rewrited.match(/(^[\[\{]|^[\d\.\-]+$)/) 
+          rewrited = parse_json(rewrited)
+        end
+      end
+      record.store(record_key, rewrited)
+    end
+    return record
+  end
+
+  def parse_json(message)
+    begin
+      return Yajl::Parser.parse(message)
+    rescue Yajl::ParseError => e
+      log.info "geoip: failed to parse '#{message}' as json.", :error_class => e.class, :error => e.message
+      return nil
+    end
+  end
+
   def get_address(record)
-    @geoip_lookup_key.map {|key|
+    address = {}
+    @geoip_lookup_key.each do |field|
+      key = field.split('.')
       obj = record
       key.each {|k|
         break obj = nil if not obj.has_key?(k)
         obj = obj[k]
       }
-      obj
-    }
+      address.store(field, obj)
+    end
+    return address
   end
 
-  def add_geoip_field(record)
-    addresses = get_address(record)
-    return record if addresses.all? {|address| address == nil }
-    results = addresses.map {|address| @geoip.look_up(address) }
-    return record if results.all? {|result| result == nil }
-    @geoip_keys_map.each do |geoip_key,record_keys|
-      record_keys.each_with_index {|record_key, idx|
-        record.store(record_key, results[idx][geoip_key.to_sym])
-      }
+  def geolocate(addresses)
+    geodata = {}
+    addresses.each do |field, ip|
+      geo = ip.nil? ? nil : @geoip.look_up(ip)
+      geodata.store(field, geo)
     end
-    return record
+    return geodata
+  end
+
+  def create_placeholder(geodata)
+    placeholder = {}
+    @placeholder_keys.each do |placeholder_key|
+      position = placeholder_key.match(REGEXP_PLACEHOLDER)
+      next if position.nil? or geodata[position[:record_key]].nil?
+      placeholder.store(placeholder_key, geodata[position[:record_key]][position[:geoip_key].to_sym])
+    end
+    return placeholder
   end
 end

data/test/plugin/test_out_geoip.rb

@@ -29,7 +29,6 @@ class GeoipOutputTest < Test::Unit::TestCase
       add_tag_prefix    geoip.
     ]
     assert_equal 'geoip_city', d.instance.config['enable_key_city']
-    assert_equal ['geoip_city'], d.instance.geoip_keys_map['city']
 
     # multiple key config
     d = create_driver %[
@@ -39,7 +38,6 @@ class GeoipOutputTest < Test::Unit::TestCase
       add_tag_prefix    geoip.
     ]
     assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
-    assert_equal ['from_city', 'to_city'], d.instance.geoip_keys_map['city']
 
     # multiple key config (bad configure)
     assert_raise(Fluent::ConfigError) {
@@ -66,6 +64,24 @@ class GeoipOutputTest < Test::Unit::TestCase
     assert_equal nil, emits[1][2]['geoip_city']
   end
 
+  def test_emit_tag_option
+    d1 = create_driver(%[
+      geoip_lookup_key  host
+      enable_key_city   geoip_city
+      remove_tag_prefix input.
+      tag               geoip.${tag}
+    ], 'input.access')
+    d1.run do
+      d1.emit({'host' => '66.102.3.80', 'message' => 'valid ip'})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+    assert_equal 'geoip.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['geoip_city']
+    assert_equal nil, emits[1][2]['geoip_city']
+  end
+
   def test_emit_nested_attr
     d1 = create_driver(%[
       geoip_lookup_key  host.ip
@@ -127,6 +143,82 @@ class GeoipOutputTest < Test::Unit::TestCase
       remove_tag_prefix input.
       add_tag_prefix    geoip.
     ], 'input.access')
+    d1.run do
+      d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
+      d1.emit({'from' => {'ip' => '66.102.3.80'}})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 3, emits.length
+    assert_equal 'geoip.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['from_city']
+    assert_equal 'United States', emits[0][2]['from_country']
+    assert_equal 'Musashino', emits[0][2]['to_city']
+    assert_equal 'Japan', emits[0][2]['to_country']
+
+    assert_equal 'Mountain View', emits[1][2]['from_city']
+    assert_equal 'United States', emits[1][2]['from_country']
+    assert_equal nil, emits[1][2]['to_city']
+    assert_equal nil, emits[1][2]['to_country']
+
+    assert_equal nil, emits[2][2]['from_city']
+    assert_equal nil, emits[2][2]['from_country']
+    assert_equal nil, emits[2][2]['to_city']
+    assert_equal nil, emits[2][2]['to_country']
+  end
+
+  def test_emit_record_directive
+    d1 = create_driver(%[
+      geoip_lookup_key  from.ip
+      <record>
+        from_city       ${city['from.ip']}
+        from_country    ${country_name['from.ip']}
+        latitude        ${latitude['from.ip']}
+        longitude       ${longitude['from.ip']}
+        location_string ${latitude['from.ip']},${longitude['from.ip']}
+        location_array  [${longitude['from.ip']},${latitude['from.ip']}]
+        location_nest   { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
+      </record>
+      remove_tag_prefix input.
+      tag               geoip.${tag}
+    ], 'input.access')
+    d1.run do
+      d1.emit({'from' => {'ip' => '66.102.3.80'}})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+    assert_equal 'geoip.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['from_city']
+    assert_equal 'United States', emits[0][2]['from_country']
+    assert_equal 37.4192008972168, emits[0][2]['latitude']
+    assert_equal -122.05740356445312, emits[0][2]['longitude']
+    assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['location_string']
+    assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['location_array']
+    location_nest = {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
+    assert_equal location_nest, emits[0][2]['location_nest']
+    assert_equal nil, emits[0][2]['undefined']
+    assert_equal nil, emits[1][2]['from_city']
+    assert_equal nil, emits[1][2]['from_country']
+    assert_equal nil, emits[1][2]['latitude']
+    assert_equal nil, emits[1][2]['longitude']
+    assert_equal ',', emits[1][2]['location_string']
+    assert_equal nil, emits[1][2]['location_array']
+    assert_equal nil, emits[1][2]['location_nest']
+  end
+
+  def test_emit_record_directive_multiple_record
+    d1 = create_driver(%[
+      geoip_lookup_key  from.ip, to.ip
+      <record>
+        from_city       ${city['from.ip']}
+        to_city         ${city['to.ip']}
+        from_country    ${country_name['from.ip']}
+        to_country      ${country_name['to.ip']}
+      </record>
+      remove_tag_prefix input.
+      tag               geoip.${tag}
+    ], 'input.access')
     d1.run do
       d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
       d1.emit({'message' => 'missing field'})
@@ -140,6 +232,44 @@ class GeoipOutputTest < Test::Unit::TestCase
     assert_equal 'Japan', emits[0][2]['to_country']
     assert_equal nil, emits[1][2]['from_city']
     assert_equal nil, emits[1][2]['to_city']
+    assert_equal nil, emits[1][2]['from_country']
+    assert_equal nil, emits[1][2]['to_country']
   end
 
+  def test_emit_record_directive_aggressive
+    d1 = create_driver(%[
+      geoip_lookup_key  from.ip
+      <record>
+        city            ${city['from.ip']}
+        latitude        ${latitude['from.ip']}
+        longitude       ${longitude['from.ip']}
+        unknown_city    ${city['unknown_key']}
+        undefined       ${city['undefined']}
+        broken_array1   [${longitude['from.ip']},${latitude['undefined']}]
+        broken_array2   [${longitude['undefined']},${latitude['undefined']}]
+      </record>
+      remove_tag_prefix input.
+      tag               geoip.${tag}
+    ], 'input.access')
+    d1.run do
+      d1.emit({'from' => {'ip' => '66.102.3.80'}})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+    assert_equal 'geoip.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['city']
+    assert_equal 37.4192008972168, emits[0][2]['latitude']
+    assert_equal -122.05740356445312, emits[0][2]['longitude']
+    assert_equal nil, emits[0][2]['unknown_city']
+    assert_equal nil, emits[0][2]['undefined']
+    assert_equal nil, emits[0][2]['broken_array1']
+    assert_equal nil, emits[0][2]['broken_array2']
+
+    assert_equal nil, emits[1][2]['city']
+    assert_equal nil, emits[1][2]['unknown_city']
+    assert_equal nil, emits[1][2]['undefined']
+    assert_equal nil, emits[1][2]['broken_array1']
+    assert_equal nil, emits[1][2]['broken_array2']
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-geoip
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-05 00:00:00.000000000 Z
+date: 2014-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -59,6 +59,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-rewrite-tag-name
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: geoip-c
   requirement: !ruby/object:Gem::Requirement