fluent-plugin-geoip 0.0.4 → 0.0.5

data/.travis.yml

@@ -1,9 +1,9 @@
 language: ruby
 
 rvm:
+  - 2.1.0
   - 2.0.0
   - 1.9.3
-  - rbx-19mode
 
 before_install:
   - sudo apt-get install libgeoip-dev

data/README.md

@@ -10,11 +10,23 @@ The accuracy details for GeoLite City (free) and GeoIP City (purchased) has desc
 * http://www.maxmind.com/en/geolite_city_accuracy ([lang:ja](http://www.maxmind.com/ja/geolite_city_accuracy))
 * http://www.maxmind.com/en/city_accuracy ([lang:ja](http://www.maxmind.com/ja/city_accuracy))
 
+## Dependency
+
+before use, install dependent library as:
+
+```bash
+# for RHEL/CentOS
+$ sudo yum install geoip-devel --enablerepo=epel
+
+# for Ubuntu/Debian
+$ sudo apt-get install libgeoip-dev
+```
+
 ## Installation
 
 install with `gem` or `fluent-gem` command as:
 
-```
+```bash
 # for fluentd
 $ gem install fluent-plugin-geoip
 
@@ -31,18 +43,22 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
   # buffering time (default: 60s)
   flush_interval           1s
 
+  # Queued chunks are flushed at shutdown process.
+  flush_at_shutdown yes
+
   # tag settings
   remove_tag_prefix        access.
   add_tag_prefix           geoip.
   include_tag_key          false
 
   # specify geoip lookup field (default: host)
+  # in the case of accessing nested value, delimit keys by dot like 'host.ip'.
   geoip_lookup_key         host
 
   # specify geoip database (using bundled GeoLiteCity databse by default)
   geoip_database           'data/GeoLiteCity.dat'
 
-  # record settings (enable more than one keys required.)
+  # record settings (enable more than one keys are required.)
   enable_key_city          geoip_city
   enable_key_latitude      geoip_lat
   enable_key_longitude     geoip_lon
@@ -98,11 +114,26 @@ $ tail /var/log/td-agent/td-agent.log
 2013-08-04 16:21:32 +0900 debug.geoip: {"host":"66.102.9.80","message":"test","city":"Mountain View","lat":37.4192008972168,"lon":-122.05740356445312}
 ```
 
+For more details of geoip data format is described at the page below in section `GeoIP City Edition CSV Database Fields`.  
+http://dev.maxmind.com/geoip/legacy/csv/
+
 ## Articles
 
 * [IPアドレスを元に位置情報をリアルタイムに付与する fluent-plugin-geoip v0.0.1をリリースしました #fluentd - Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-has-released)  
 http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-has-released
 
+* [初の安定版 fluent-plugin-geoip v0.0.3 をリリースしました #fluentd- Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.3)  
+http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.3
+
+* [fluent-plugin-geoip v0.0.4 をリリースしました。ElasticSearch＋Kibanaの世界地図に位置情報をプロットするために必要なFluentdの設定サンプルも紹介します- Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.4)  
+http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.4
+
+* [Released GeoIP plugin to work together with ElasticSearch + Kibana v3](https://groups.google.com/d/topic/fluentd/OVIcH_SKBwM/discussion)  
+https://groups.google.com/d/topic/fluentd/OVIcH_SKBwM/discussion
+
+* [Fluentd、Amazon RedshiftとTableauを用いたカジュアルなデータ可視化 | SmartNews開発者ブログ](http://developer.smartnews.be/blog/2013/10/03/easy-data-analysis-using-fluentd-redshift-and-tableau/)  
+http://developer.smartnews.be/blog/2013/10/03/easy-data-analysis-using-fluentd-redshift-and-tableau/
+
 ## TODO
 
 Pull requests are very welcome!!

data/fluent-plugin-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-geoip"
-  spec.version       = "0.0.4"
+  spec.version       = "0.0.5"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}

data/lib/fluent/plugin/out_geoip.rb

@@ -1,6 +1,6 @@
 class Fluent::GeoipOutput < Fluent::BufferedOutput
   Fluent::Plugin.register_output('geoip', self)
-  
+
   GEOIP_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
   config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
   config_param :geoip_lookup_key, :string, :default => 'host'
@@ -13,7 +13,7 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
     require 'geoip'
     super
   end
-  
+
   def configure(conf)
     super
 
@@ -24,6 +24,9 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
       @geoip_keys_map.store(geoip_key_name, conf[key])
     end
 
+    @geoip_lookup_key = @geoip_lookup_key.split(".")
+
+
     if ( !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
       raise Fluent::ConfigError, "geoip: missing remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix."
     end
@@ -34,11 +37,11 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
   def start
     super
   end
-  
+
   def format(tag, time, record)
     [tag, time, record].to_msgpack
   end
-  
+
   def shutdown
     super
   end
@@ -49,8 +52,17 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
     end
   end
 
+  def get_address(record)
+    obj = record
+    @geoip_lookup_key.each do |key|
+      return nil if not obj.has_key?(key)
+      obj = obj[key]
+    end
+    obj
+  end
+
   def add_geoip_field(record)
-    address = record[@geoip_lookup_key]
+    address = get_address(record)
     return record if address.nil?
     result = @geoip.look_up(address)
     return record if result.nil?

data/test/plugin/test_out_geoip.rb

@@ -40,6 +40,26 @@ class GeoipOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 2, emits.length
+    # p emits[0]
+    assert_equal 'geoip.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['geoip_city']
+    # p emits[1]
+    assert_equal nil, emits[1][2]['geoip_city']
+  end
+
+  def test_emit_nested_attr
+    d1 = create_driver(%[
+      geoip_lookup_key  host.ip
+      enable_key_city   geoip_city
+      remove_tag_prefix input.
+      add_tag_prefix    geoip.
+    ], 'input.access')
+    d1.run do
+      d1.emit({'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip'})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
     p emits[0]
     assert_equal 'geoip.access', emits[0][0] # tag
     assert_equal 'Mountain View', emits[0][2]['geoip_city']
@@ -56,10 +76,10 @@ class GeoipOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 2, emits.length
-    p emits[0]
+    # p emits[0]
     assert_equal 'geoip.access', emits[0][0] # tag
     assert_equal nil, emits[0][2]['geoip_city']
-    p emits[1]
+    # p emits[1]
     assert_equal 'geoip.access', emits[1][0] # tag
     assert_equal nil, emits[1][2]['geoip_city']
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-geoip
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-03 00:00:00.000000000 Z
+date: 2014-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler