fluent-plugin-geoip 0.4.0 → 0.5.0.beta

data/README.md

@@ -16,9 +16,11 @@ before use, install dependent library as:
 
 ```bash
 # for RHEL/CentOS
+$ sudo yum group install "Development Tools"
 $ sudo yum install geoip-devel --enablerepo=epel
 
 # for Ubuntu/Debian
+$ sudo apt-get install build-essential
 $ sudo apt-get install libgeoip-dev
 ```
 
@@ -67,6 +69,9 @@ $ sudo td-agent-gem install fluent-plugin-geoip
   remove_tag_prefix access.
   tag               geoip.${tag}
 
+  # To avoid get stacktrace error with `[null, null]` array for elasticsearch.
+  skip_adding_null_record  true
+
   # Set log_level for fluentd-v0.10.43 or earlier (default: warn)
   log_level         info
 
@@ -108,12 +113,15 @@ It is a sample to get friendly geo point recdords for elasticsearch with Yajl (J
     # ex. "37.4192008972168,-122.05740356445312"
     location_string      ${latitude["host"]},${longitude["host"]}
     
-    # lat lon as array (it is useful for Kibana's bettermap.)
+    # GeoJSON (lat lon as array) is useful for Kibana's bettermap.
     # ex. [-122.05740356445312, 37.4192008972168]
     location_array       [${longitude["host"]},${latitude["host"]}]
   </record>
   remove_tag_prefix      access.
   tag                    geoip.${tag}
+
+  # To avoid get stacktrace error with `[null, null]` array for elasticsearch.
+  skip_adding_null_record  true
 </match>
 ```
 
@@ -130,6 +138,7 @@ On the case of using td-agent2 (v1-config), it have to quote `{ ... }` or `[ ...
   </record>
   remove_tag_prefix      access.
   tag                    geoip.${tag}
+  skip_adding_null_record  true
 </match>
 ```
 
@@ -182,17 +191,22 @@ http://dev.maxmind.com/geoip/legacy/csv/
 
 ## Placeholders
 
-Provides these placeholders for adding field of geolocate results.
+Provides these placeholders for adding field of geolocate results.<br />
+For more example of geolocating, you can try these websites like [Geo IP Address View](http://www.geoipview.com/) or [View my IP information](http://www.geoiptool.com/en/).
+
+| placeholder attributes         | output example    | type         | note |
+|--------------------------------|-------------------|--------------|------|
+| ${city[lookup_field]}          | "Ithaca"          | varchar(255) |  -   |
+| ${latitude[lookup_field]}      | 42.4277992248535  | decimal      |  -   |
+| ${longitude[lookup_field]}     | -76.4981994628906 | decimal      |  -   |
+| ${country_code3[lookup_field]} | "USA"             | varchar(3)   |  -   |
+| ${country_code[lookup_field]}  | "US"              | varchar(2)   | A two-character ISO 3166-1 country code      |
+| ${country_name[lookup_field]}  | "United States"   | varchar(50)  |  -   |
+| ${dma_code[lookup_field]}      | 555               | unsigned int | **only for US**  |
+| ${area_code[lookup_field]}     | 607               | char(3)      | **only for US**  |
+| ${region[lookup_field]}        | "NY"              | char(2)      | A two character ISO-3166-2 or FIPS 10-4 code |
 
-* ${city[lookup_field]}
-* ${latitude[lookup_field]}
-* ${longitude[lookup_field]}
-* ${country_code3[lookup_field]}
-* ${country_code[lookup_field]}
-* ${country_name[lookup_field]}
-* ${dma_code[lookup_field]}
-* ${area_code[lookup_field]}
-* ${region[lookup_field]}
+Further more specification available at http://dev.maxmind.com/geoip/legacy/csv/#GeoIP_City_Edition_CSV_Database_Fields
 
 ## Parameters
 
@@ -202,6 +216,11 @@ Provides these placeholders for adding field of geolocate results.
 Add original tag name into filtered record using SetTagKeyMixin.<br />
 Further details are written at http://docs.fluentd.org/articles/in_exec
 
+* `skip_adding_null_record` (default: false)
+
+Skip adding geoip fields when this valaues to `true`.
+On the case of getting nothing of GeoIP info (such as local IP), it will output the original record without changing anything.
+
 * `remove_tag_prefix`
 * `remove_tag_suffix`
 * `add_tag_prefix`

data/fluent-plugin-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-geoip"
-  spec.version       = "0.4.0"
+  spec.version       = "0.5.0.beta"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}
@@ -18,6 +18,11 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
+
+  if defined?(RUBY_VERSION) && RUBY_VERSION > '2.2'
+    spec.add_development_dependency "test-unit", '~> 3'
+  end
+
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
   spec.add_runtime_dependency "geoip-c"

data/lib/fluent/plugin/out_geoip.rb

@@ -11,6 +11,7 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
   config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
   config_param :geoip_lookup_key, :string, :default => 'host'
   config_param :tag, :string, :default => nil
+  config_param :skip_adding_null_record, :bool, :default => false
 
   include Fluent::HandleTagNameMixin
   include Fluent::SetTagKeyMixin
@@ -119,6 +120,7 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
 
   def add_geoip_field(record)
     placeholder = create_placeholder(geolocate(get_address(record)))
+    return record if @skip_adding_null_record && placeholder.values.first.nil?
     @map.each do |record_key, value|
       if value.match(REGEXP_PLACEHOLDER_SINGLE)
         rewrited = placeholder[value]

data/test/plugin/test_out_geoip.rb

@@ -162,7 +162,16 @@ class GeoipOutputTest < Test::Unit::TestCase
   end
 
   def test_emit_with_unknown_address
-    d1 = create_driver(CONFIG, 'input.access')
+    d1 = create_driver(%[
+      geoip_lookup_key  host
+      <record>
+        geoip_city      ${city['host']}
+        geopoint        [${longitude['host']}, ${latitude['host']}]
+      </record>
+      skip_adding_null_record false
+      remove_tag_prefix input.
+      tag               geoip.${tag}
+    ], 'input.access')
     d1.run do
       # 203.0.113.1 is a test address described in RFC5737
       d1.emit({'host' => '203.0.113.1', 'message' => 'invalid ip'})
@@ -176,6 +185,35 @@ class GeoipOutputTest < Test::Unit::TestCase
     assert_equal nil, emits[1][2]['geoip_city']
   end
 
+  def test_emit_with_skip_unknown_address
+    d1 = create_driver(%[
+      geoip_lookup_key  host
+      <record>
+        geoip_city      ${city['host']}
+        geopoint        [${longitude['host']}, ${latitude['host']}]
+      </record>
+      skip_adding_null_record true
+      remove_tag_prefix input.
+      tag               geoip.${tag}
+    ], 'input.access')
+    d1.run do
+      # 203.0.113.1 is a test address described in RFC5737
+      d1.emit({'host' => '203.0.113.1', 'message' => 'invalid ip'})
+      d1.emit({'host' => '0', 'message' => 'invalid ip'})
+      d1.emit({'host' => '8.8.8.8', 'message' => 'google public dns'})
+    end
+    emits = d1.emits
+    assert_equal 3, emits.length
+    assert_equal 'geoip.access', emits[0][0] # tag
+    assert_equal nil, emits[0][2]['geoip_city']
+    assert_equal nil, emits[0][2]['geopoint']
+    assert_equal 'geoip.access', emits[1][0] # tag
+    assert_equal nil, emits[1][2]['geoip_city']
+    assert_equal nil, emits[1][2]['geopoint']
+    assert_equal 'Mountain View', emits[2][2]['geoip_city']
+    assert_equal [-122.08380126953125, 37.38600158691406], emits[2][2]['geopoint']
+  end
+
   def test_emit_multiple_key
     d1 = create_driver(%[
       geoip_lookup_key  from.ip, to.ip

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-geoip
 version: !ruby/object:Gem::Version
-  version: 0.4.0
-  prerelease: 
+  version: 0.5.0.beta
+  prerelease: 6
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-28 00:00:00.000000000 Z
+date: 2015-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -125,9 +125,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23