fluent-plugin-geoip 0.0.5 → 0.0.6

data/README.md

@@ -36,29 +36,18 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
 
 ## Usage
 
-```
+```xml
 <match access.apache>
   type geoip
 
-  # buffering time (default: 60s)
-  flush_interval           1s
-
-  # Queued chunks are flushed at shutdown process.
-  flush_at_shutdown yes
-
-  # tag settings
-  remove_tag_prefix        access.
-  add_tag_prefix           geoip.
-  include_tag_key          false
-
-  # specify geoip lookup field (default: host)
+  # Specify geoip lookup field (default: host)
   # in the case of accessing nested value, delimit keys by dot like 'host.ip'.
   geoip_lookup_key         host
 
-  # specify geoip database (using bundled GeoLiteCity databse by default)
+  # Specify geoip database (using bundled GeoLiteCity databse by default)
   geoip_database           'data/GeoLiteCity.dat'
 
-  # record settings (enable more than one keys are required.)
+  # Set adding field of geolocate results (more than one settings are required.)
   enable_key_city          geoip_city
   enable_key_latitude      geoip_lat
   enable_key_longitude     geoip_lon
@@ -68,6 +57,36 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
   enable_key_dma_code      geoip_dma
   enable_key_area_code     geoip_area
   enable_key_region        geoip_region
+
+  # Setting for tag
+  remove_tag_prefix        access.
+  add_tag_prefix           geoip.
+  include_tag_key          false
+
+  # Buffering time (default: 60s)
+  flush_interval           1s
+</match>
+```
+
+#### Tips: how to geolocate multiple key
+
+```xml
+<match access.apache>
+  type geoip
+
+  # Set ip address key to geolocate
+  geoip_lookup_key        user1_host, user2_host
+
+  # Set adding field of geolocate results
+  enable_key_city         user1_city, user2_city
+  enable_key_country_name user1_country, user2_country
+
+  # Setting for tag
+  remove_tag_prefix       access.
+  add_tag_prefix          geoip.
+
+  # Buffering time
+  flush_interval          1s
 </match>
 ```
 
@@ -75,7 +94,7 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
 
 #### configuration
 
-```
+```xml
 <source>
   type forward
 </source>
@@ -104,7 +123,7 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
 
 #### result
 
-```
+```bash
 # forward record with Google's ip address.
 $ echo '{"host":"66.102.9.80","message":"test"}' | fluent-cat test.geoip
 

data/fluent-plugin-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-geoip"
-  spec.version       = "0.0.5"
+  spec.version       = "0.0.6"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}

data/lib/fluent/plugin/out_geoip.rb

@@ -8,6 +8,7 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
   include Fluent::HandleTagNameMixin
   include Fluent::SetTagKeyMixin
   config_set_default :include_tag_key, false
+  attr_reader :geoip_keys_map
 
   def initialize
     require 'geoip'
@@ -21,11 +22,19 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
     conf.keys.select{|k| k =~ /^enable_key_/}.each do |key|
       geoip_key_name = key.sub('enable_key_','')
       raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key_name}" unless GEOIP_KEYS.include?(geoip_key_name)
-      @geoip_keys_map.store(geoip_key_name, conf[key])
+      @geoip_keys_map.store(geoip_key_name, conf[key].split(/\s*,\s*/))
     end
 
-    @geoip_lookup_key = @geoip_lookup_key.split(".")
-
+    @geoip_lookup_key = @geoip_lookup_key.split(/\s*,\s*/).map {|lookupkey|
+      lookupkey.split(".")
+    }
+    if @geoip_lookup_key.size > 1
+      @geoip_keys_map.each{|name, key|
+        if key.size != @geoip_lookup_key.size
+          raise Fluent::ConfigError, "geoip: lookup key length is not match #{name}"
+        end
+      }
+    end
 
     if ( !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
       raise Fluent::ConfigError, "geoip: missing remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix."
@@ -53,21 +62,25 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
   end
 
   def get_address(record)
-    obj = record
-    @geoip_lookup_key.each do |key|
-      return nil if not obj.has_key?(key)
-      obj = obj[key]
-    end
-    obj
+    @geoip_lookup_key.map {|key|
+      obj = record
+      key.each {|k|
+        break obj = nil if not obj.has_key?(k)
+        obj = obj[k]
+      }
+      obj
+    }
   end
 
   def add_geoip_field(record)
-    address = get_address(record)
-    return record if address.nil?
-    result = @geoip.look_up(address)
-    return record if result.nil?
-    @geoip_keys_map.each do |geoip_key,record_key|
-      record.store(record_key, result[geoip_key.to_sym])
+    addresses = get_address(record)
+    return record if addresses.all? {|address| address == nil }
+    results = addresses.map {|address| @geoip.look_up(address) }
+    return record if results.all? {|result| result == nil }
+    @geoip_keys_map.each do |geoip_key,record_keys|
+      record_keys.each_with_index {|record_key, idx|
+        record.store(record_key, results[idx][geoip_key.to_sym])
+      }
     end
     return record
   end

data/test/plugin/test_out_geoip.rb

@@ -28,8 +28,29 @@ class GeoipOutputTest < Test::Unit::TestCase
       remove_tag_prefix input.
       add_tag_prefix    geoip.
     ]
-    puts d.instance.inspect
     assert_equal 'geoip_city', d.instance.config['enable_key_city']
+    assert_equal ['geoip_city'], d.instance.geoip_keys_map['city']
+
+    # multiple key config
+    d = create_driver %[
+      geoip_lookup_key  from.ip, to.ip
+      enable_key_city   from_city, to_city
+      remove_tag_prefix input.
+      add_tag_prefix    geoip.
+    ]
+    assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
+    assert_equal ['from_city', 'to_city'], d.instance.geoip_keys_map['city']
+
+    # multiple key config (bad configure)
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        geoip_lookup_key  from.ip, to.ip
+        enable_key_city   from_city
+        enable_key_region from_region
+        remove_tag_prefix input.
+        add_tag_prefix    geoip.
+      ]
+    }
   end
 
   def test_emit
@@ -40,10 +61,8 @@ class GeoipOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 2, emits.length
-    # p emits[0]
     assert_equal 'geoip.access', emits[0][0] # tag
     assert_equal 'Mountain View', emits[0][2]['geoip_city']
-    # p emits[1]
     assert_equal nil, emits[1][2]['geoip_city']
   end
 
@@ -60,10 +79,8 @@ class GeoipOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 2, emits.length
-    p emits[0]
     assert_equal 'geoip.access', emits[0][0] # tag
     assert_equal 'Mountain View', emits[0][2]['geoip_city']
-    p emits[1]
     assert_equal nil, emits[1][2]['geoip_city']
   end
 
@@ -76,12 +93,53 @@ class GeoipOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 2, emits.length
-    # p emits[0]
     assert_equal 'geoip.access', emits[0][0] # tag
     assert_equal nil, emits[0][2]['geoip_city']
-    # p emits[1]
     assert_equal 'geoip.access', emits[1][0] # tag
     assert_equal nil, emits[1][2]['geoip_city']
   end
 
+  def test_emit_multiple_key
+    d1 = create_driver(%[
+      geoip_lookup_key  from.ip, to.ip
+      enable_key_city   from_city, to_city
+      remove_tag_prefix input.
+      add_tag_prefix    geoip.
+    ], 'input.access')
+    d1.run do
+      d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+    assert_equal 'geoip.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['from_city']
+    assert_equal 'Musashino', emits[0][2]['to_city']
+    assert_equal nil, emits[1][2]['from_city']
+    assert_equal nil, emits[1][2]['to_city']
+  end
+
+  def test_emit_multiple_key_multiple_record
+    d1 = create_driver(%[
+      geoip_lookup_key  from.ip, to.ip
+      enable_key_city   from_city, to_city
+      enable_key_country_name from_country, to_country
+      remove_tag_prefix input.
+      add_tag_prefix    geoip.
+    ], 'input.access')
+    d1.run do
+      d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+    assert_equal 'geoip.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['from_city']
+    assert_equal 'United States', emits[0][2]['from_country']
+    assert_equal 'Musashino', emits[0][2]['to_city']
+    assert_equal 'Japan', emits[0][2]['to_country']
+    assert_equal nil, emits[1][2]['from_city']
+    assert_equal nil, emits[1][2]['to_city']
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-geoip
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-16 00:00:00.000000000 Z
+date: 2014-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler