fluent-plugin-geoip 0.6.0 → 0.6.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8fdb172feeffd90a77f0a83f3f44b58b88ddb138
+  data.tar.gz: 3abc009ded4a7ced18e046c27d9f50094f11b865
+SHA512:
+  metadata.gz: c5ef5143cdd8f2ea9fe372d5a2e0ee5bbe5981a4a1c2cd84b34364fe7f6679a897b5a40732d8b97ad9dfdf5f36b3dc257ec529357c2411e817cfdb34822e05f0
+  data.tar.gz: 70d27191a5cc8f7f4ee5153845149f7d0881c5bf08ed7554ab5424a65f7806b78f36909fa8319d1850d49c13d2247fa41f466cf954777e85d80d45065948eedb

data/.travis.yml

@@ -18,3 +18,9 @@ gemfile:
   - gemfiles/fluentd_v0.10.gemfile
   - gemfiles/fluentd_v0.12.gemfile
 
+matrix:
+  fast_finish: true
+  allow_failures:
+    - gemfile: gemfiles/fluentd_v0.10.gemfile
+    - rvm: 1.9.3
+    - rvm: 2.0.0

data/README.md

@@ -22,6 +22,9 @@ $ sudo yum install geoip-devel --enablerepo=epel
 # for Ubuntu/Debian
 $ sudo apt-get install build-essential
 $ sudo apt-get install libgeoip-dev
+
+# for OS X
+$ brew install geoip
 ```
 
 ## Installation
@@ -41,6 +44,8 @@ $ sudo td-agent-gem install fluent-plugin-geoip
 
 ## Usage
 
+### For GeoipOutput
+
 ```xml
 <match access.apache>
   type geoip
@@ -54,7 +59,6 @@ $ sudo td-agent-gem install fluent-plugin-geoip
 
   # Set adding field with placeholder (more than one settings are required.)
   <record>
-    city            ${city["host"]}
     latitude        ${latitude["host"]}
     longitude       ${longitude["host"]}
     country_code3   ${country_code3["host"]}
@@ -63,6 +67,7 @@ $ sudo td-agent-gem install fluent-plugin-geoip
     dma             ${dma_code["host"]}
     area            ${area_code["host"]}
     region          ${region["host"]}
+    city            ${city["host"]}
   </record>
 
   # Settings for tag
@@ -142,8 +147,49 @@ On the case of using td-agent2 (v1-config), it have to quote `{ ... }` or `[ ...
 </match>
 ```
 
+### For GeoipFilter
+
+Note that filter version of geoip plugin does not have handling tag feature.
+
+```xml
+<filter access.apache>
+  @type geoip
+
+  # Specify one or more geoip lookup field which has ip address (default: host)
+  # in the case of accessing nested value, delimit keys by dot like 'host.ip'.
+  geoip_lookup_key  host
+
+  # Specify optional geoip database (using bundled GeoLiteCity databse by default)
+  geoip_database    "/path/to/your/GeoIPCity.dat"
+
+  # Set adding field with placeholder (more than one settings are required.)
+  <record>
+    city            ${city["host"]}
+    latitude        ${latitude["host"]}
+    longitude       ${longitude["host"]}
+    country_code3   ${country_code3["host"]}
+    country         ${country_code["host"]}
+    country_name    ${country_name["host"]}
+    dma             ${dma_code["host"]}
+    area            ${area_code["host"]}
+    region          ${region["host"]}
+  </record>
+
+  # To avoid get stacktrace error with `[null, null]` array for elasticsearch.
+  skip_adding_null_record  true
+
+  # Set log_level for fluentd-v0.10.43 or earlier (default: warn)
+  log_level         info
+
+  # Set buffering time (default: 0s)
+  flush_interval    1s
+</filter>
+```
+
 ## Tutorial
 
+### For GeoipOutput
+
 #### configuration
 
 ```xml
@@ -160,9 +206,9 @@ On the case of using td-agent2 (v1-config), it have to quote `{ ... }` or `[ ...
     type    geoip
     geoip_lookup_key  host
     <record>
-      city  ${city["host"]}
-      lat   ${latitude["host"]}
-      lon   ${longitude["host"]}
+      lat     ${latitude["host"]}
+      lon     ${longitude["host"]}
+      country ${country_code["host"]}
     </record>
     remove_tag_prefix test.
     tag     debug.${tag}
@@ -183,7 +229,45 @@ $ echo '{"host":"66.102.9.80","message":"test"}' | fluent-cat test.geoip
 # check the result at stdout
 $ tail /var/log/td-agent/td-agent.log
 2013-08-04 16:21:32 +0900 test.geoip: {"host":"66.102.9.80","message":"test"}
-2013-08-04 16:21:32 +0900 debug.geoip: {"host":"66.102.9.80","message":"test","city":"Mountain View","lat":37.4192008972168,"lon":-122.05740356445312}
+2013-08-04 16:21:32 +0900 debug.geoip: {"host":"66.102.9.80","message":"test","lat":37.4192008972168,"lon":-122.05740356445312,"country":"US"}
+```
+
+For more details of geoip data format is described at the page below in section `GeoIP City Edition CSV Database Fields`.<br />
+http://dev.maxmind.com/geoip/legacy/csv/
+
+### For GeoipFilter
+
+#### configuration
+
+```xml
+<source>
+  @type forward
+</source>
+
+<filter test.geoip>
+  @type    geoip
+  geoip_lookup_key  host
+  <record>
+    city  ${city["host"]}
+    lat   ${latitude["host"]}
+    lon   ${longitude["host"]}
+  </record>
+</filter>
+
+<match test.**>
+  @type stdout
+</match>
+```
+
+#### result
+
+```bash
+# forward record with Google's ip address.
+$ echo '{"host":"66.102.9.80","message":"test"}' | fluent-cat test.geoip
+
+# check the result at stdout
+$ tail /var/log/td-agent/td-agent.log
+2016-02-01 12:04:37 +0900 test.geoip: {"host":"66.102.9.80","message":"test","city":"Mountain View","lat":37.4192008972168,"lon":-122.05740356445312}
 ```
 
 For more details of geoip data format is described at the page below in section `GeoIP City Edition CSV Database Fields`.<br />
@@ -210,6 +294,8 @@ Further more specification available at http://dev.maxmind.com/geoip/legacy/csv/
 
 ## Parameters
 
+### GeoipOutput
+
 * `include_tag_key` (default: false)
 * `tag_key`
 
@@ -236,6 +322,24 @@ On using this option with tag placeholder like `tag geoip.${tag}` (test code is
 
 Set buffering time to execute bulk lookup geoip.
 
+### GeoipFilter
+
+Note that filter version of `geoip` plugin does not have handling `tag` feature.
+
+* `include_tag_key` (default: false)
+
+Add original tag name into filtered record using SetTagKeyMixin.<br />
+Further details are written at http://docs.fluentd.org/articles/in_exec
+
+* `skip_adding_null_record` (default: false)
+
+Skip adding geoip fields when this valaues to `true`.
+On the case of getting nothing of GeoIP info (such as local IP), it will output the original record without changing anything.
+
+* `flush_interval` (default: 0 sec)
+
+Set buffering time to execute bulk lookup geoip.
+
 ## Articles
 
 * [IPアドレスを元に位置情報をリアルタイムに付与する fluent-plugin-geoip v0.0.1をリリースしました #fluentd - Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-has-released)<br />

data/fluent-plugin-geoip.gemspec

@@ -4,10 +4,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-geoip"
-  spec.version       = "0.6.0"
+  spec.version       = "0.6.1"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
-  spec.summary       = %q{Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}
+  spec.summary       = %q{Fluentd Filter plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}
   spec.homepage      = "https://github.com/y-ken/fluent-plugin-geoip"
   spec.license       = "Apache-2.0"
 

data/lib/fluent/plugin/filter_geoip.rb

@@ -25,8 +25,17 @@ module Fluent
       @geoip = Fluent::GeoIP.new(self, conf)
     end
 
-    def filter(tag, time, record)
-      @geoip.add_geoip_field(record)
+    def filter_stream(tag, es)
+      new_es = MultiEventStream.new
+      es.each do |time, record|
+        begin
+          filtered_record = @geoip.add_geoip_field(record)
+          new_es.add(time, filtered_record) if filtered_record
+        rescue => e
+          router.emit_error_event(tag, time, record, e)
+        end
+      end
+      new_es
     end
   end
 end

data/lib/fluent/plugin/geoip.rb

@@ -56,7 +56,7 @@ module Fluent
         raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
       end
 
-      if plugin.is_a?(Fluent::Output)
+      if plugin.is_a?(Fluent::BufferedOutput)
         @placeholder_expander = PlaceholderExpander.new
         unless have_tag_option?(plugin)
           raise Fluent::ConfigError, "geoip: required at least one option of 'tag', 'remove_tag_prefix', 'remove_tag_suffix', 'add_tag_prefix', 'add_tag_suffix'."

data/lib/fluent/plugin/out_geoip.rb

@@ -55,8 +55,12 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
   end
 
   def write(chunk)
-    chunk.msgpack_each do |tag, time, record|
-      router.emit(tag, time, @geoip.add_geoip_field(record))
+    es = Fluent::MultiEventStream.new
+    tag = ""
+    chunk.msgpack_each do |_tag, time, record|
+      tag = _tag
+      es.add(time, @geoip.add_geoip_field(record))
     end
+    router.emit_stream(tag, es)
   end
 end

data/test/plugin/test_out_geoip.rb

@@ -13,7 +13,7 @@ class GeoipOutputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf=CONFIG,tag='test',use_v1=false)
-    Fluent::Test::OutputTestDriver.new(Fluent::GeoipOutput, tag).configure(conf, use_v1)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::GeoipOutput, tag).configure(conf, use_v1)
   end
 
   def test_configure

metadata

@@ -1,126 +1,111 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-geoip
 version: !ruby/object:Gem::Version
-  version: 0.6.0
-  prerelease: 
+  version: 0.6.1
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-29 00:00:00.000000000 Z
+date: 2016-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluent-mixin-rewrite-tag-name
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: geoip-c
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: 
@@ -130,8 +115,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Appraisals
 - Gemfile
 - LICENSE
@@ -150,28 +135,27 @@ files:
 homepage: https://github.com/y-ken/fluent-plugin-geoip
 licenses:
 - Apache-2.0
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.4.5
 signing_key: 
-specification_version: 3
-summary: Fluentd Output plugin to add information about geographical location of IP
+specification_version: 4
+summary: Fluentd Filter plugin to add information about geographical location of IP
   addresses with Maxmind GeoIP databases.
 test_files:
 - test/helper.rb