fluent-plugin-geoip 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d8ae0d73eec8f27ed950ce080d02d0ada455fdc7
-  data.tar.gz: 3e442d004f0e4852ea8b713749eedb1f05991eaa
+SHA256:
+  metadata.gz: 6f265530c6153294e8c0e5c35176a8e51f926708757622feee23177dbf388252
+  data.tar.gz: fd1b525e0e78306a8d87356531ddd5106de541b60e0521cad6e5cbeca15a84ff
 SHA512:
-  metadata.gz: a026783f1b3578c27081c142dc4829cdda7ed0ec0be13867ee42e6052818fde41630154d8311fc63d52b6808213bec0caf84281006143cbc8f9845f96a3e01b6
-  data.tar.gz: 70836a38b6e1596ff5dcae93dda3961b67e7a16f4dd050b2ddaa7c33ff17df63475526e8baf2b8a8e103eb81cc250be029d0ed96d7388c871fcd7bfcd8e5f79e
+  metadata.gz: 32c7fb52de7c2a970fffe128e1c86cd327b8615e6e124d3019e50c74e4808674179de9a6761a4929375c8f159866364fe2e93e1042311e310bf52639b24058ca
+  data.tar.gz: d7f3f9f3cb1d7be3bf294bdc1b000c74d379e3454f64b188ab05a2804fc1d3379c775c758e2dab37e9f0477980f9466ba239542b868d7eaaf3b9a92ab50dcee6

data/README.md

@@ -1,8 +1,8 @@
 # fluent-plugin-geoip [![Build Status](https://travis-ci.org/y-ken/fluent-plugin-geoip.png?branch=master)](https://travis-ci.org/y-ken/fluent-plugin-geoip)
 
-Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.
+Fluentd Filter/Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.
 
-fluent-plugin-geoip has bundled cost-free [GeoLite City database](http://dev.maxmind.com/geoip/legacy/geolite/) by default.<br />
+fluent-plugin-geoip has bundled cost-free [GeoLite2 Free Downloadable Databases](https://dev.maxmind.com/geoip/geoip2/geolite2/) and [GeoLite City database](http://dev.maxmind.com/geoip/legacy/geolite/) by default.<br />
 Also you can use purchased [GeoIP City database](http://www.maxmind.com/en/city) ([lang:ja](http://www.maxmind.com/ja/city)) which costs starting from $50.
 
 The accuracy details for GeoLite City (free) and GeoIP City (purchased) has described at the page below.
@@ -22,7 +22,7 @@ If you want to use this plugin with Fluentd v0.12.x or earlier use 0.8.x.
 ### Compatibility notice
 
 We've used Fluentd v1 API in this plugin since 1.0.0.
-So we have dropped some features.
+So we have dropped some features from GeoipOutput.
 
 See also [official document](http://docs.fluentd.org/v1.0/articles/plugin-update-from-v12)
 
@@ -107,10 +107,12 @@ $ sudo td-agent-gem install fluent-plugin-geoip
 
 ## Usage
 
-### For GeoipOutput
+### For GeoipFilter
+
+Note that filter version of geoip plugin does not have handling tag feature.
 
 ```xml
-<match access.apache>
+<filter access.apache>
   @type geoip
 
   # Specify one or more geoip lookup field which has ip address (default: host)
@@ -118,108 +120,89 @@ $ sudo td-agent-gem install fluent-plugin-geoip
   geoip_lookup_key  host
 
   # Specify optional geoip database (using bundled GeoLiteCity databse by default)
-  geoip_database    "/path/to/your/GeoIPCity.dat"
+  # geoip_database    "/path/to/your/GeoIPCity.dat"
   # Specify optional geoip2 database
-  # geoip2_database   "/path/to/your/GeoLite2-City.mmdb"
-  # Specify backend library (geoip, geoip2_compat, geoip2_c)
-  backend_library geoip
+  # geoip2_database   "/path/to/your/GeoLite2-City.mmdb" (using bundled GeoLite2-City.mmdb by default)
+  # Specify backend library (geoip2_c, geoip, geoip2_compat)
+  backend_library geoip2_c
 
   # Set adding field with placeholder (more than one settings are required.)
   <record>
-    latitude        ${latitude["host"]}
-    longitude       ${longitude["host"]}
-    country_code3   ${country_code3["host"]}
-    country         ${country_code["host"]}
-    country_name    ${country_name["host"]}
-    dma             ${dma_code["host"]}
-    area            ${area_code["host"]}
-    region          ${region["host"]}
-    city            ${city["host"]}
+    city            ${city.names.en["host"]}
+    latitude        ${location.latitude["host"]}
+    longitude       ${location.longitude["host"]}
+    country         ${country.iso_code["host"]}
+    country_name    ${country.names.en["host"]}
+    postal_code     ${postal.code["host"]}
   </record>
 
-  # Settings for tag
-  tag               geoip.${tag[1]}
-
   # To avoid get stacktrace error with `[null, null]` array for elasticsearch.
   skip_adding_null_record  true
 
-  # Set log_level for fluentd-v0.10.43 or earlier (default: warn)
-  log_level         info
-
-  <buffer tag>
-    # Set buffering time (default: 0s)
-    flush_interval    1s
-  </buffer>
-</match>
+  # Set @log_level (default: warn)
+  @log_level         info
+</filter>
 ```
 
 #### Tips: how to geolocate multiple key
 
 ```xml
-<match access.apache>
+<filter access.apache>
   @type geoip
   geoip_lookup_key  user1_host, user2_host
   <record>
-    user1_city      ${city["user1_host"]}
-    user2_city      ${city["user2_host"]}
+    user1_city      ${city.names.en["user1_host"]}
+    user2_city      ${city.names.en["user2_host"]}
   </record>
-  tag               geoip.${tag[1]}
-</match>
+</filter>
 ```
 
 #### Advanced config samples
 
 It is a sample to get friendly geo point recdords for elasticsearch with Yajl (JSON) parser.<br />
 
-**Notice** v0 config will be deprecated in the future.
-
 ```
-<match access.apache>
+<filter access.apache>
   @type                  geoip
   geoip_lookup_key       host
   <record>
     # lat lon as properties
     # ex. {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
-    location_properties  '{ "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }'
+    location_properties  '{ "lat" : ${location.latitude["host"]}, "lon" : ${location.longitude["host"]} }'
   
     # lat lon as string
     # ex. "37.4192008972168,-122.05740356445312"
-    location_string      ${latitude["host"]},${longitude["host"]}
+    location_string      ${location.latitude["host"]},${location.longitude["host"]}
     
     # GeoJSON (lat lon as array) is useful for Kibana's bettermap.
     # ex. [-122.05740356445312, 37.4192008972168]
-    location_array       '[${longitude["host"]},${latitude["host"]}]'
+    location_array       '[${location.longitude["host"]},${location.latitude["host"]}]'
   </record>
-  tag                    geoip.${tag[1]}
 
   # To avoid get stacktrace error with `[null, null]` array for elasticsearch.
   skip_adding_null_record  true
-</match>
+</filter>
 ```
 
 On the case of using td-agent3 (v1-config), it have to quote `{ ... }` or `[ ... ]` block with quotation like below.
 
 ```
-<match access.apache>
+<filter access.apache>
   @type                  geoip
   geoip_lookup_key       host
   <record>
-    location_properties  '{ "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }'
-    location_string      ${latitude["host"]},${longitude["host"]}
-    location_array       '[${longitude["host"]},${latitude["host"]}]'
+    location_properties  '{ "lat" : ${location.latitude["host"]}, "lon" : ${location.longitude["host"]} }'
+    location_string      ${location.latitude["host"]},${location.longitude["host"]}
+    location_array       '[${location.longitude["host"]},${location.latitude["host"]}]'
   </record>
-  remove_tag_prefix      access.
-  tag                    geoip.${tag}
   skip_adding_null_record  true
-</match>
+</filter>
 ```
 
-### For GeoipFilter
-
-Note that filter version of geoip plugin does not have handling tag feature.
+### For GeoipOutput
 
 ```xml
-<filter access.apache>
+<match access.apache>
   @type geoip
 
   # Specify one or more geoip lookup field which has ip address (default: host)
@@ -235,28 +218,35 @@ Note that filter version of geoip plugin does not have handling tag feature.
 
   # Set adding field with placeholder (more than one settings are required.)
   <record>
-    city            ${city["host"]}
-    latitude        ${latitude["host"]}
-    longitude       ${longitude["host"]}
-    country_code3   ${country_code3["host"]}
-    country         ${country_code["host"]}
-    country_name    ${country_name["host"]}
-    dma             ${dma_code["host"]}
-    area            ${area_code["host"]}
-    region          ${region["host"]}
+    latitude        ${location.latitude["host"]}
+    longitude       ${location.longitude["host"]}
+    country         ${country.iso_code["host"]}
+    country_name    ${country.names.en["host"]}
+    postal_code     ${postal.code["host"]}
+    region          ${subdivisions.0.iso_code["host"]}
+    region_name     ${subdivisions.0.names.en["host"]}
+    city            ${city.names.en["host"]}
   </record>
 
+  # Settings for tag
+  tag               geoip.${tag[1]}
+
   # To avoid get stacktrace error with `[null, null]` array for elasticsearch.
   skip_adding_null_record  true
 
-  # Set log_level for fluentd-v0.10.43 or earlier (default: warn)
-  log_level         info
-</filter>
+  # Set @log_level (default: warn)
+  @log_level         info
+
+  <buffer tag>
+    # Set buffering time (default: 0s)
+    flush_interval    1s
+  </buffer>
+</match>
 ```
 
 ## Tutorial
 
-### For GeoipOutput
+### For GeoipFilter
 
 #### configuration
 
@@ -265,24 +255,17 @@ Note that filter version of geoip plugin does not have handling tag feature.
   @type forward
 </source>
 
-<match test.geoip>
-  @type copy
-  <store>
-    @type stdout
-  </store>
-  <store>
-    @type    geoip
-    geoip_lookup_key  host
-    <record>
-      lat     ${latitude["host"]}
-      lon     ${longitude["host"]}
-      country ${country_code["host"]}
-    </record>
-    tag     debug.${tag[1]}
-  </store>
-</match>
+<filter test.geoip>
+  @type    geoip
+  geoip_lookup_key  host
+  <record>
+    city  ${city.names.en["host"]}
+    lat   ${location.latitude["host"]}
+    lon   ${location.longitude["host"]}
+  </record>
+</filter>
 
-<match debug.**>
+<match test.**>
   @type stdout
 </match>
 ```
@@ -295,14 +278,18 @@ $ echo '{"host":"66.102.9.80","message":"test"}' | fluent-cat test.geoip
 
 # check the result at stdout
 $ tail /var/log/td-agent/td-agent.log
-2013-08-04 16:21:32 +0900 test.geoip: {"host":"66.102.9.80","message":"test"}
-2013-08-04 16:21:32 +0900 debug.geoip: {"host":"66.102.9.80","message":"test","lat":37.4192008972168,"lon":-122.05740356445312,"country":"US"}
+2016-02-01 12:04:37 +0900 test.geoip: {"host":"66.102.9.80","message":"test","city":"Mountain View","lat":37.4192008972168,"lon":-122.05740356445312}
 ```
 
-For more details of geoip data format is described at the page below in section `GeoIP City Edition CSV Database Fields`.<br />
-http://dev.maxmind.com/geoip/legacy/csv/
+You can check geoip data format using [utils/dump.rb](https://github.com/okkez/fluent-plugin-geoip/utils/dump.rb).
 
-### For GeoipFilter
+```
+$ bundle exec ruby urils/dump.rb geoip2 66.102.3.80
+$ bundle exec ruby urils/dump.rb geoip2_compat 66.102.3.80
+$ bundle exec ruby urils/dump.rb geoip 66.102.3.80
+```
+
+### For GeoipOutput
 
 #### configuration
 
@@ -311,17 +298,24 @@ http://dev.maxmind.com/geoip/legacy/csv/
   @type forward
 </source>
 
-<filter test.geoip>
-  @type    geoip
-  geoip_lookup_key  host
-  <record>
-    city  ${city["host"]}
-    lat   ${latitude["host"]}
-    lon   ${longitude["host"]}
-  </record>
-</filter>
+<match test.geoip>
+  @type copy
+  <store>
+    @type stdout
+  </store>
+  <store>
+    @type    geoip
+    geoip_lookup_key  host
+    <record>
+      lat     ${location.latitude["host"]}
+      lon     ${location.longitude["host"]}
+      country ${country.iso_code["host"]}
+    </record>
+    tag     debug.${tag[1]}
+  </store>
+</match>
 
-<match test.**>
+<match debug.**>
   @type stdout
 </match>
 ```
@@ -334,37 +328,19 @@ $ echo '{"host":"66.102.9.80","message":"test"}' | fluent-cat test.geoip
 
 # check the result at stdout
 $ tail /var/log/td-agent/td-agent.log
-2016-02-01 12:04:37 +0900 test.geoip: {"host":"66.102.9.80","message":"test","city":"Mountain View","lat":37.4192008972168,"lon":-122.05740356445312}
+2013-08-04 16:21:32 +0900 test.geoip: {"host":"66.102.9.80","message":"test"}
+2013-08-04 16:21:32 +0900 debug.geoip: {"host":"66.102.9.80","message":"test","lat":37.4192008972168,"lon":-122.05740356445312,"country":"US"}
 ```
 
-For more details of geoip data format is described at the page below in section `GeoIP City Edition CSV Database Fields`.<br />
-http://dev.maxmind.com/geoip/legacy/csv/
-
-## Placeholders
-
-### GeoIP legacy
-
-Provides these placeholders for adding field of geolocate results.<br />
-For more example of geolocating, you can try these websites like [Geo IP Address View](http://www.geoipview.com/) or [View my IP information](http://www.geoiptool.com/en/).
-
-| placeholder attributes         | output example    | type         | note |
-|--------------------------------|-------------------|--------------|------|
-| ${city[lookup_field]}          | "Ithaca"          | varchar(255) |  -   |
-| ${latitude[lookup_field]}      | 42.4277992248535  | decimal      |  -   |
-| ${longitude[lookup_field]}     | -76.4981994628906 | decimal      |  -   |
-| ${country_code3[lookup_field]} | "USA"             | varchar(3)   |  -   |
-| ${country_code[lookup_field]}  | "US"              | varchar(2)   | A two-character ISO 3166-1 country code      |
-| ${country_name[lookup_field]}  | "United States"   | varchar(50)  |  -   |
-| ${dma_code[lookup_field]}      | 555               | unsigned int | **only for US**  |
-| ${area_code[lookup_field]}     | 607               | char(3)      | **only for US**  |
-| ${region[lookup_field]}        | "NY"              | char(2)      | A two character ISO-3166-2 or FIPS 10-4 code |
-
-Further more specification available at http://dev.maxmind.com/geoip/legacy/csv/#GeoIP_City_Edition_CSV_Database_Fields
+You can check geoip data format using [utils/dump.rb](https://github.com/okkez/fluent-plugin-geoip/utils/dump.rb).
 
-Related configurations:
+```
+$ bundle exec ruby urils/dump.rb geoip2 66.102.3.80
+$ bundle exec ruby urils/dump.rb geoip2_compat 66.102.3.80
+$ bundle exec ruby urils/dump.rb geoip 66.102.3.80
+```
 
-* `backend_library`: `geoip` (default)
-* `geoip_database`: path to your GeoLiteCity.dat
+## Placeholders
 
 ### GeoIP2
 
@@ -405,43 +381,139 @@ Related configurations:
 * `backend_library`: `geoip2_compat` or `geoip2_c`
 * `geoip2_database`: path to your GeoLite2-City.mmdb
 
+### GeoIP legacy
+
+Provides these placeholders for adding field of geolocate results.<br />
+For more example of geolocating, you can try these websites like [Geo IP Address View](http://www.geoipview.com/) or [View my IP information](http://www.geoiptool.com/en/).
+
+| placeholder attributes         | output example    | type         | note |
+|--------------------------------|-------------------|--------------|------|
+| ${city[lookup_field]}          | "Ithaca"          | varchar(255) |  -   |
+| ${latitude[lookup_field]}      | 42.4277992248535  | decimal      |  -   |
+| ${longitude[lookup_field]}     | -76.4981994628906 | decimal      |  -   |
+| ${country_code3[lookup_field]} | "USA"             | varchar(3)   |  -   |
+| ${country_code[lookup_field]}  | "US"              | varchar(2)   | A two-character ISO 3166-1 country code      |
+| ${country_name[lookup_field]}  | "United States"   | varchar(50)  |  -   |
+| ${dma_code[lookup_field]}      | 555               | unsigned int | **only for US**  |
+| ${area_code[lookup_field]}     | 607               | char(3)      | **only for US**  |
+| ${region[lookup_field]}        | "NY"              | char(2)      | A two character ISO-3166-2 or FIPS 10-4 code |
+
+Further more specification available at http://dev.maxmind.com/geoip/legacy/csv/#GeoIP_City_Edition_CSV_Database_Fields
+
+Related configurations:
+
+* `backend_library`: `geoip` (default)
+* `geoip_database`: path to your GeoLiteCity.dat
+
 ## Parameters
 
-### GeoipOutput
+### GeoipFilter
+
+Note that filter version of `geoip` plugin does not have handling `tag` feature.
+
+#### Plugin helpers
+
+* [compat_parameters](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-compat_parameters)
+* [inject](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-inject)
+
+See also [Filter Plugin Overview](https://docs.fluentd.org/v1.0/articles/filter-plugin-overview)
+
+#### Supported sections
+
+* [Inject section configurations](https://docs.fluentd.org/v1.0/articles/inject-section)
+
+#### Parameters
+
+[Plugin Common Paramteters](https://docs.fluentd.org/v1.0/articles/plugin-common-parameters)
+
+**geoip_database** (string) (optional)
+
+* Default value: bundled database `GeoLiteCity.dat`
+
+Path to GeoIP database file.
+
+**geoip2_database** (string) (optional)
+
+* Default value: bundled database `GeoLite2-City.mmdb`.
+
+Path to GeoIP2 database file.
+
+**geoip_lookup_key** (string) (optional)
+
+* Default value: `host`.
 
-* `include_tag_key` (default: false)
-* `tag_key`
+Specify one or more geoip lookup field which has IP address.
 
-Add original tag name into filtered record using SetTagKeyMixin.<br />
-Further details are written at http://docs.fluentd.org/articles/in_exec
+**skip_adding_null_record** (bool) (optional)
 
-* `skip_adding_null_record` (default: false)
+* Default value: `nil`
 
 Skip adding geoip fields when this valaues to `true`.
 On the case of getting nothing of GeoIP info (such as local IP), it will output the original record without changing anything.
 
-* `tag`
+**backend_library** (enum) (optional)
 
-On using this option with tag placeholder like `tag geoip.${tag}` (test code is available at [test_out_geoip.rb](https://github.com/y-ken/fluent-plugin-geoip/blob/master/test/plugin/test_out_geoip.rb)).
+* Available values: `geoip`, `geoip2_compat`, `geoip2_c`
+* Default value: `geoip2_c`.
 
-* `flush_interval` (default: 0 sec)
+Set backend library.
 
-Set buffering time to execute bulk lookup geoip.
+### GeoipOutput
 
-### GeoipFilter
+#### Plugin helpers
 
-Note that filter version of `geoip` plugin does not have handling `tag` feature.
+* [event_emitter](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-event_emitter)
+* [compat_parameters](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-compat_parameters)
+* [inject](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-inject)
+
+See also [Output Plugin Overview](https://docs.fluentd.org/v1.0/articles/output-plugin-overview)
+
+#### Sections
+
+* [Inject section configurations](https://docs.fluentd.org/v1.0/articles/inject-section)
+* [Buffer section configurations](https://docs.fluentd.org/v1.0/articles/buffer-section)
+
+#### Parameters
+
+[Plugin Common Paramteters](https://docs.fluentd.org/v1.0/articles/plugin-common-parameters)
+
+**geoip_database** (string) (optional)
+
+* Default value: bundled database `GeoLiteCity.dat`
+
+Path to GeoIP database file.
+
+**geoip2_database** (string) (optional)
 
-* `include_tag_key` (default: false)
+* Default value: bundled database `GeoLite2-City.mmdb`.
 
-Add original tag name into filtered record using SetTagKeyMixin.<br />
-Further details are written at http://docs.fluentd.org/articles/in_exec
+Path to GeoIP2 database file.
 
-* `skip_adding_null_record` (default: false)
+**geoip_lookup_key** (string) (optional)
+
+* Default value: `host`.
+
+Specify one or more geoip lookup field which has IP address.
+
+**skip_adding_null_record** (bool) (optional)
+
+* Default value: `nil`
 
 Skip adding geoip fields when this valaues to `true`.
 On the case of getting nothing of GeoIP info (such as local IP), it will output the original record without changing anything.
 
+**backend_library** (enum) (optional)
+
+* Available values: `geoip`, `geoip2_compat`, `geoip2_c`
+* Default value: `geoip2_c`.
+
+Set backend library.
+
+**tag** (string) (optional)
+
+On using this option with tag placeholder like `tag geoip.${tag}` (test code is available at [test_out_geoip.rb](https://github.com/y-ken/fluent-plugin-geoip/blob/master/test/plugin/test_out_geoip.rb)).
+
+
 ## Articles
 
 * [IPアドレスを元に位置情報をリアルタイムに付与する fluent-plugin-geoip v0.0.1をリリースしました #fluentd - Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-has-released)<br />
@@ -463,8 +535,6 @@ http://developer.smartnews.be/blog/2013/10/03/easy-data-analysis-using-fluentd-r
 
 Pull requests are very welcome!!
 
-* support [GeoIP2](http://dev.maxmind.com/geoip/geoip2/whats-new-in-geoip2/)
-
 ## Contributing
 
 1. Fork it

data/fluent-plugin-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-geoip"
-  spec.version       = "1.0.0"
+  spec.version       = "1.1.0"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd Filter plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}

data/lib/fluent/plugin/filter_geoip.rb

@@ -5,20 +5,19 @@ module Fluent::Plugin
   class GeoipFilter < Fluent::Plugin::Filter
     Fluent::Plugin.register_filter('geoip', self)
 
-    config_param :geoip_database, :string, default: File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
-    config_param :geoip2_database, :string, default: File.dirname(__FILE__) + '/../../../data/GeoLite2-City.mmdb'
+    helpers :compat_parameters, :inject
+
+    config_param :geoip_database, :string, default: File.expand_path('../../../data/GeoLiteCity.dat', __dir__)
+    config_param :geoip2_database, :string, default: File.expand_path('../../../data/GeoLite2-City.mmdb', __dir__)
     config_param :geoip_lookup_key, :string, default: 'host'
     config_param :skip_adding_null_record, :bool, default: false
 
-    config_set_default :include_tag_key, false
-
-    config_param :hostname_command, :string, default: 'hostname'
-
-    config_param :log_level, :string, default: 'warn'
+    config_set_default :@log_level, "warn"
 
     config_param :backend_library, :enum, list: Fluent::GeoIP::BACKEND_LIBRARIES, default: :geoip2_c
 
     def configure(conf)
+      compat_parameters_convert(conf, :inject)
       super
       @geoip = Fluent::GeoIP.new(self, conf)
     end
@@ -28,7 +27,12 @@ module Fluent::Plugin
       if filtered_record
         record = filtered_record
       end
+      record = inject_values_to_record(tag, time, record)
       record
     end
+
+    def multi_workers_ready?
+      true
+    end
   end
 end

data/lib/fluent/plugin/geoip.rb

@@ -56,7 +56,9 @@ module Fluent
               dummy_text = Yajl::Encoder.encode('dummy_text')
               Yajl::Parser.parse(v.gsub(REGEXP_PLACEHOLDER_SCAN, dummy_text))
             rescue Yajl::ParseError => e
-              raise Fluent::ConfigError, "geoip: failed to parse '#{v}' as json."
+              message = "geoip: failed to parse '#{v}' as json."
+              log.error message, error: e
+              raise Fluent::ConfigError, message
             end
           }
           validate_json.call if json?(v.tr('\'"\\', ''))
@@ -108,7 +110,6 @@ module Fluent
 
     def quoted_value?(text)
       # to improbe compatibility with fluentd v1-config
-      trim_quote = text[1..text.size-2]
       text.match(/(^'.+'$|^".+"$)/)
     end
 
@@ -151,7 +152,13 @@ module Fluent
         position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
         next if position.nil? or geodata[position[:record_key]].nil?
         keys = [position[:record_key]] + position[:geoip_key].split('.').map(&:to_sym)
-        placeholder[placeholder_key] = geodata.dig(*keys)
+        value = geodata.dig(*keys)
+        value = if [:latitude, :longitude].include?(keys.last)
+                  value || 0.0
+                else
+                  value
+                end
+        placeholder[placeholder_key] = value
       end
       placeholder
     end

data/lib/fluent/plugin/out_geoip.rb

@@ -6,23 +6,23 @@ class Fluent::Plugin::GeoipOutput < Fluent::Plugin::Output
 
   helpers :event_emitter, :inject, :compat_parameters
 
-  config_param :geoip_database, :string, default: File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
-  config_param :geoip2_database, :string, default: File.dirname(__FILE__) + '/../../../data/GeoLite2-City.mmdb'
+  config_param :geoip_database, :string, default: File.expand_path('../../../data/GeoLiteCity.dat', __dir__)
+  config_param :geoip2_database, :string, default: File.expand_path('../../../data/GeoLite2-City.mmdb', __dir__)
   config_param :geoip_lookup_key, :string, default: 'host'
   config_param :tag, :string, default: nil
   config_param :skip_adding_null_record, :bool, default: false
 
-  config_param :flush_interval, :time, default: 0
-  config_param :log_level, :string, default: 'warn'
+  config_set_default :@log_level, "warn"
 
   config_param :backend_library, :enum, list: Fluent::GeoIP::BACKEND_LIBRARIES, default: :geoip2_c
   config_section :buffer do
     config_set_default :@type, :memory
     config_set_default :chunk_keys, ['tag']
+    config_set_default :flush_interval, 0
   end
 
   def configure(conf)
-    compat_parameters_convert(conf, :buffer, default_chunk_key: 'tag')
+    compat_parameters_convert(conf, :buffer, :inject, default_chunk_key: 'tag')
     super
     raise Fluetn::ConfigError, "chunk key must include 'tag'" unless @chunk_key_tag
     placeholder_validate!(:tag, @tag) if @tag
@@ -48,4 +48,8 @@ class Fluent::Plugin::GeoipOutput < Fluent::Plugin::Output
     tag = extract_placeholders(@tag, chunk.metadata) if @tag
     router.emit_stream(tag, es)
   end
+
+  def multi_workers_ready?
+    true
+  end
 end

data/test/plugin/test_filter_geoip.rb

@@ -19,6 +19,7 @@ class GeoipFilterTest < Test::Unit::TestCase
 
   def filter(config, messages, syntax: :v1)
     d = create_driver(config, syntax: syntax)
+    yield d if block_given?
     d.run(default_tag: "input.access") {
       messages.each {|message|
         d.feed(@time, message)
@@ -27,6 +28,15 @@ class GeoipFilterTest < Test::Unit::TestCase
     d.filtered_records
   end
 
+  def setup_geoip_mock(d)
+    geoip = d.instance.instance_variable_get(:@geoip)
+    db = Object.new
+    def db.lookup(ip)
+      {}
+    end
+    geoip.instance_variable_set(:@geoip, db)
+  end
+
   sub_test_case "configure" do
     test "empty" do
       assert_nothing_raised {
@@ -392,6 +402,32 @@ class GeoipFilterTest < Test::Unit::TestCase
       filtered = filter(config, messages)
       assert_equal(expected, filtered)
     end
+
+    def test_filter_when_latitude_longitude_is_nil
+      config = %[
+        backend_library   geoip2_c
+        geoip_lookup_key  host
+        <record>
+          latitude  ${location.latitude['host']}
+          longitude ${location.longitude['host']}
+        </record>
+      ]
+      messages = [
+        { "host" => "180.94.85.84", "message" => "nil latitude and longitude" }
+      ]
+      expected = [
+        {
+          "host" => "180.94.85.84",
+          "message" => "nil latitude and longitude",
+          "latitude" => 0.0,
+          "longitude" => 0.0
+        }
+      ]
+      filtered = filter(config, messages) do |d|
+        setup_geoip_mock(d)
+      end
+      assert_equal(expected, filtered)
+    end
   end
 
   sub_test_case "geoip2_compat" do
@@ -661,6 +697,32 @@ class GeoipFilterTest < Test::Unit::TestCase
       filtered = filter(config, messages)
       assert_equal(expected, filtered)
     end
+
+    def test_filter_when_latitude_longitude_is_nil
+      config = %[
+        backend_library   geoip2_compat
+        geoip_lookup_key  host
+        <record>
+          latitude  ${latitude['host']}
+          longitude ${longitude['host']}
+        </record>
+      ]
+      messages = [
+        { "host" => "180.94.85.84", "message" => "nil latitude and longitude" }
+      ]
+      expected = [
+        {
+          "host" => "180.94.85.84",
+          "message" => "nil latitude and longitude",
+          "latitude" => 0.0,
+          "longitude" => 0.0
+        }
+      ]
+      filtered = filter(config, messages) do |d|
+        setup_geoip_mock(d)
+      end
+      assert_equal(expected, filtered)
+    end
   end
 
   sub_test_case "geoip legacy" do
@@ -1025,6 +1087,32 @@ class GeoipFilterTest < Test::Unit::TestCase
       filtered = filter(config, messages)
       assert_equal(expected, filtered)
     end
+
+    def test_filter_when_latitude_longitude_is_nil
+      config = %[
+        backend_library   geoip
+        geoip_lookup_key  host
+        <record>
+          latitude  ${latitude['host']}
+          longitude ${longitude['host']}
+        </record>
+      ]
+      messages = [
+        { "host" => "180.94.85.84", "message" => "nil latitude and longitude" }
+      ]
+      expected = [
+        {
+          "host" => "180.94.85.84",
+          "message" => "nil latitude and longitude",
+          "latitude" => 0.0,
+          "longitude" => 0.0
+        }
+      ]
+      filtered = filter(config, messages) do |d|
+        setup_geoip_mock(d)
+      end
+      assert_equal(expected, filtered)
+    end
   end
 end
 

data/test/plugin/test_out_geoip.rb

@@ -273,7 +273,7 @@ class GeoipOutputTest < Test::Unit::TestCase
       assert_equal 'Mountain View', events[0][2]['from_city']
       assert_equal 'United States', events[0][2]['from_country']
       assert_equal 37.419200000000004, events[0][2]['latitude']
-      assert_equal -122.0574, events[0][2]['longitude']
+      assert_equal(-122.0574, events[0][2]['longitude'])
       assert_equal '37.419200000000004,-122.0574', events[0][2]['float_concat']
       assert_equal [-122.0574, 37.419200000000004], events[0][2]['float_array']
       float_nest = {"lat" => 37.419200000000004, "lon" => -122.0574 }
@@ -559,7 +559,7 @@ class GeoipOutputTest < Test::Unit::TestCase
       assert_equal 'Mountain View', events[0][2]['from_city']
       assert_equal 'United States', events[0][2]['from_country']
       assert_equal 37.419200000000004, events[0][2]['latitude']
-      assert_equal -122.0574, events[0][2]['longitude']
+      assert_equal(-122.0574, events[0][2]['longitude'])
       assert_equal '37.419200000000004,-122.0574', events[0][2]['float_concat']
       assert_equal [-122.0574, 37.419200000000004], events[0][2]['float_array']
       float_nest = {"lat" => 37.419200000000004, "lon" => -122.0574 }
@@ -933,7 +933,7 @@ class GeoipOutputTest < Test::Unit::TestCase
       assert_equal 'Mountain View', events[0][2]['from_city']
       assert_equal 'United States', events[0][2]['from_country']
       assert_equal 37.4192008972168, events[0][2]['latitude']
-      assert_equal -122.05740356445312, events[0][2]['longitude']
+      assert_equal(-122.05740356445312, events[0][2]['longitude'])
       assert_equal '37.4192008972168,-122.05740356445312', events[0][2]['float_concat']
       assert_equal [-122.05740356445312, 37.4192008972168], events[0][2]['float_array']
       float_nest = {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }

data/utils/dump.rb

@@ -0,0 +1,27 @@
+#! /usr/bin/env ruby
+
+require "geoip"
+require "geoip2_compat"
+require "geoip2"
+require "pp"
+
+backend = ARGV[0]
+ip = ARGV[1]
+
+geoip_database = File.expand_path("../data/GeoLiteCity.dat", __dir__)
+geoip2_database = File.expand_path("../data/GeoLite2-City.mmdb", __dir__)
+
+geoip = GeoIP::City.new(geoip_database, :memory, false)
+geoip2_compat = GeoIP2Compat.new(geoip2_database)
+geoip2 = GeoIP2::Database.new(geoip2_database)
+
+record = case backend
+         when "geoip"
+           geoip.look_up(ip)
+         when "geoip2_compat"
+           geoip2_compat.lookup(ip)
+         when "geoip2"
+           geoip2.lookup(ip)
+         end
+
+pp record.to_h

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-geoip
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-19 00:00:00.000000000 Z
+date: 2017-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -175,6 +175,7 @@ files:
 - test/helper.rb
 - test/plugin/test_filter_geoip.rb
 - test/plugin/test_out_geoip.rb
+- utils/dump.rb
 homepage: https://github.com/y-ken/fluent-plugin-geoip
 licenses:
 - Apache-2.0
@@ -195,7 +196,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: Fluentd Filter plugin to add information about geographical location of IP