fluent-plugin-geoip 0.8.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd066ccaec269f0423567cb6b67fb2f740bd75bf
-  data.tar.gz: 59ceed4885348c00333ae21b859311474ed1a01e
+  metadata.gz: d8ae0d73eec8f27ed950ce080d02d0ada455fdc7
+  data.tar.gz: 3e442d004f0e4852ea8b713749eedb1f05991eaa
 SHA512:
-  metadata.gz: 7e1c15b06c1916d28656c6607e0d31c716622a0c1436a45d15eeab7d4cea049b9a728a87af7603693305399fa98180a63b43f954298feff9a7cbfa439bccf93d
-  data.tar.gz: 1699358cd3e9841ba975d07853a05a5af2ead21c42469fa892f1cce9917f08baabc711905011d6547d8bd10da00616e53b437ff4adb64dd0baddb99f0276c57b
+  metadata.gz: a026783f1b3578c27081c142dc4829cdda7ed0ec0be13867ee42e6052818fde41630154d8311fc63d52b6808213bec0caf84281006143cbc8f9845f96a3e01b6
+  data.tar.gz: 70836a38b6e1596ff5dcae93dda3961b67e7a16f4dd050b2ddaa7c33ff17df63475526e8baf2b8a8e103eb81cc250be029d0ed96d7388c871fcd7bfcd8e5f79e

data/.travis.yml

@@ -10,10 +10,6 @@ env:
     - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v1.0_with_backport_dig.gemfile
     - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v1.0_without_backport_dig.gemfile
     - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v1.0_without_backport_dig.gemfile
-    - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12_with_backport_dig.gemfile
-    - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12_with_backport_dig.gemfile
-    - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12_without_backport_dig.gemfile
-    - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12_without_backport_dig.gemfile
 
 before_install:
   #- sudo apt-get update

data/Appraisals

@@ -1,12 +1,3 @@
-appraise "fluentd v0.12 with backport_dig" do
-  gem "fluentd", "~> 0.12.0"
-  gem "backport_dig", "~> 1.0.0"
-end
-
-appraise "fluentd v0.12 without backport_dig" do
-  gem "fluentd", "~> 0.12.0"
-end
-
 appraise "fluentd v1.0 with backport_dig" do
   gem "fluentd", "~> 1.0.0"
   gem "backport_dig", "~> 1.0.0"

data/README.md

@@ -10,26 +10,84 @@ The accuracy details for GeoLite City (free) and GeoIP City (purchased) has desc
 * http://www.maxmind.com/en/geolite_city_accuracy ([lang:ja](http://www.maxmind.com/ja/geolite_city_accuracy))
 * http://www.maxmind.com/en/city_accuracy ([lang:ja](http://www.maxmind.com/ja/city_accuracy))
 
+## Requirements
+
+| fluent-plugin-geoip | fluentd    | ruby   |
+|---------------------|------------|--------|
+| >= 1.0.0            | >= v1.0.2  | >= 2.1 |
+| < 1.0.0             | >= v0.12.0 | >= 1.9 |
+
+If you want to use this plugin with Fluentd v0.12.x or earlier use 0.8.x.
+
+### Compatibility notice
+
+We've used Fluentd v1 API in this plugin since 1.0.0.
+So we have dropped some features.
+
+See also [official document](http://docs.fluentd.org/v1.0/articles/plugin-update-from-v12)
+
+#### Fluent::Mixin::RewriteTagName
+
+* `${tag}`, `__TAG__`
+
+    Alternative: Use `${tag}` placeholder
+
+* `${tag_parts[n]}`, `__TAG_PARTS[n]__`
+
+    Alternative: Use `${tag[n]}` placeholder
+
+* `${hostname}`, `__HOSTNAME__`
+
+    Alternative1: Use filter before this plugin and chunk keys:
+    ```
+    <filter>
+      @type record_transformer
+      <record>
+        hostname ${hostname}
+      </record>
+    </filter>
+    <match **>
+      @type geoip
+      tag geoip.${tag[1]}.${hostname}
+      <record>
+        city ${city["host"]}
+      </record>
+      <buffer tag, hostname>
+        flush_interval 1s
+      </buffer>
+    </match>
+    ```
+
+    Alternative2: Just inject hostname into record you can use `<inject>` section instead:
+    ```
+    <match **>
+      @type geoip
+      tag geoip.${tag[1]}.${hostname}
+      <record>
+        city ${city["host"]}
+      </record>
+      <inject>
+        hostname_key hostname
+      </inject>
+    </match>
+    ```
+
 ## Dependency
 
 before use, install dependent library as:
 
 ```bash
 # for RHEL/CentOS
-$ sudo yum groupinstall "Development Tools"
+$ sudo yum group install "Development Tools"
 $ sudo yum install geoip-devel --enablerepo=epel
-$ sudo yum install libmaxminddb-devel --enablerepo=epel
 
 # for Ubuntu/Debian
 $ sudo apt-get install build-essential
 $ sudo apt-get install libgeoip-dev
-$ sudo apt-get install libmaxminddb-dev
 
 # for OS X
 $ brew install geoip
-$ brew install libmaxminddb
 $ bundle config build.geoip-c --with-geoip-dir=/usr/local/include/
-$ bundle config build.geoip2_c --with-opt-include=/usr/local/include/
 ```
 
 ## Installation
@@ -61,6 +119,10 @@ $ sudo td-agent-gem install fluent-plugin-geoip
 
   # Specify optional geoip database (using bundled GeoLiteCity databse by default)
   geoip_database    "/path/to/your/GeoIPCity.dat"
+  # Specify optional geoip2 database
+  # geoip2_database   "/path/to/your/GeoLite2-City.mmdb"
+  # Specify backend library (geoip, geoip2_compat, geoip2_c)
+  backend_library geoip
 
   # Set adding field with placeholder (more than one settings are required.)
   <record>
@@ -76,8 +138,7 @@ $ sudo td-agent-gem install fluent-plugin-geoip
   </record>
 
   # Settings for tag
-  remove_tag_prefix access.
-  tag               geoip.${tag}
+  tag               geoip.${tag[1]}
 
   # To avoid get stacktrace error with `[null, null]` array for elasticsearch.
   skip_adding_null_record  true
@@ -85,8 +146,10 @@ $ sudo td-agent-gem install fluent-plugin-geoip
   # Set log_level for fluentd-v0.10.43 or earlier (default: warn)
   log_level         info
 
-  # Set buffering time (default: 0s)
-  flush_interval    1s
+  <buffer tag>
+    # Set buffering time (default: 0s)
+    flush_interval    1s
+  </buffer>
 </match>
 ```
 
@@ -100,8 +163,7 @@ $ sudo td-agent-gem install fluent-plugin-geoip
     user1_city      ${city["user1_host"]}
     user2_city      ${city["user2_host"]}
   </record>
-  remove_tag_prefix access.
-  tag               geoip.${tag}
+  tag               geoip.${tag[1]}
 </match>
 ```
 
@@ -109,6 +171,7 @@ $ sudo td-agent-gem install fluent-plugin-geoip
 
 It is a sample to get friendly geo point recdords for elasticsearch with Yajl (JSON) parser.<br />
 
+**Notice** v0 config will be deprecated in the future.
 
 ```
 <match access.apache>
@@ -127,15 +190,14 @@ It is a sample to get friendly geo point recdords for elasticsearch with Yajl (J
     # ex. [-122.05740356445312, 37.4192008972168]
     location_array       '[${longitude["host"]},${latitude["host"]}]'
   </record>
-  remove_tag_prefix      access.
-  tag                    geoip.${tag}
+  tag                    geoip.${tag[1]}
 
   # To avoid get stacktrace error with `[null, null]` array for elasticsearch.
   skip_adding_null_record  true
 </match>
 ```
 
-On the case of using td-agent2 (v1-config), it have to quote `{ ... }` or `[ ... ]` block with quotation like below.
+On the case of using td-agent3 (v1-config), it have to quote `{ ... }` or `[ ... ]` block with quotation like below.
 
 ```
 <match access.apache>
@@ -166,6 +228,10 @@ Note that filter version of geoip plugin does not have handling tag feature.
 
   # Specify optional geoip database (using bundled GeoLiteCity databse by default)
   geoip_database    "/path/to/your/GeoIPCity.dat"
+  # Specify optional geoip2 database
+  # geoip2_database   "/path/to/your/GeoLite2-City.mmdb"
+  # Specify backend library (geoip, geoip2_compat, geoip2_c)
+  backend_library geoip
 
   # Set adding field with placeholder (more than one settings are required.)
   <record>
@@ -185,9 +251,6 @@ Note that filter version of geoip plugin does not have handling tag feature.
 
   # Set log_level for fluentd-v0.10.43 or earlier (default: warn)
   log_level         info
-
-  # Set buffering time (default: 0s)
-  flush_interval    1s
 </filter>
 ```
 
@@ -215,8 +278,7 @@ Note that filter version of geoip plugin does not have handling tag feature.
       lon     ${longitude["host"]}
       country ${country_code["host"]}
     </record>
-    remove_tag_prefix test.
-    tag     debug.${tag}
+    tag     debug.${tag[1]}
   </store>
 </match>
 
@@ -299,6 +361,11 @@ For more example of geolocating, you can try these websites like [Geo IP Address
 
 Further more specification available at http://dev.maxmind.com/geoip/legacy/csv/#GeoIP_City_Edition_CSV_Database_Fields
 
+Related configurations:
+
+* `backend_library`: `geoip` (default)
+* `geoip_database`: path to your GeoLiteCity.dat
+
 ### GeoIP2
 
 You can get any fields in the
@@ -333,6 +400,11 @@ For example(geoip2_compat backend):
 
 **NOTE**: geoip2_compat backend supports only above fields.
 
+Related configurations:
+
+* `backend_library`: `geoip2_compat` or `geoip2_c`
+* `geoip2_database`: path to your GeoLite2-City.mmdb
+
 ## Parameters
 
 ### GeoipOutput
@@ -348,16 +420,9 @@ Further details are written at http://docs.fluentd.org/articles/in_exec
 Skip adding geoip fields when this valaues to `true`.
 On the case of getting nothing of GeoIP info (such as local IP), it will output the original record without changing anything.
 
-* `remove_tag_prefix`
-* `remove_tag_suffix`
-* `add_tag_prefix`
-* `add_tag_suffix`
-
-Set one or more option are required unless using `tag` option for editing tag name. (HandleTagNameMixin feature)
-
 * `tag`
 
-On using this option with tag placeholder like `tag geoip.${tag}` (test code is available at [test_out_geoip.rb](https://github.com/y-ken/fluent-plugin-geoip/blob/master/test/plugin/test_out_geoip.rb)), it will be overwrite after these options affected. which are remove_tag_prefix, remove_tag_suffix, add_tag_prefix and add_tag_suffix.
+On using this option with tag placeholder like `tag geoip.${tag}` (test code is available at [test_out_geoip.rb](https://github.com/y-ken/fluent-plugin-geoip/blob/master/test/plugin/test_out_geoip.rb)).
 
 * `flush_interval` (default: 0 sec)
 
@@ -377,10 +442,6 @@ Further details are written at http://docs.fluentd.org/articles/in_exec
 Skip adding geoip fields when this valaues to `true`.
 On the case of getting nothing of GeoIP info (such as local IP), it will output the original record without changing anything.
 
-* `flush_interval` (default: 0 sec)
-
-Set buffering time to execute bulk lookup geoip.
-
 ## Articles
 
 * [IPアドレスを元に位置情報をリアルタイムに付与する fluent-plugin-geoip v0.0.1をリリースしました #fluentd - Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-has-released)<br />

data/dockerfiles/Dockerfile-ruby2.1

@@ -2,7 +2,7 @@ FROM ruby:2.1-alpine
 
 ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
 
-RUN apk --no-cache --update add build-base automake autoconf libtool ruby-dev libc6-compat geoip-dev git
+RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
 
 WORKDIR /app
 COPY . .

data/dockerfiles/Dockerfile-ruby2.2

@@ -2,7 +2,7 @@ FROM ruby:2.2-alpine
 
 ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
 
-RUN apk --no-cache --update add build-base automake autoconf libtool ruby-dev libc6-compat geoip-dev git
+RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
 
 WORKDIR /app
 COPY . .

data/dockerfiles/Dockerfile-ruby2.3

@@ -2,7 +2,7 @@ FROM ruby:2.3-alpine
 
 ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
 
-RUN apk --no-cache --update add build-base automake autoconf libtool ruby-dev libc6-compat geoip-dev git
+RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
 
 WORKDIR /app
 COPY . .

data/dockerfiles/Dockerfile-ruby2.4

@@ -2,7 +2,7 @@ FROM ruby:2.4-alpine
 
 ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
 
-RUN apk --no-cache --update add build-base automake autoconf libtool ruby-dev libc6-compat geoip-dev git
+RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
 
 WORKDIR /app
 COPY . .

data/fluent-plugin-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-geoip"
-  spec.version       = "0.8.2"
+  spec.version       = "1.0.0"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd Filter plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}
@@ -21,10 +21,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "appraisal"
   spec.add_development_dependency "test-unit", ">= 3.1.0"
   spec.add_development_dependency "geoip2_compat"
-  spec.add_development_dependency "geoip2_c"
 
-  spec.add_runtime_dependency "fluentd"
-  spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
+  spec.add_runtime_dependency "fluentd", [">= 0.14.8", "< 2"]
   spec.add_runtime_dependency "geoip-c"
+  spec.add_runtime_dependency "geoip2_c"
   spec.add_runtime_dependency "dig_rb"
 end

data/lib/fluent/plugin/filter_geoip.rb

@@ -1,40 +1,34 @@
+require 'fluent/plugin/filter'
 require 'fluent/plugin/geoip'
 
-module Fluent
-  class GeoipFilter < Filter
-    Plugin.register_filter('geoip', self)
+module Fluent::Plugin
+  class GeoipFilter < Fluent::Plugin::Filter
+    Fluent::Plugin.register_filter('geoip', self)
 
-    config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
-    config_param :geoip2_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLite2-City.mmdb'
-    config_param :geoip_lookup_key, :string, :default => 'host'
-    config_param :skip_adding_null_record, :bool, :default => false
+    config_param :geoip_database, :string, default: File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
+    config_param :geoip2_database, :string, default: File.dirname(__FILE__) + '/../../../data/GeoLite2-City.mmdb'
+    config_param :geoip_lookup_key, :string, default: 'host'
+    config_param :skip_adding_null_record, :bool, default: false
 
-    include Fluent::SetTagKeyMixin
     config_set_default :include_tag_key, false
 
-    config_param :hostname_command, :string, :default => 'hostname'
+    config_param :hostname_command, :string, default: 'hostname'
 
-    config_param :log_level, :string, :default => 'warn'
+    config_param :log_level, :string, default: 'warn'
 
-    config_param :backend_library, :enum, :list => Fluent::GeoIP::BACKEND_LIBRARIES, :default => :geoip
+    config_param :backend_library, :enum, list: Fluent::GeoIP::BACKEND_LIBRARIES, default: :geoip2_c
 
     def configure(conf)
       super
       @geoip = Fluent::GeoIP.new(self, conf)
     end
 
-    def filter_stream(tag, es)
-      new_es = MultiEventStream.new
-      es.each do |time, record|
-        begin
-          filter_record(tag, time, record)
-          filtered_record = @geoip.add_geoip_field(record)
-          new_es.add(time, filtered_record) if filtered_record
-        rescue => e
-          router.emit_error_event(tag, time, record, e)
-        end
+    def filter(tag, time, record)
+      filtered_record = @geoip.add_geoip_field(record)
+      if filtered_record
+        record = filtered_record
       end
-      new_es
+      record
     end
   end
 end

data/lib/fluent/plugin/geoip.rb

@@ -77,13 +77,6 @@ module Fluent
         end
       end
 
-      if plugin.is_a?(Fluent::BufferedOutput)
-        @placeholder_expander = PlaceholderExpander.new
-        unless have_tag_option?(plugin)
-          raise Fluent::ConfigError, "geoip: required at least one option of 'tag', 'remove_tag_prefix', 'remove_tag_suffix', 'add_tag_prefix', 'add_tag_suffix'."
-        end
-      end
-
       @geoip = load_database(plugin)
     end
 
@@ -109,12 +102,6 @@ module Fluent
 
     private
 
-    def have_tag_option?(plugin)
-      plugin.tag ||
-        plugin.remove_tag_prefix || plugin.remove_tag_suffix ||
-        plugin.add_tag_prefix || plugin.add_tag_suffix
-    end
-
     def json?(text)
       text.match(/^\[.+\]$/) || text.match(/^\{.+\}$/)
     end
@@ -129,7 +116,7 @@ module Fluent
       begin
         return Yajl::Parser.parse(message)
       rescue Yajl::ParseError => e
-        log.info "geoip: failed to parse '#{message}' as json.", :error_class => e.class, :error => e.message
+        log.info "geoip: failed to parse '#{message}' as json.", error_class: e.class, error: e.message
         return nil
       end
     end
@@ -164,13 +151,7 @@ module Fluent
         position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
         next if position.nil? or geodata[position[:record_key]].nil?
         keys = [position[:record_key]] + position[:geoip_key].split('.').map(&:to_sym)
-        value = geodata.dig(*keys)
-        value = if [:latitude, :longitude].include?(keys.last)
-                  value || 0.0
-                else
-                  value
-                end
-        placeholder[placeholder_key] = value
+        placeholder[placeholder_key] = geodata.dig(*keys)
       end
       placeholder
     end

data/lib/fluent/plugin/out_geoip.rb

@@ -1,54 +1,51 @@
-require 'fluent/mixin/rewrite_tag_name'
+require 'fluent/plugin/output'
 require 'fluent/plugin/geoip'
 
-class Fluent::GeoipOutput < Fluent::BufferedOutput
+class Fluent::Plugin::GeoipOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('geoip', self)
 
-  config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
-  config_param :geoip2_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLite2-City.mmdb'
-  config_param :geoip_lookup_key, :string, :default => 'host'
-  config_param :tag, :string, :default => nil
-  config_param :skip_adding_null_record, :bool, :default => false
+  helpers :event_emitter, :inject, :compat_parameters
 
-  include Fluent::HandleTagNameMixin
-  include Fluent::SetTagKeyMixin
-  config_set_default :include_tag_key, false
+  config_param :geoip_database, :string, default: File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
+  config_param :geoip2_database, :string, default: File.dirname(__FILE__) + '/../../../data/GeoLite2-City.mmdb'
+  config_param :geoip_lookup_key, :string, default: 'host'
+  config_param :tag, :string, default: nil
+  config_param :skip_adding_null_record, :bool, default: false
 
-  include Fluent::Mixin::RewriteTagName
-  config_param :hostname_command, :string, :default => 'hostname'
+  config_param :flush_interval, :time, default: 0
+  config_param :log_level, :string, default: 'warn'
 
-  config_param :flush_interval, :time, :default => 0
-  config_param :log_level, :string, :default => 'warn'
-
-  config_param :backend_library, :enum, :list => Fluent::GeoIP::BACKEND_LIBRARIES, :default => :geoip
+  config_param :backend_library, :enum, list: Fluent::GeoIP::BACKEND_LIBRARIES, default: :geoip2_c
+  config_section :buffer do
+    config_set_default :@type, :memory
+    config_set_default :chunk_keys, ['tag']
+  end
 
   def configure(conf)
+    compat_parameters_convert(conf, :buffer, default_chunk_key: 'tag')
     super
-    Fluent::GeoIP.class_eval do
-      include Fluent::Mixin::RewriteTagName
-    end
+    raise Fluetn::ConfigError, "chunk key must include 'tag'" unless @chunk_key_tag
+    placeholder_validate!(:tag, @tag) if @tag
     @geoip = Fluent::GeoIP.new(self, conf)
   end
 
-  def start
-    super
-  end
-
   def format(tag, time, record)
+    record = inject_values_to_record(tag, time, record)
     [tag, time, record].to_msgpack
   end
 
-  def shutdown
-    super
+  def formatted_to_msgpack_binary
+    true
   end
 
   def write(chunk)
     es = Fluent::MultiEventStream.new
     tag = ""
-    chunk.msgpack_each do |_tag, time, record|
+    chunk.each do |_tag, time, record|
       tag = _tag
       es.add(time, @geoip.add_geoip_field(record))
     end
+    tag = extract_placeholders(@tag, chunk.metadata) if @tag
     router.emit_stream(tag, es)
   end
 end