fluent-plugin-geoip 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8fdb172feeffd90a77f0a83f3f44b58b88ddb138
-  data.tar.gz: 3abc009ded4a7ced18e046c27d9f50094f11b865
+  metadata.gz: 6064156b8f481d751a7f1ee92c5e80b189c52f2c
+  data.tar.gz: 95e5509ad5ec16ff48e9d71687caa205bd4dbf85
 SHA512:
-  metadata.gz: c5ef5143cdd8f2ea9fe372d5a2e0ee5bbe5981a4a1c2cd84b34364fe7f6679a897b5a40732d8b97ad9dfdf5f36b3dc257ec529357c2411e817cfdb34822e05f0
-  data.tar.gz: 70d27191a5cc8f7f4ee5153845149f7d0881c5bf08ed7554ab5424a65f7806b78f36909fa8319d1850d49c13d2247fa41f466cf954777e85d80d45065948eedb
+  metadata.gz: c3fcd73dcac19ab974974099f05147e1ba082cf339b0c45ae00971049cadf94cf6d8c1a438351c0c08cd7a51f5fcba03c01013760ff21422275757cc7adcd270
+  data.tar.gz: f6d8bb596b0c423e041d69f520cbb0fd6268c7a36bf7910e32b1a189fe2c5da560a6c41f2f95fa5a93b83c70cce1edf999994bac65f28a0f88b7f0fdc01e3bd6

data/.travis.yml

@@ -1,26 +1,37 @@
 sudo: required
-language: ruby
+services:
+  - docker
 
-rvm:
-  - 2.3.0
-  - 2.2
-  - 2.1
-  - 2.0.0
-  - 1.9.3
+env:
+  global:
+    DOCKER_COMPOSE_VERSION: 1.8.1
+  matrix:
+    - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/Gemfile
+    - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/Gemfile
+    - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/Gemfile
+    - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/Gemfile
+    - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
+    - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
+    - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
+    - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
 
 before_install:
-  - sudo apt-get update
-  - sudo apt-get install libgeoip-dev
-  - gem update bundler
+  #- sudo apt-get update
+  #- sudo apt-get install -y docker-engine
+  - sudo rm /usr/local/bin/docker-compose
+  - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m) > docker-compose
+  - chmod +x docker-compose
+  - sudo mv docker-compose /usr/local/bin/
+  - test -f data/GeoLite2-City.mmdb || curl http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz | gunzip > data/GeoLite2-City.mmdb
 
-gemfile:
-  - Gemfile
-  - gemfiles/fluentd_v0.10.gemfile
-  - gemfiles/fluentd_v0.12.gemfile
+install: true
 
-matrix:
-  fast_finish: true
-  allow_failures:
-    - gemfile: gemfiles/fluentd_v0.10.gemfile
-    - rvm: 1.9.3
-    - rvm: 2.0.0
+before_script:
+  - echo BUNDLE_GEMFILE=${BUNDLE_GEMFILE} > env
+  - docker-compose build test-ruby${TARGET_RUBY_VERSION}
+  - docker-compose ps
+  - docker-compose up -d test-ruby${TARGET_RUBY_VERSION}
+  - docker-compose exec test-ruby${TARGET_RUBY_VERSION} bundle install
+
+script:
+  - docker-compose exec test-ruby${TARGET_RUBY_VERSION} bundle exec rake test

data/Appraisals

@@ -1,7 +1,3 @@
-appraise "fluentd v0.10" do
-  gem "fluentd", "~> 0.10.46"
-end
-
 appraise "fluentd v0.12" do
   gem "fluentd", "~> 0.12.0"
 end

data/README.md

@@ -18,13 +18,16 @@ before use, install dependent library as:
 # for RHEL/CentOS
 $ sudo yum group install "Development Tools"
 $ sudo yum install geoip-devel --enablerepo=epel
+$ sudo yum install libmaxminddb-devel --enablerepo=epel
 
 # for Ubuntu/Debian
 $ sudo apt-get install build-essential
 $ sudo apt-get install libgeoip-dev
+$ sudo apt-get install libmaxminddb-dev
 
 # for OS X
 $ brew install geoip
+$ brew install libmaxminddb
 ```
 
 ## Installation
@@ -48,7 +51,7 @@ $ sudo td-agent-gem install fluent-plugin-geoip
 
 ```xml
 <match access.apache>
-  type geoip
+  @type geoip
 
   # Specify one or more geoip lookup field which has ip address (default: host)
   # in the case of accessing nested value, delimit keys by dot like 'host.ip'.
@@ -89,7 +92,7 @@ $ sudo td-agent-gem install fluent-plugin-geoip
 
 ```xml
 <match access.apache>
-  type geoip
+  @type geoip
   geoip_lookup_key  user1_host, user2_host
   <record>
     user1_city      ${city["user1_host"]}
@@ -107,12 +110,12 @@ It is a sample to get friendly geo point recdords for elasticsearch with Yajl (J
 
 ```
 <match access.apache>
-  type                   geoip
+  @type                  geoip
   geoip_lookup_key       host
   <record>
     # lat lon as properties
     # ex. {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
-    location_properties  { "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }
+    location_properties  '{ "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }'
   
     # lat lon as string
     # ex. "37.4192008972168,-122.05740356445312"
@@ -120,7 +123,7 @@ It is a sample to get friendly geo point recdords for elasticsearch with Yajl (J
     
     # GeoJSON (lat lon as array) is useful for Kibana's bettermap.
     # ex. [-122.05740356445312, 37.4192008972168]
-    location_array       [${longitude["host"]},${latitude["host"]}]
+    location_array       '[${longitude["host"]},${latitude["host"]}]'
   </record>
   remove_tag_prefix      access.
   tag                    geoip.${tag}
@@ -134,7 +137,7 @@ On the case of using td-agent2 (v1-config), it have to quote `{ ... }` or `[ ...
 
 ```
 <match access.apache>
-  type                   geoip
+  @type                  geoip
   geoip_lookup_key       host
   <record>
     location_properties  '{ "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }'
@@ -194,16 +197,16 @@ Note that filter version of geoip plugin does not have handling tag feature.
 
 ```xml
 <source>
-  type forward
+  @type forward
 </source>
 
 <match test.geoip>
-  type copy
+  @type copy
   <store>
-    type stdout
+    @type stdout
   </store>
   <store>
-    type    geoip
+    @type    geoip
     geoip_lookup_key  host
     <record>
       lat     ${latitude["host"]}
@@ -216,7 +219,7 @@ Note that filter version of geoip plugin does not have handling tag feature.
 </match>
 
 <match debug.**>
-  type stdout
+  @type stdout
 </match>
 ```
 
@@ -275,6 +278,8 @@ http://dev.maxmind.com/geoip/legacy/csv/
 
 ## Placeholders
 
+### GeoIP legacy
+
 Provides these placeholders for adding field of geolocate results.<br />
 For more example of geolocating, you can try these websites like [Geo IP Address View](http://www.geoipview.com/) or [View my IP information](http://www.geoiptool.com/en/).
 
@@ -292,6 +297,40 @@ For more example of geolocating, you can try these websites like [Geo IP Address
 
 Further more specification available at http://dev.maxmind.com/geoip/legacy/csv/#GeoIP_City_Edition_CSV_Database_Fields
 
+### GeoIP2
+
+You can get any fields in the
+[GeoLite2](http://dev.maxmind.com/geoip/geoip2/geolite2/) database and
+[GeoIP2 Downloadable Databases](http://dev.maxmind.com/geoip/geoip2/downloadable/).
+
+For example(geoip2_c backend):
+
+| placeholder attributes                   | output example     | note |
+|------------------------------------------|--------------------|------|
+| ${city.names.en[lookup_field]}           | "Mountain View"    | -    |
+| ${location.latitude[lookup_field]}       | 37.419200000000004 | -    |
+| ${location.longitude[lookup_field]}      | -122.0574          | -    |
+| ${country.iso_code[lookup_field]}        | "US"               | -    |
+| ${country.names.en[lookup_field]}        | "United States"    | -    |
+| ${postal.code[lookup_field]}             | "94043"            | -    |
+| ${subdivisions.0.iso_code[lookup_field]} | "CA"               | -    |
+| ${subdivisions.0.names.en[lookup_field]} | "California"       | -    |
+
+For example(geoip2_compat backend):
+
+| placeholder attributes        | output example     | note |
+|-------------------------------|--------------------|------|
+| ${city[lookup_field]}         | "Mountain View"    | -    |
+| ${latitude[lookup_field]}     | 37.419200000000004 | -    |
+| ${longitude[lookup_field]}    | -122.0574          | -    |
+| ${country_code[lookup_field]} | "US"               | -    |
+| ${country_name[lookup_field]} | "United States"    | -    |
+| ${postal_code[lookup_field]}  | "94043"            |      |
+| ${region[lookup_field]}       | "CA"               | -    |
+| ${region_name[lookup_field]}  | "California"       | -    |
+
+**NOTE**: geoip2_compat backend supports only above fields.
+
 ## Parameters
 
 ### GeoipOutput

data/docker-compose.yml

@@ -0,0 +1,31 @@
+version: "2"
+services:
+  test-ruby2.4:
+    build:
+      context: .
+      dockerfile: dockerfiles/Dockerfile-ruby2.4
+    env_file:
+      - env
+    command: ruby -run -e httpd .
+  test-ruby2.3:
+    build:
+      context: .
+      dockerfile: dockerfiles/Dockerfile-ruby2.3
+    env_file:
+      - env
+    command: ruby -run -e httpd .
+  test-ruby2.2:
+    build:
+      context: .
+      dockerfile: dockerfiles/Dockerfile-ruby2.2
+    env_file:
+      - env
+    command: ruby -run -e httpd .
+  test-ruby2.1:
+    build:
+      context: .
+      dockerfile: dockerfiles/Dockerfile-ruby2.1
+    env_file:
+      - env
+    command: ruby -run -e httpd .
+

data/dockerfiles/Dockerfile-ruby2.1

@@ -0,0 +1,8 @@
+FROM ruby:2.1-alpine
+
+ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
+
+RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
+
+WORKDIR /app
+COPY . .

data/dockerfiles/Dockerfile-ruby2.2

@@ -0,0 +1,8 @@
+FROM ruby:2.2-alpine
+
+ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
+
+RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
+
+WORKDIR /app
+COPY . .

data/dockerfiles/Dockerfile-ruby2.3

@@ -0,0 +1,8 @@
+FROM ruby:2.3-alpine
+
+ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
+
+RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
+
+WORKDIR /app
+COPY . .

data/dockerfiles/Dockerfile-ruby2.4

@@ -0,0 +1,8 @@
+FROM ruby:2.4-alpine
+
+ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
+
+RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
+
+WORKDIR /app
+COPY . .

data/fluent-plugin-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-geoip"
-  spec.version       = "0.6.1"
+  spec.version       = "0.7.0"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd Filter plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}
@@ -20,9 +20,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake"
   spec.add_development_dependency "appraisal"
   spec.add_development_dependency "test-unit", ">= 3.1.0"
+  spec.add_development_dependency "geoip2_compat"
+  spec.add_development_dependency "geoip2_c"
 
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
   spec.add_runtime_dependency "geoip-c"
+  spec.add_runtime_dependency "dig_rb"
 end
-

data/lib/fluent/plugin/filter_geoip.rb

@@ -1,8 +1,11 @@
+require 'fluent/plugin/geoip'
+
 module Fluent
   class GeoipFilter < Filter
     Plugin.register_filter('geoip', self)
 
     config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
+    config_param :geoip2_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLite2-City.mmdb'
     config_param :geoip_lookup_key, :string, :default => 'host'
     config_param :skip_adding_null_record, :bool, :default => false
 
@@ -13,12 +16,7 @@ module Fluent
     config_param :flush_interval, :time, :default => 0
     config_param :log_level, :string, :default => 'warn'
 
-
-    def initialize
-      require 'fluent/plugin/geoip'
-
-      super
-    end
+    config_param :backend_library, :enum, :list => Fluent::GeoIP::BACKEND_LIBRARIES, :default => :geoip
 
     def configure(conf)
       super

data/lib/fluent/plugin/geoip.rb

@@ -1,12 +1,16 @@
 require 'geoip'
 require 'yajl'
+require 'dig_rb'
 
 module Fluent
   class GeoIP
+    BACKEND_LIBRARIES = [:geoip, :geoip2_compat, :geoip2_c]
+
     REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<geoip_key>-?[^\[]+)\[['"](?<record_key>-?[^'"]+)['"]\]\}$/
     REGEXP_PLACEHOLDER_SCAN = /['"]?(\$\{[^\}]+?\})['"]?/
 
     GEOIP_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
+    GEOIP2_COMPAT_KEYS = %w(city country_code country_name latitude longitude postal_code region region_name)
 
     attr_reader :log
 
@@ -25,7 +29,7 @@ module Fluent
           if record_key.nil?
             raise Fluent::ConfigError, "geoip: missing value found at '#{key} #{lookup_field}'"
           end
-          @map.store(record_key, "${#{geoip_key}['#{lookup_field}']}")
+          @map[record_key] = "${#{geoip_key}['#{lookup_field}']}"
         end
       end
       if conf.keys.select{|k| k =~ /^enable_key_/}.size > 0
@@ -53,7 +57,16 @@ module Fluent
       @placeholder_keys = @map.values.join.scan(REGEXP_PLACEHOLDER_SCAN).map{ |placeholder| placeholder[0] }.uniq
       @placeholder_keys.each do |key|
         geoip_key = key.match(REGEXP_PLACEHOLDER_SINGLE)[:geoip_key]
-        raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
+        case plugin.backend_library
+        when :geoip
+          raise Fluent::ConfigError, "#{plugin.backend_library}: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
+        when :geoip2_compat
+          raise Fluent::ConfigError, "#{plugin.backend_library}: unsupported key #{geoip_key}" unless GEOIP2_COMPAT_KEYS.include?(geoip_key)
+        when :geoip2_c
+          # Nothing to do.
+          # We cannot define supported key(s) before we fetch values from GeoIP2 database
+          # because geoip2_c can fetch any fields in GeoIP2 database.
+        end
       end
 
       if plugin.is_a?(Fluent::BufferedOutput)
@@ -63,7 +76,7 @@ module Fluent
         end
       end
 
-      @geoip = ::GeoIP::City.new(plugin.geoip_database, :memory, false)
+      @geoip = load_database(plugin)
     end
 
     def add_geoip_field(record)
@@ -81,7 +94,7 @@ module Fluent
         else
           rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN, placeholder)
         end
-        record.store(record_key, rewrited)
+        record[record_key] = rewrited
       end
       return record
     end
@@ -116,25 +129,25 @@ module Fluent
     def get_address(record)
       address = {}
       @geoip_lookup_key.each do |field|
-        address.store(field, record[field]); next if not record[field].nil?
-        key = field.split('.')
-        obj = record
-        key.each {|k|
-          break obj = nil if not obj.has_key?(k)
-          obj = obj[k]
-        }
-        address.store(field, obj)
+        address[field] = record[field] || record.dig(*field.split('.'))
       end
-      return address
+      address
     end
 
     def geolocate(addresses)
       geodata = {}
       addresses.each do |field, ip|
-        geo = ip.nil? ? nil : @geoip.look_up(ip)
-        geodata.store(field, geo)
+        geo = nil
+        if ip
+          geo = if @geoip.respond_to?(:look_up)
+                  @geoip.look_up(ip)
+                else
+                  @geoip.lookup(ip)
+                end
+        end
+        geodata[field] = geo
       end
-      return geodata
+      geodata
     end
 
     def create_placeholder(geodata)
@@ -142,9 +155,25 @@ module Fluent
       @placeholder_keys.each do |placeholder_key|
         position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
         next if position.nil? or geodata[position[:record_key]].nil?
-        placeholder.store(placeholder_key, geodata[position[:record_key]][position[:geoip_key].to_sym])
+        keys = [position[:record_key]] + position[:geoip_key].split('.').map(&:to_sym)
+        placeholder[placeholder_key] = geodata.dig(*keys)
+      end
+      placeholder
+    end
+
+    def load_database(plugin)
+      case plugin.backend_library
+      when :geoip
+        ::GeoIP::City.new(plugin.geoip_database, :memory, false)
+      when :geoip2_compat
+        require 'geoip2_compat'
+        GeoIP2Compat.new(plugin.geoip2_database)
+      when :geoip2_c
+        require 'geoip2'
+        GeoIP2::Database.new(plugin.geoip2_database)
       end
-      return placeholder
+    rescue LoadError
+      raise Fluent::ConfigError, "You must install #{plugin.backend_library} gem."
     end
   end
 end