fluent-plugin-geoip 1.0.0 → 1.3.2

data/docker-compose.yml

@@ -1,31 +1,38 @@
-version: "2"
+version: "3"
 services:
+  test-ruby2.5:
+    build:
+      context: .
+      dockerfile: dockerfiles/Dockerfile-ruby2.5
+    env_file:
+      - env
+    command: tail -f /dev/null
   test-ruby2.4:
     build:
       context: .
       dockerfile: dockerfiles/Dockerfile-ruby2.4
     env_file:
       - env
-    command: ruby -run -e httpd .
+    command: tail -f /dev/null
   test-ruby2.3:
     build:
       context: .
       dockerfile: dockerfiles/Dockerfile-ruby2.3
     env_file:
       - env
-    command: ruby -run -e httpd .
+    command: tail -f /dev/null
   test-ruby2.2:
     build:
       context: .
       dockerfile: dockerfiles/Dockerfile-ruby2.2
     env_file:
       - env
-    command: ruby -run -e httpd .
+    command: tail -f /dev/null
   test-ruby2.1:
     build:
       context: .
       dockerfile: dockerfiles/Dockerfile-ruby2.1
     env_file:
       - env
-    command: ruby -run -e httpd .
+    command: tail -f /dev/null
 

data/dockerfiles/Dockerfile-ruby2.1

@@ -2,7 +2,7 @@ FROM ruby:2.1-alpine
 
 ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
 
-RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
+RUN apk --no-cache --update add build-base automake autoconf libtool ruby-dev libc6-compat geoip-dev git
 
 WORKDIR /app
 COPY . .

data/dockerfiles/Dockerfile-ruby2.2

@@ -2,7 +2,7 @@ FROM ruby:2.2-alpine
 
 ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
 
-RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
+RUN apk --no-cache --update add build-base automake autoconf libtool ruby-dev libc6-compat geoip-dev git
 
 WORKDIR /app
 COPY . .

data/dockerfiles/Dockerfile-ruby2.3

@@ -2,7 +2,7 @@ FROM ruby:2.3-alpine
 
 ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
 
-RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
+RUN apk --no-cache --update add build-base automake autoconf libtool ruby-dev libc6-compat geoip-dev git
 
 WORKDIR /app
 COPY . .

data/dockerfiles/Dockerfile-ruby2.4

@@ -2,7 +2,7 @@ FROM ruby:2.4-alpine
 
 ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
 
-RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
+RUN apk --no-cache --update add build-base automake autoconf libtool ruby-dev libc6-compat geoip-dev git
 
 WORKDIR /app
 COPY . .

data/dockerfiles/Dockerfile-ruby2.5

@@ -0,0 +1,8 @@
+FROM ruby:2.5-alpine
+
+ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
+
+RUN apk --no-cache --update add build-base automake autoconf libtool ruby-dev libc6-compat geoip-dev git
+
+WORKDIR /app
+COPY . .

data/fluent-plugin-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-geoip"
-  spec.version       = "1.0.0"
+  spec.version       = "1.3.2"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd Filter plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}
@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake"
   spec.add_development_dependency "appraisal"
   spec.add_development_dependency "test-unit", ">= 3.1.0"
+  spec.add_development_dependency "test-unit-rr"
   spec.add_development_dependency "geoip2_compat"
 
   spec.add_runtime_dependency "fluentd", [">= 0.14.8", "< 2"]

data/lib/fluent/plugin/filter_geoip.rb

@@ -1,34 +1,219 @@
 require 'fluent/plugin/filter'
-require 'fluent/plugin/geoip'
+
+require 'geoip'
+require 'yajl'
+unless {}.respond_to?(:dig)
+  begin
+    # backport_dig is faster than dig_rb so prefer backport_dig.
+    # And Fluentd v1.0.1 uses backport_dig
+    require 'backport_dig'
+  rescue LoadError
+    require 'dig_rb'
+  end
+end
 
 module Fluent::Plugin
   class GeoipFilter < Fluent::Plugin::Filter
     Fluent::Plugin.register_filter('geoip', self)
 
-    config_param :geoip_database, :string, default: File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
-    config_param :geoip2_database, :string, default: File.dirname(__FILE__) + '/../../../data/GeoLite2-City.mmdb'
-    config_param :geoip_lookup_key, :string, default: 'host'
-    config_param :skip_adding_null_record, :bool, default: false
+    BACKEND_LIBRARIES = [:geoip, :geoip2_compat, :geoip2_c]
+
+    REGEXP_PLACEHOLDER_SINGLE = /^\$\{
+                                  (?<geoip_key>-?[^\[\]]+)
+                                    \[
+                                      (?:(?<dq>")|(?<sq>'))
+                                        (?<record_key>-?(?(<dq>)[^"{}]+|[^'{}]+))
+                                      (?(<dq>)"|')
+                                    \]
+                                  \}$/x
+    REGEXP_PLACEHOLDER_SCAN = /['"]?(\$\{[^\}]+?\})['"]?/
 
-    config_set_default :include_tag_key, false
+    GEOIP_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
+    GEOIP2_COMPAT_KEYS = %w(city country_code country_name latitude longitude postal_code region region_name)
 
-    config_param :hostname_command, :string, default: 'hostname'
+    helpers :compat_parameters, :inject, :record_accessor
 
-    config_param :log_level, :string, default: 'warn'
+    config_param :geoip_database, :string, default: File.expand_path('../../../data/GeoLiteCity.dat', __dir__)
+    config_param :geoip2_database, :string, default: File.expand_path('../../../data/GeoLite2-City.mmdb', __dir__)
+    config_param :geoip_lookup_keys, :array, value_type: :string, default: ["host"]
+    config_param :geoip_lookup_key, :string, default: nil, deprecated: "Use geoip_lookup_keys instead"
+    config_param :skip_adding_null_record, :bool, default: false
 
-    config_param :backend_library, :enum, list: Fluent::GeoIP::BACKEND_LIBRARIES, default: :geoip2_c
+    config_set_default :@log_level, "warn"
+
+    config_param :backend_library, :enum, list: BACKEND_LIBRARIES, default: :geoip2_c
 
     def configure(conf)
+      compat_parameters_convert(conf, :inject)
       super
-      @geoip = Fluent::GeoIP.new(self, conf)
+
+      @map = {}
+      if @geoip_lookup_key
+        @geoip_lookup_keys = @geoip_lookup_key.split(/\s*,\s*/)
+      end
+
+      @geoip_lookup_keys.each do |key|
+        if key.include?(".") && !key.start_with?("$")
+          $log.warn("#{key} is not treated as nested attributes")
+        end
+      end
+      @geoip_lookup_accessors = @geoip_lookup_keys.map {|key| [key, record_accessor_create(key)] }.to_h
+
+      if conf.keys.any? {|k| k =~ /^enable_key_/ }
+        raise Fluent::ConfigError, "geoip: 'enable_key_*' config format is obsoleted. use <record></record> directive instead."
+      end
+
+      # <record></record> directive
+      conf.elements.select { |element| element.name == 'record' }.each { |element|
+        element.each_pair { |k, v|
+          element.has_key?(k) # to suppress unread configuration warning
+          v = v[1..v.size-2] if quoted_value?(v)
+          @map[k] = v
+          validate_json = Proc.new {
+            begin
+              dummy_text = Yajl::Encoder.encode('dummy_text')
+              Yajl::Parser.parse(v.gsub(REGEXP_PLACEHOLDER_SCAN, dummy_text))
+            rescue Yajl::ParseError => e
+              message = "geoip: failed to parse '#{v}' as json."
+              log.error message, error: e
+              raise Fluent::ConfigError, message
+            end
+          }
+          validate_json.call if json?(v.tr('\'"\\', ''))
+        }
+      }
+
+      @placeholder_keys = @map.values.join.scan(REGEXP_PLACEHOLDER_SCAN).map{|placeholder| placeholder[0] }.uniq
+      @placeholder_keys.each do |key|
+        m = key.match(REGEXP_PLACEHOLDER_SINGLE)
+        raise Fluent::ConfigError, "Invalid placeholder attributes: #{key}" unless m
+        geoip_key = m[:geoip_key]
+        case @backend_library
+        when :geoip
+          raise Fluent::ConfigError, "#{@backend_library}: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
+        when :geoip2_compat
+          raise Fluent::ConfigError, "#{@backend_library}: unsupported key #{geoip_key}" unless GEOIP2_COMPAT_KEYS.include?(geoip_key)
+        when :geoip2_c
+          # Nothing to do.
+          # We cannot define supported key(s) before we fetch values from GeoIP2 database
+          # because geoip2_c can fetch any fields in GeoIP2 database.
+        end
+      end
+
+      @geoip = load_database
     end
 
     def filter(tag, time, record)
-      filtered_record = @geoip.add_geoip_field(record)
+      filtered_record = add_geoip_field(record)
       if filtered_record
         record = filtered_record
       end
+      record = inject_values_to_record(tag, time, record)
       record
     end
+
+    def multi_workers_ready?
+      true
+    end
+
+    private
+
+    def add_geoip_field(record)
+      placeholder = create_placeholder(geolocate(get_address(record)))
+      return record if @skip_adding_null_record && placeholder.values.first.nil?
+      @map.each do |record_key, value|
+        if value.match(REGEXP_PLACEHOLDER_SINGLE) #|| value.match(REGEXP_PLACEHOLDER_BRACKET_SINGLE)
+          rewrited = placeholder[value]
+        elsif json?(value)
+          rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN) {|match|
+            match = match[1..match.size-2] if quoted_value?(match)
+            Yajl::Encoder.encode(placeholder[match])
+          }
+          rewrited = parse_json(rewrited)
+        else
+          rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN, placeholder)
+        end
+        record[record_key] = rewrited
+      end
+      record
+    end
+
+    def json?(text)
+      text.match(/^\[.+\]$/) || text.match(/^\{.+\}$/)
+    end
+
+    def quoted_value?(text)
+      # to improbe compatibility with fluentd v1-config
+      text.match(/(^'.+'$|^".+"$)/)
+    end
+
+    def parse_json(message)
+      begin
+        return Yajl::Parser.parse(message)
+      rescue Yajl::ParseError => e
+        log.info "geoip: failed to parse '#{message}' as json.", error_class: e.class, error: e.message
+        return nil
+      end
+    end
+
+    def get_address(record)
+      address = {}
+      @geoip_lookup_accessors.each do |field, accessor|
+        address[field] = accessor.call(record)
+      end
+      address
+    end
+
+    def geolocate(addresses)
+      geodata = {}
+      addresses.each do |field, ip|
+        geo = nil
+        if ip
+          if ip.empty?
+            log.warn "#{field} is empty string"
+          else
+            geo = if @geoip.respond_to?(:look_up)
+                    @geoip.look_up(ip)
+                  else
+                    @geoip.lookup(ip)
+                  end
+          end
+        end
+        geodata[field] = geo
+      end
+      geodata
+    end
+
+    def create_placeholder(geodata)
+      placeholder = {}
+      @placeholder_keys.each do |placeholder_key|
+        position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
+        next if position.nil? or geodata[position[:record_key]].nil?
+        keys = [position[:record_key]] + position[:geoip_key].split('.').map(&:to_sym)
+        value = geodata.dig(*keys)
+        value = if [:latitude, :longitude].include?(keys.last)
+                  value || 0.0
+                else
+                  value
+                end
+        placeholder[placeholder_key] = value
+      end
+      placeholder
+    end
+
+    def load_database
+      case @backend_library
+      when :geoip
+        ::GeoIP::City.new(@geoip_database, :memory, false)
+      when :geoip2_compat
+        require 'geoip2_compat'
+        GeoIP2Compat.new(@geoip2_database)
+      when :geoip2_c
+        require 'geoip2'
+        GeoIP2::Database.new(@geoip2_database)
+      end
+    rescue LoadError
+      raise Fluent::ConfigError, "You must install #{@backend_library} gem."
+    end
   end
 end

data/test/helper.rb

@@ -1,5 +1,6 @@
 require 'bundler/setup'
 require 'test/unit'
+require 'test/unit/rr'
 
 $LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
 $LOAD_PATH.unshift(__dir__)

data/test/plugin/test_filter_geoip.rb

@@ -9,8 +9,10 @@ class GeoipFilterTest < Test::Unit::TestCase
   end
 
   CONFIG = %[
-    geoip_lookup_key  host
-    enable_key_city   geoip_city
+    geoip_lookup_keys  host
+    <record>
+      geoip_city ${city.names.en['host']}
+    </record>
   ]
 
   def create_driver(conf = CONFIG, syntax: :v1)
@@ -19,6 +21,7 @@ class GeoipFilterTest < Test::Unit::TestCase
 
   def filter(config, messages, syntax: :v1)
     d = create_driver(config, syntax: syntax)
+    yield d if block_given?
     d.run(default_tag: "input.access") {
       messages.each {|message|
         d.feed(@time, message)
@@ -27,6 +30,15 @@ class GeoipFilterTest < Test::Unit::TestCase
     d.filtered_records
   end
 
+  def setup_geoip_mock(d)
+    plugin = d.instance
+    db = Object.new
+    def db.lookup(ip)
+      {}
+    end
+    plugin.instance_variable_set(:@geoip, db)
+  end
+
   sub_test_case "configure" do
     test "empty" do
       assert_nothing_raised {
@@ -34,42 +46,27 @@ class GeoipFilterTest < Test::Unit::TestCase
       }
     end
 
-    test "missing required parameters" do
+    test "obsoleted configuration" do
       assert_raise(Fluent::ConfigError) {
-        create_driver('enable_key_cities')
+        create_driver('enable_key_city geoip_city')
       }
     end
 
-    test "minimum" do
-      d = create_driver %[
-        enable_key_city   geoip_city
-      ]
-      assert_equal 'geoip_city', d.instance.config['enable_key_city']
-    end
-
-    test "multiple key config" do
-      d = create_driver %[
-        geoip_lookup_key  from.ip, to.ip
-        enable_key_city   from_city, to_city
+    test "deprecated configuration geoip_lookup_key" do
+      conf = %[
+        geoip_lookup_key  host,ip
+        <record>
+          geoip_city ${city['host']}
+        </record>
       ]
-      assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
-    end
-
-    test "multiple key config (bad configure)" do
-      assert_raise(Fluent::ConfigError) {
-        create_driver %[
-          geoip_lookup_key  from.ip, to.ip
-          enable_key_city   from_city
-          enable_key_region from_region
-        ]
-      }
+      d = create_driver(conf)
+      assert_equal(["host", "ip"], d.instance.geoip_lookup_keys)
     end
 
     test "invalid json structure w/ Ruby hash like" do
-
       assert_raise(Fluent::ConfigParseError) {
         create_driver %[
-          geoip_lookup_key  host
+          geoip_lookup_keys host
           <record>
             invalid_json    {"foo" => 123}
           </record>
@@ -80,7 +77,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     test "invalid json structure w/ unquoted string literal" do
       assert_raise(Fluent::ConfigParseError) {
         create_driver %[
-          geoip_lookup_key  host
+          geoip_lookup_keys host
           <record>
             invalid_json    {"foo" : string, "bar" : 123}
           </record>
@@ -88,6 +85,49 @@ class GeoipFilterTest < Test::Unit::TestCase
       }
     end
 
+    test "dotted key is not treated as nested attributes" do
+      mock($log).warn("host.ip is not treated as nested attributes")
+      create_driver %[
+        geoip_lookup_keys host.ip
+        <record>
+          city ${city.names.en['host.ip']}
+        </record>
+      ]
+    end
+
+    test "nested attributes bracket style" do
+      mock($log).warn(anything).times(0)
+      create_driver %[
+        geoip_lookup_keys  $["host"]["ip"]
+        <record>
+          geoip_city ${city.names.en['$["host"]["ip"]']}
+        </record>
+      ]
+    end
+
+    test "nested attributes dot style" do
+      mock($log).warn(anything).times(0)
+      create_driver %[
+        geoip_lookup_keys  $.host.ip
+        <record>
+          geoip_city ${city['$.host.ip']}
+        </record>
+      ]
+    end
+
+    test "invalid placeholder attributes" do
+      assert_raise(Fluent::ConfigError) do
+        create_driver %[
+          geoip_lookup_keys host
+          backend_library geoip2_c
+
+          <record>
+            geoip.city_name       ${city.names.en["host]}
+          </record>
+        ]
+      end
+    end
+
     data(geoip: "geoip",
          geoip2_compat: "geoip2_compat")
     test "unsupported key" do |backend|
@@ -129,7 +169,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_with_dot_key
       config = %[
         backend_library   geoip2_c
-        geoip_lookup_key  ip.origin, ip.dest
+        geoip_lookup_keys ip.origin, ip.dest
         <record>
           origin_country  ${country.iso_code['ip.origin']}
           dest_country    ${country.iso_code['ip.dest']}
@@ -149,7 +189,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_with_unknown_address
       config = %[
         backend_library   geoip2_c
-        geoip_lookup_key  host
+        geoip_lookup_keys host
         <record>
           geoip_city      ${city.names.en['host']}
           geopoint        [${location.longitude['host']}, ${location.latitude['host']}]
@@ -169,10 +209,30 @@ class GeoipFilterTest < Test::Unit::TestCase
       assert_equal(expected, filtered)
     end
 
+    def test_filter_with_empty_string
+      config = %[
+        backend_library   geoip2_c
+        geoip_lookup_keys host
+        <record>
+          geoip_city      '${city.names.en["host"]}'
+          geopoint        '[${location.longitude["host"]}, ${location.latitude["host"]}]'
+        </record>
+        skip_adding_null_record false
+      ]
+      messages = [
+        {'host' => '', 'message' => 'empty string ip'},
+      ]
+      expected = [
+        {'host' => '', 'message' => 'empty string ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]},
+      ]
+      filtered = filter(config, messages)
+      assert_equal(expected, filtered)
+    end
+
     def test_filter_with_skip_unknown_address
       config = %[
         backend_library   geoip2_c
-        geoip_lookup_key  host
+        geoip_lookup_keys host
         <record>
           geoip_city      ${city.names.en['host']}
           geopoint        [${location.longitude['host']}, ${location.latitude['host']}]
@@ -198,21 +258,21 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_record_directive
       config = %[
         backend_library   geoip2_c
-        geoip_lookup_key  from.ip
+        geoip_lookup_keys $.from.ip
         <record>
-          from_city       ${city.names.en['from.ip']}
-          from_country    ${country.names.en['from.ip']}
-          latitude        ${location.latitude['from.ip']}
-          longitude       ${location.longitude['from.ip']}
-          float_concat    ${location.latitude['from.ip']},${location.longitude['from.ip']}
-          float_array     [${location.longitude['from.ip']}, ${location.latitude['from.ip']}]
-          float_nest      { "lat" : ${location.latitude['from.ip']}, "lon" : ${location.longitude['from.ip']}}
-          string_concat   ${city.names.en['from.ip']},${country.names.en['from.ip']}
-          string_array    [${city.names.en['from.ip']}, ${country.names.en['from.ip']}]
-          string_nest     { "city" : ${city.names.en['from.ip']}, "country_name" : ${country.names.en['from.ip']}}
+          from_city       ${city.names.en['$.from.ip']}
+          from_country    ${country.names.en['$.from.ip']}
+          latitude        ${location.latitude['$.from.ip']}
+          longitude       ${location.longitude['$.from.ip']}
+          float_concat    ${location.latitude['$.from.ip']},${location.longitude['$.from.ip']}
+          float_array     [${location.longitude['$.from.ip']}, ${location.latitude['$.from.ip']}]
+          float_nest      { "lat" : ${location.latitude['$.from.ip']}, "lon" : ${location.longitude['$.from.ip']}}
+          string_concat   ${city.names.en['$.from.ip']},${country.names.en['$.from.ip']}
+          string_array    [${city.names.en['$.from.ip']}, ${country.names.en['$.from.ip']}]
+          string_nest     { "city" : ${city.names.en['$.from.ip']}, "country_name" : ${country.names.en['$.from.ip']}}
           unknown_city    ${city.names.en['unknown_key']}
           undefined       ${city.names.en['undefined']}
-          broken_array1   [${location.longitude['from.ip']}, ${location.latitude['undefined']}]
+          broken_array1   [${location.longitude['$.from.ip']}, ${location.latitude['undefined']}]
           broken_array2   [${location.longitude['undefined']}, ${location.latitude['undefined']}]
         </record>
       ]
@@ -265,13 +325,13 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_record_directive_multiple_record
       config = %[
         backend_library   geoip2_c
-        geoip_lookup_key  from.ip, to.ip
+        geoip_lookup_keys $.from.ip, $.to.ip
         <record>
-          from_city       ${city.names.en['from.ip']}
-          to_city         ${city.names.en['to.ip']}
-          from_country    ${country.names.en['from.ip']}
-          to_country      ${country.names.en['to.ip']}
-          string_array    [${country.names.en['from.ip']}, ${country.names.en['to.ip']}]
+          from_city       ${city.names.en['$.from.ip']}
+          to_city         ${city.names.en['$.to.ip']}
+          from_country    ${country.names.en['$.from.ip']}
+          to_country      ${country.names.en['$.to.ip']}
+          string_array    [${country.names.en['$.from.ip']}, ${country.names.en['$.to.ip']}]
         </record>
       ]
       messages = [
@@ -304,7 +364,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     def config_quoted_record
       %[
         backend_library   geoip2_c
-        geoip_lookup_key  host
+        geoip_lookup_keys host
         <record>
           location_properties  '{ "country_code" : "${country.iso_code["host"]}", "lat": ${location.latitude["host"]}, "lon": ${location.longitude["host"]} }'
           location_string      ${location.latitude['host']},${location.longitude['host']}
@@ -365,7 +425,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_multiline_v1_config
       config = %[
         backend_library   geoip2_c
-        geoip_lookup_key  host
+        geoip_lookup_keys host
         <record>
           location_properties  {
             "city": "${city.names.en['host']}",
@@ -392,13 +452,79 @@ class GeoipFilterTest < Test::Unit::TestCase
       filtered = filter(config, messages)
       assert_equal(expected, filtered)
     end
+
+    def test_filter_when_latitude_longitude_is_nil
+      config = %[
+        backend_library   geoip2_c
+        geoip_lookup_keys  host
+        <record>
+          latitude  ${location.latitude['host']}
+          longitude ${location.longitude['host']}
+        </record>
+      ]
+      messages = [
+        { "host" => "180.94.85.84", "message" => "nil latitude and longitude" }
+      ]
+      expected = [
+        {
+          "host" => "180.94.85.84",
+          "message" => "nil latitude and longitude",
+          "latitude" => 0.0,
+          "longitude" => 0.0
+        }
+      ]
+      filtered = filter(config, messages) do |d|
+        setup_geoip_mock(d)
+      end
+      assert_equal(expected, filtered)
+    end
+
+    def test_filter_nested_attr_bracket_style_double_quote
+      config = %[
+        backend_library geoip2_c
+        geoip_lookup_keys  $["host"]["ip"]
+        <record>
+          geoip_city ${city.names.en['$["host"]["ip"]']}
+        </record>
+      ]
+      messages = [
+        {'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip'},
+        {'message' => 'missing field'}
+      ]
+      expected = [
+        {'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip', 'geoip_city' => 'Mountain View'},
+        {'message' => 'missing field', 'geoip_city' => nil}
+      ]
+      filtered = filter(config, messages)
+      assert_equal(expected, filtered)
+    end
+
+    def test_filter_nested_attr_bracket_style_single_quote
+      config = %[
+        backend_library geoip2_c
+        geoip_lookup_keys  $['host']['ip']
+        <record>
+          geoip_city ${city.names.en["$['host']['ip']"]}
+        </record>
+      ]
+      messages = [
+        {'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip'},
+        {'message' => 'missing field'}
+      ]
+      expected = [
+        {'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip', 'geoip_city' => 'Mountain View'},
+        {'message' => 'missing field', 'geoip_city' => nil}
+      ]
+      filtered = filter(config, messages)
+      assert_equal(expected, filtered)
+    end
   end
 
   sub_test_case "geoip2_compat" do
     def test_filter_with_dot_key
       config = %[
         backend_library   geoip2_compat
-        geoip_lookup_key  ip.origin, ip.dest
+        geoip_lookup_keys ip.origin, ip.dest
         <record>
           origin_country  ${country_code['ip.origin']}
           dest_country    ${country_code['ip.dest']}
@@ -418,7 +544,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_with_unknown_address
       config = %[
         backend_library   geoip2_compat
-        geoip_lookup_key  host
+        geoip_lookup_keys host
         <record>
           geoip_city      ${city['host']}
           geopoint        [${longitude['host']}, ${latitude['host']}]
@@ -441,7 +567,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_with_skip_unknown_address
       config = %[
         backend_library   geoip2_compat
-        geoip_lookup_key  host
+        geoip_lookup_keys host
         <record>
           geoip_city      ${city['host']}
           geopoint        [${longitude['host']}, ${latitude['host']}]
@@ -467,21 +593,21 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_record_directive
       config = %[
         backend_library   geoip2_compat
-        geoip_lookup_key  from.ip
+        geoip_lookup_keys $.from.ip
         <record>
-          from_city       ${city['from.ip']}
-          from_country    ${country_name['from.ip']}
-          latitude        ${latitude['from.ip']}
-          longitude       ${longitude['from.ip']}
-          float_concat    ${latitude['from.ip']},${longitude['from.ip']}
-          float_array     [${longitude['from.ip']}, ${latitude['from.ip']}]
-          float_nest      { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
-          string_concat   ${city['from.ip']},${country_name['from.ip']}
-          string_array    [${city['from.ip']}, ${country_name['from.ip']}]
-          string_nest     { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
+          from_city       ${city['$.from.ip']}
+          from_country    ${country_name['$.from.ip']}
+          latitude        ${latitude['$.from.ip']}
+          longitude       ${longitude['$.from.ip']}
+          float_concat    ${latitude['$.from.ip']},${longitude['$.from.ip']}
+          float_array     [${longitude['$.from.ip']}, ${latitude['$.from.ip']}]
+          float_nest      { "lat" : ${latitude['$.from.ip']}, "lon" : ${longitude['$.from.ip']}}
+          string_concat   ${city['$.from.ip']},${country_name['$.from.ip']}
+          string_array    [${city['$.from.ip']}, ${country_name['$.from.ip']}]
+          string_nest     { "city" : ${city['$.from.ip']}, "country_name" : ${country_name['$.from.ip']}}
           unknown_city    ${city['unknown_key']}
           undefined       ${city['undefined']}
-          broken_array1   [${longitude['from.ip']}, ${latitude['undefined']}]
+          broken_array1   [${longitude['$.from.ip']}, ${latitude['undefined']}]
           broken_array2   [${longitude['undefined']}, ${latitude['undefined']}]
         </record>
       ]
@@ -534,13 +660,13 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_record_directive_multiple_record
       config = %[
         backend_library   geoip2_compat
-        geoip_lookup_key  from.ip, to.ip
+        geoip_lookup_keys $.from.ip, $.to.ip
         <record>
-          from_city       ${city['from.ip']}
-          to_city         ${city['to.ip']}
-          from_country    ${country_name['from.ip']}
-          to_country      ${country_name['to.ip']}
-          string_array    [${country_name['from.ip']}, ${country_name['to.ip']}]
+          from_city       ${city['$.from.ip']}
+          to_city         ${city['$.to.ip']}
+          from_country    ${country_name['$.from.ip']}
+          to_country      ${country_name['$.to.ip']}
+          string_array    [${country_name['$.from.ip']}, ${country_name['$.to.ip']}]
         </record>
       ]
       messages = [
@@ -573,7 +699,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     def config_quoted_record
       %[
         backend_library   geoip2_compat
-        geoip_lookup_key  host
+        geoip_lookup_keys host
         <record>
           location_properties  '{ "country_code" : "${country_code["host"]}", "lat": ${latitude["host"]}, "lon": ${longitude["host"]} }'
           location_string      ${latitude['host']},${longitude['host']}
@@ -634,7 +760,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_multiline_v1_config
       config = %[
         backend_library   geoip2_compat
-        geoip_lookup_key  host
+        geoip_lookup_keys host
         <record>
           location_properties  {
             "city": "${city['host']}",
@@ -661,14 +787,42 @@ class GeoipFilterTest < Test::Unit::TestCase
       filtered = filter(config, messages)
       assert_equal(expected, filtered)
     end
+
+    def test_filter_when_latitude_longitude_is_nil
+      config = %[
+        backend_library   geoip2_compat
+        geoip_lookup_keys  host
+        <record>
+          latitude  ${latitude['host']}
+          longitude ${longitude['host']}
+        </record>
+      ]
+      messages = [
+        { "host" => "180.94.85.84", "message" => "nil latitude and longitude" }
+      ]
+      expected = [
+        {
+          "host" => "180.94.85.84",
+          "message" => "nil latitude and longitude",
+          "latitude" => 0.0,
+          "longitude" => 0.0
+        }
+      ]
+      filtered = filter(config, messages) do |d|
+        setup_geoip_mock(d)
+      end
+      assert_equal(expected, filtered)
+    end
   end
 
   sub_test_case "geoip legacy" do
     def test_filter
       config = %[
         backend_library geoip
-        geoip_lookup_key  host
-        enable_key_city   geoip_city
+        geoip_lookup_keys  host
+        <record>
+          geoip_city ${city['host']}
+        </record>
       ]
       messages = [
         {'host' => '66.102.3.80', 'message' => 'valid ip'},
@@ -685,7 +839,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_with_dot_key
       config = %[
         backend_library geoip
-        geoip_lookup_key  ip.origin, ip.dest
+        geoip_lookup_keys ip.origin, ip.dest
         <record>
           origin_country  ${country_code['ip.origin']}
           dest_country    ${country_code['ip.dest']}
@@ -705,8 +859,10 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_nested_attr
       config = %[
         backend_library geoip
-        geoip_lookup_key  host.ip
-        enable_key_city   geoip_city
+        geoip_lookup_keys  $.host.ip
+        <record>
+          geoip_city ${city['$.host.ip']}
+        </record>
       ]
       messages = [
         {'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip'},
@@ -723,7 +879,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_with_unknown_address
       config = %[
         backend_library geoip
-        geoip_lookup_key  host
+        geoip_lookup_keys host
         <record>
           geoip_city      ${city['host']}
           geopoint        [${longitude['host']}, ${latitude['host']}]
@@ -746,7 +902,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_with_skip_unknown_address
       config = %[
         backend_library geoip
-        geoip_lookup_key  host
+        geoip_lookup_keys host
         <record>
           geoip_city      ${city['host']}
           geopoint        [${longitude['host']}, ${latitude['host']}]
@@ -772,8 +928,11 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_multiple_key
       config = %[
         backend_library geoip
-        geoip_lookup_key  from.ip, to.ip
-        enable_key_city   from_city, to_city
+        geoip_lookup_keys  $.from.ip, $.to.ip
+        <record>
+          from_city ${city['$.from.ip']}
+          to_city   ${city['$.to.ip']}
+        </record>
       ]
       messages = [
         {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
@@ -791,9 +950,13 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_multiple_key_multiple_record
       config = %[
         backend_library geoip
-        geoip_lookup_key  from.ip, to.ip
-        enable_key_city   from_city, to_city
-        enable_key_country_name from_country, to_country
+        geoip_lookup_keys  $.from.ip, $.to.ip
+        <record>
+          from_city    ${city['$.from.ip']}
+          from_country ${country_name['$.from.ip']}
+          to_city      ${city['$.to.ip']}
+          to_country   ${country_name['$.to.ip']}
+        </record>
       ]
       messages = [
         {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
@@ -831,21 +994,21 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_record_directive
       config = %[
         backend_library geoip
-        geoip_lookup_key  from.ip
+        geoip_lookup_keys $.from.ip
         <record>
-          from_city       ${city['from.ip']}
-          from_country    ${country_name['from.ip']}
-          latitude        ${latitude['from.ip']}
-          longitude       ${longitude['from.ip']}
-          float_concat    ${latitude['from.ip']},${longitude['from.ip']}
-          float_array     [${longitude['from.ip']}, ${latitude['from.ip']}]
-          float_nest      { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
-          string_concat   ${city['from.ip']},${country_name['from.ip']}
-          string_array    [${city['from.ip']}, ${country_name['from.ip']}]
-          string_nest     { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
+          from_city       ${city['$.from.ip']}
+          from_country    ${country_name['$.from.ip']}
+          latitude        ${latitude['$.from.ip']}
+          longitude       ${longitude['$.from.ip']}
+          float_concat    ${latitude['$.from.ip']},${longitude['$.from.ip']}
+          float_array     [${longitude['$.from.ip']}, ${latitude['$.from.ip']}]
+          float_nest      { "lat" : ${latitude['$.from.ip']}, "lon" : ${longitude['$.from.ip']}}
+          string_concat   ${city['$.from.ip']},${country_name['$.from.ip']}
+          string_array    [${city['$.from.ip']}, ${country_name['$.from.ip']}]
+          string_nest     { "city" : ${city['$.from.ip']}, "country_name" : ${country_name['$.from.ip']}}
           unknown_city    ${city['unknown_key']}
           undefined       ${city['undefined']}
-          broken_array1   [${longitude['from.ip']}, ${latitude['undefined']}]
+          broken_array1   [${longitude['$.from.ip']}, ${latitude['undefined']}]
           broken_array2   [${longitude['undefined']}, ${latitude['undefined']}]
         </record>
       ]
@@ -898,13 +1061,13 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_record_directive_multiple_record
       config = %[
         backend_library geoip
-        geoip_lookup_key  from.ip, to.ip
+        geoip_lookup_keys $.from.ip, $.to.ip
         <record>
-          from_city       ${city['from.ip']}
-          to_city         ${city['to.ip']}
-          from_country    ${country_name['from.ip']}
-          to_country      ${country_name['to.ip']}
-          string_array    [${country_name['from.ip']}, ${country_name['to.ip']}]
+          from_city       ${city['$.from.ip']}
+          to_city         ${city['$.to.ip']}
+          from_country    ${country_name['$.from.ip']}
+          to_country      ${country_name['$.to.ip']}
+          string_array    [${country_name['$.from.ip']}, ${country_name['$.to.ip']}]
         </record>
       ]
       messages = [
@@ -937,7 +1100,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     def config_quoted_record
       %[
         backend_library geoip
-        geoip_lookup_key  host
+        geoip_lookup_keys host
         <record>
           location_properties  '{ "country_code" : "${country_code["host"]}", "lat": ${latitude["host"]}, "lon": ${longitude["host"]} }'
           location_string      ${latitude['host']},${longitude['host']}
@@ -998,7 +1161,7 @@ class GeoipFilterTest < Test::Unit::TestCase
     def test_filter_multiline_v1_config
       config = %[
         backend_library geoip
-        geoip_lookup_key  host
+        geoip_lookup_keys host
         <record>
           location_properties  {
             "city": "${city['host']}",
@@ -1025,6 +1188,32 @@ class GeoipFilterTest < Test::Unit::TestCase
       filtered = filter(config, messages)
       assert_equal(expected, filtered)
     end
+
+    def test_filter_when_latitude_longitude_is_nil
+      config = %[
+        backend_library   geoip
+        geoip_lookup_keys  host
+        <record>
+          latitude  ${latitude['host']}
+          longitude ${longitude['host']}
+        </record>
+      ]
+      messages = [
+        { "host" => "180.94.85.84", "message" => "nil latitude and longitude" }
+      ]
+      expected = [
+        {
+          "host" => "180.94.85.84",
+          "message" => "nil latitude and longitude",
+          "latitude" => 0.0,
+          "longitude" => 0.0
+        }
+      ]
+      filtered = filter(config, messages) do |d|
+        setup_geoip_mock(d)
+      end
+      assert_equal(expected, filtered)
+    end
   end
 end