fluent-plugin-geoip-kk 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d13a01e2ed55d640277b58763d852b7bbf910c82e2464437ca58e5ac95e66974
-  data.tar.gz: ae8541754e93dce45c9570b1687c85c19ddee1ba7e9a614d46258ecc963d1417
+  metadata.gz: 3343f0a6464f46969141b4c6dbc731bf58ff1bddabeecd0f55bfad2a4c21b842
+  data.tar.gz: 7c90087650a115cd023d3cde331d7504ee0540432747554e2150abd6d0ea780d
 SHA512:
-  metadata.gz: b5d2ce0a2c3e86f30e4a33a75b3b1fde813b7df100396e6cfd597906ac58feb0a0056d3de49329528fa9b226a11fc6befca23294625e45a77e4aa02129b264dd
-  data.tar.gz: d3ebbf140b28a7e427a2e21331dc43828274a165c7add6dde19c33843be7e3bd93a080c376166a5bde69ef8f9541a356d0ab93aba559b2bf5e59703e0bef8213
+  metadata.gz: dfd08a491d65964d1ed9ec4927ed1041f07f5a376b91b36cabf4a18c430854a7e1241fb3cbc73c65d9eb1d912f9f509659939dae5223cc4e0a63ca9cfafa6855
+  data.tar.gz: 4f534578358e07d02a193f9019c143304e8c3ef4997a22bf14f52ce8ec384884b804a12d9130ab84c281c4d34ec873dc270f63c216a5a6c6c8246613913f36df

data/.github/workflows/gem-push.yml

@@ -0,0 +1,34 @@
+name: Ruby Gem
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.2'   # 你可以指定 3.1、3.2、3.3 等
+
+      - name: Publish to RubyGems
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem
+        env:
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}"

data/README.md

@@ -2,12 +2,14 @@
 
 [Fluentd](http://fluentd.org) filter plugin to add geoip.
 
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-geoip-kk.svg)](https://badge.fury.io/rb/fluent-plugin-geoip-kk)
+[![Ruby](https://github.com/kevin197011/fluent-plugin-geoip-kk/actions/workflows/gem-push.yml/badge.svg)](https://github.com/kevin197011/fluent-plugin-geoip-kk/actions/workflows/gem-push.yml)
 
 ## Requirements
 
 | fluent-plugin-geoip-kk | fluentd    | ruby   |
 |----------------------------|------------|--------|
-| >= 1.0.0                   | >= v0.14.0 | >= 2.1 |
+| >= 1.0.1                   | >= v0.14.0 | >= 2.1 |
 | < 1.0.0                    | >= v0.12.0 | >= 1.9 |
 
 
@@ -94,14 +96,32 @@ It is able to customize fields with placeholder.
     * See [benchmark](test/bench_geoip_filter.rb).
 
 
-## TODO
+## Development
 
-* patches welcome!
+After checking out the repo, run `bundle install` to install dependencies. Then, run `rake test` to run the tests.
 
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+## Publishing
+
+This gem uses GitHub Actions for automated publishing. To publish a new version:
+
+1. Update the version number in `fluent-plugin-geoip-kk.gemspec`
+2. Commit the changes:
+   ```bash
+   git add fluent-plugin-geoip-kk.gemspec
+   git commit -m "Bump version to x.x.x"
+   ```
+3. Create and push a new tag:
+   ```bash
+   git tag vx.x.x
+   git push origin main vx.x.x
+   ```
+4. The GitHub Action will automatically build and publish the gem to RubyGems.org
 
 ## Contributing
 
-1. Fork it ( https://github.com/bungoume/fluent-plugin-geoip-filter/fork )
+1. Fork it ( https://github.com/kevin197011/fluent-plugin-geoip-kk/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)

data/fluent-plugin-geoip-kk.gemspec

@@ -5,12 +5,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-geoip-kk'
-  spec.version       = '1.0.1'
+  spec.version       = '1.0.2'
   spec.authors       = ['kevin197011']
   spec.email         = ['kevin197011@outlook.com']
   spec.homepage      = 'https://github.com/kevin197011/fluent-plugin-geoip-kk'
   spec.summary       = 'Fluentd filter plugin to add geoip'
-  spec.description   = spec.summary
+  spec.description   = 'A Fluentd filter plugin that adds GeoIP information to records. Supports both legacy GeoIP and GeoIP2 databases, with LRU caching for improved performance.'
   spec.license       = 'Apache-2.0'
 
   spec.files         = `git ls-files -z`.split("\x0")
@@ -18,11 +18,14 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'fluentd', '>= 0.14.0'
-  spec.add_runtime_dependency 'geoip', '>= 1.6.4'
-  spec.add_runtime_dependency 'lru_redux', '>= 1.0.0'
-  spec.add_runtime_dependency 'maxminddb', '>= 0.1.22'
-  spec.add_development_dependency 'bundler'
-  spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'test-unit' if defined?(RUBY_VERSION) && RUBY_VERSION > '2.2'
+  spec.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
+  spec.add_runtime_dependency 'geoip', '~> 0.1', '>= 0.1.22'
+  spec.add_runtime_dependency 'lru_redux', '~> 1.0', '>= 1.0.0'
+  spec.add_runtime_dependency 'maxminddb', '~> 1.5', '>= 1.5.0'
+
+  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'test-unit', '~> 3.0' if defined?(RUBY_VERSION) && RUBY_VERSION > '2.2'
+
+  spec.required_ruby_version = '>= 2.1.0'
 end

data/lib/fluent/plugin/filter_geoip.rb

@@ -3,45 +3,93 @@
 require 'maxminddb'
 require 'lru_redux'
 require 'fluent/plugin/filter'
+require 'ipaddr'
 
 module Fluent
   module Plugin
     class GeoipFilter < Filter
       Fluent::Plugin.register_filter('geoip', self)
 
-      def initialize
-        @geoip_cache = LruRedux::Cache.new(8192)
-        super
-      end
+      helpers :record_accessor
 
+      # Default cache size (number of IP addresses to cache)
+      DEFAULT_CACHE_SIZE = 8192
+      DEFAULT_CACHE_TTL = 3600 # 1 hour in seconds
+
+      desc 'Path to the GeoIP database file'
       config_param :database_path, :string, default: "#{File.dirname(__FILE__)}/../../../data/GeoLite2-City.mmdb"
+
+      desc 'Field name that contains the IP address'
       config_param :key_name, :string, default: 'client_ip'
+
+      desc 'Output field name to store GeoIP data'
       config_param :out_key, :string, default: 'geo'
+
+      desc 'Flatten the GeoIP data structure'
       config_param :flatten, :bool, default: false
 
+      desc 'Size of the LRU cache'
+      config_param :cache_size, :integer, default: DEFAULT_CACHE_SIZE
+
+      desc 'TTL for cached items in seconds'
+      config_param :cache_ttl, :integer, default: DEFAULT_CACHE_TTL
+
+      desc 'Skip adding GeoIP data for private IP addresses'
+      config_param :skip_private_ip, :bool, default: true
+
+      def initialize
+        super
+        @geoip = nil
+        @ip_accessor = nil
+      end
+
       def configure(conf)
         super
+
+        unless File.exist?(@database_path)
+          raise Fluent::ConfigError, "GeoIP database file '#{@database_path}' does not exist"
+        end
+
+        # Initialize MaxMindDB
         begin
           @geoip = MaxMindDB.new(@database_path)
         rescue StandardError => e
-          log.warn 'Failed to configure parser. Use default pattern.', error_class: e.class, error: e.message
-          log.warn_backtrace
+          raise Fluent::ConfigError, "Failed to load GeoIP database: #{e.message}"
         end
+
+        # Initialize IP field accessor
+        @ip_accessor = record_accessor_create(@key_name)
+
+        # Initialize cache with TTL support
+        @geoip_cache = LruRedux::TTL::Cache.new(@cache_size, @cache_ttl)
+
+        log.info "Initialized GeoIP filter", database: @database_path, cache_size: @cache_size, cache_ttl: @cache_ttl
       end
 
-      def filter(_tag, _time, record)
-        ip_addr = record[@key_name]
+      def filter(tag, time, record)
+        ip_addr = @ip_accessor.call(record)
+        return record if ip_addr.nil? || ip_addr.empty? || ip_addr == '-'
 
-        return record if ip_addr.nil? || ip_addr == '-'
+        begin
+          ip = IPAddr.new(ip_addr)
+          return record if @skip_private_ip && ip.private?
+        rescue IPAddr::InvalidAddressError => e
+          log.debug "Invalid IP address", ip: ip_addr, error: e.message
+          return record
+        end
 
         geo_ip = @geoip_cache.getset(ip_addr) { get_geoip(ip_addr) }
+        return record if geo_ip.empty?
 
-        if flatten
-          record.merge! hash_flatten(geo_ip, [@out_key])
+        if @flatten
+          record.merge!(hash_flatten(geo_ip, [@out_key]))
         else
           record[@out_key] = geo_ip
         end
 
+        record
+      rescue StandardError => e
+        log.error "Failed to process GeoIP lookup", error_class: e.class, error: e.message, tag: tag, time: time
         record
       end
 
@@ -49,24 +97,73 @@ module Fluent
 
       def get_geoip(ip_addr)
         geo_ip = @geoip.lookup(ip_addr)
-        data = {}
-        return data if geo_ip.nil? || ip_addr == '-'
-
-        data['coordinates'] = [geo_ip.location.longitude, geo_ip.location.latitude] if geo_ip.location
-        data['country_code'] = geo_ip.country.iso_code if geo_ip.country
-        data['city'] = geo_ip.city.name if geo_ip.city
-        data['region_code'] = geo_ip.subdivisions.first.iso_code if geo_ip.subdivisions.any?
-        data
+        return {} if geo_ip.nil?
+
+        {
+          'coordinates' => get_coordinates(geo_ip),
+          'country' => get_country_info(geo_ip),
+          'city' => get_city_info(geo_ip),
+          'region' => get_region_info(geo_ip),
+          'postal' => get_postal_info(geo_ip),
+          'timezone' => geo_ip.location&.time_zone
+        }.compact
+      end
+
+      def get_coordinates(geo_ip)
+        return nil unless geo_ip.location
+
+        {
+          'latitude' => geo_ip.location.latitude,
+          'longitude' => geo_ip.location.longitude,
+          'accuracy_radius' => geo_ip.location.accuracy_radius
+        }.compact
+      end
+
+      def get_country_info(geo_ip)
+        return nil unless geo_ip.country
+
+        {
+          'code' => geo_ip.country.iso_code,
+          'name' => geo_ip.country.name
+        }.compact
+      end
+
+      def get_city_info(geo_ip)
+        return nil unless geo_ip.city
+
+        {
+          'name' => geo_ip.city.name,
+          'confidence' => geo_ip.city.confidence
+        }.compact
+      end
+
+      def get_region_info(geo_ip)
+        return nil unless geo_ip.subdivisions&.any?
+
+        subdivision = geo_ip.subdivisions.first
+        {
+          'code' => subdivision.iso_code,
+          'name' => subdivision.name
+        }.compact
+      end
+
+      def get_postal_info(geo_ip)
+        return nil unless geo_ip.postal
+
+        {
+          'code' => geo_ip.postal.code,
+          'confidence' => geo_ip.postal.confidence
+        }.compact
       end
 
-      def hash_flatten(a, keys = [])
+      def hash_flatten(hash, keys = [])
         ret = {}
-        a.each do |k, v|
-          ks = keys + [k]
-          if v.instance_of?(Hash)
-            ret.merge!(hash_flatten(v, ks))
+        hash.each do |k, v|
+          key_chain = keys + [k]
+          if v.is_a?(Hash)
+            ret.merge!(hash_flatten(v, key_chain))
           else
-            ret.merge!({ ks.join('_') => v })
+            ret[key_chain.join('_')] = v
           end
         end
         ret

data/push.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+system 'rake'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-geoip-kk
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - kevin197011
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-06 00:00:00.000000000 Z
+date: 2025-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -17,6 +17,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -24,24 +27,36 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: geoip
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.6.4
+        version: 0.1.22
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.6.4
+        version: 0.1.22
 - !ruby/object:Gem::Dependency
   name: lru_redux
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
@@ -49,6 +64,9 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
@@ -56,84 +74,90 @@ dependencies:
   name: maxminddb
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.22
+        version: 1.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.22
+        version: 1.5.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-description: Fluentd filter plugin to add geoip
+        version: '3.0'
+description: A Fluentd filter plugin that adds GeoIP information to records. Supports
+  both legacy GeoIP and GeoIP2 databases, with LRU caching for improved performance.
 email:
 - kevin197011@outlook.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitattributes"
+- ".github/workflows/gem-push.yml"
 - ".gitignore"
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - data/GeoLite2-City.mmdb
-- data/GeoLiteCity.dat
 - fluent-plugin-geoip-kk.gemspec
 - lib/fluent/plugin/filter_geoip.rb
+- push.rb
 - test.rb
 - test/bench_geoip_filter.rb
 - test/helper.rb
-- test/plugin/test_filter_geoip.rb
 homepage: https://github.com/kevin197011/fluent-plugin-geoip-kk
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -141,18 +165,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.18
-signing_key: 
+rubygems_version: 3.4.19
+signing_key:
 specification_version: 4
 summary: Fluentd filter plugin to add geoip
 test_files:
 - test/bench_geoip_filter.rb
 - test/helper.rb
-- test/plugin/test_filter_geoip.rb

data/.gitattributes

@@ -1 +0,0 @@
-*.dat filter=lfs diff=lfs merge=lfs -text

data/data/GeoLiteCity.dat

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:25bce9e7f5fe9ffd12afa663c2e57b311a4286549a858965c4ff01596f604c8a
-size 19148749

data/test/plugin/test_filter_geoip.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-require 'helper'
-
-class GeoipFilterTest < Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-  end
-
-  CONFIG = %(
-    @type geoip
-    key_name client_ip
-    out_key geo
-  )
-
-  def create_driver(conf = CONFIG)
-    Fluent::Test::Driver::Filter.new(Fluent::Plugin::GeoipFilter).configure(conf)
-  end
-
-  def test_configure
-    d = create_driver(CONFIG)
-    assert_equal 'client_ip', d.instance.config['key_name']
-    assert_equal 'geo', d.instance.config['out_key']
-  end
-
-  def test_emit
-    d1 = create_driver(CONFIG)
-    ip_iddr = '93.184.216.34'
-
-    d1.run(default_tag: 'test') do
-      d1.feed({ 'client_ip' => ip_iddr })
-    end
-    emits = d1.filtered
-    assert_equal 1, emits.length
-    geo_object = emits[0][1]['geo']
-    assert_equal [-70.8228, 42.150800000000004], geo_object['coordinates']
-    assert_equal 'US', geo_object['country_code']
-    assert_equal 'Norwell', geo_object['city']
-    assert_equal 'MA', geo_object['region_code']
-  end
-
-  def test_emit_flatten
-    d1 = create_driver(%(
-      @type geoip
-      key_name ip_iddr
-      flatten
-    ))
-    ip_iddr = '93.184.216.34'
-
-    d1.run(default_tag: 'test') do
-      d1.feed({ 'ip_iddr' => ip_iddr })
-    end
-
-    emits = d1.filtered
-    assert_equal 1, emits.length
-    geo_object = emits[0][1]
-    assert_equal [-70.8228, 42.150800000000004], geo_object['geo_coordinates']
-    assert_equal 'US', geo_object['geo_country_code']
-    assert_equal 'Norwell', geo_object['geo_city']
-    assert_equal 'MA', geo_object['geo_region_code']
-  end
-end