fluent-plugin-geoip-kk 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ed458000a7ed8af776a38bf822027286ea3770a8f0b5668083419bd79516304b
+  data.tar.gz: d988309e6a245f51bf670bbc8d6d42dfafb087546a0ccb183e55a3926adb6459
+SHA512:
+  metadata.gz: e45cca487405e1c11fb97a3866f9f9c5d20d724ee4285432b78b1ed097de16464532455972ee7222a8b1d3d538d95e849c899bc5b8bfc8501e6d76b24a6e508b
+  data.tar.gz: dc58642c973dbd96d887fc1160521feaa84372697dfbe812043bc9f11763cfa03551b0095bec43f284fd2fa5029f5590321668fd43bb31229dfcf73d937a4777

data/.gitattributes

@@ -0,0 +1 @@
+*.dat filter=lfs diff=lfs merge=lfs -text

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-geoip-kk.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2015 Yuri UMEZAKI
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,121 @@
+# fluent-plugin-geoip-kk
+
+[Fluentd](http://fluentd.org) filter plugin to add geoip.
+
+
+## Requirements
+
+| fluent-plugin-geoip-kk | fluentd    | ruby   |
+|----------------------------|------------|--------|
+| >= 1.0.0                   | >= v0.14.0 | >= 2.1 |
+| < 1.0.0                    | >= v0.12.0 | >= 1.9 |
+
+
+## Installation
+
+```bash
+# for fluentd
+$ gem install fluent-plugin-geoip-kk
+
+# for td-agent2
+$ sudo td-agent-gem install fluent-plugin-geoip-kk
+```
+
+
+## Usage
+
+### Example 1:
+
+```xml
+<filter access.nginx.**>
+  @type geoip
+  # key_name client_ip
+  # database_path /data/geoip/GeoLite2-City.mmdb
+  # out_key geo
+</filter>
+```
+
+Assuming following inputs are coming:
+
+```json
+access.nginx: {
+  "client_ip":"93.184.216.34",
+  "scheme":"http", "method":"GET", "host":"example.com",
+  "path":"/", "query":"-", "req_bytes":200, "referer":"-",
+  "status":200, "res_bytes":800, "res_body_bytes":600, "taken_time":0.001, "user_agent":"Mozilla/5.0"
+}
+```
+
+then output bocomes as belows:
+
+```json
+access.nginx: {
+  "client_ip":"93.184.216.34",
+  "scheme":"http", "method":"GET", "host":"example.com",
+  "path":"/", "query":"-", "req_bytes":200, "referer":"-",
+  "status":200, "res_bytes":800, "res_body_bytes":600, "taken_time":0.001, "user_agent":"Mozilla/5.0",
+  "geo": {
+    "coordinates": [-70.8228, 42.150800000000004],
+    "country_code": "US",
+    "city": "Norwell",
+    "region_code": "MA",
+  }
+}
+```
+
+
+## Parameters
+- key_name *field_key*
+
+    Target key name. default client_ip.
+
+- out_key *string*
+
+    Output prefix key name. default geo.
+
+- database_path *file_path*
+
+    Database file(GeoIPCity.dat) path.
+    Get from [MaxMind](http://dev.maxmind.com/geoip/legacy/geolite/)
+
+- flatten *bool*
+    join hashed data by '_'. default false.
+
+
+## VS.
+[fluent-plugin-geoip](https://github.com/y-ken/fluent-plugin-geoip)
+Fluentd output plugin to geolocate with geoip.
+It is able to customize fields with placeholder.
+
+* Easy to install.
+    * Not require to install Development Tools and geoip-dev library.
+    * ( fluent-plugin-geoip use geoip-c gem but our plugin use geoip. It's conflict. )
+* 5-10 times faster by the LRU cache.
+    * See [benchmark](test/bench_geoip_filter.rb).
+
+
+## TODO
+
+* patches welcome!
+
+
+## Contributing
+
+1. Fork it ( https://github.com/bungoume/fluent-plugin-geoip-filter/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+
+## Copyright
+
+Copyright (c) 2015 Yuri Umezaki
+
+
+## License
+
+[Apache License, Version 2.0.](http://www.apache.org/licenses/LICENSE-2.0)
+
+This product includes GeoLite data created by MaxMind, available from
+[http://www.maxmind.com](http://www.maxmind.com).

data/Rakefile

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+task default: :push
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :push do
+  system 'git add .'
+  system "git commit -m 'Update #{Time.now}.'"
+  system 'git pull'
+  system 'git push origin main'
+end

data/data/GeoLiteCity.dat

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25bce9e7f5fe9ffd12afa663c2e57b311a4286549a858965c4ff01596f604c8a
+size 19148749

data/fluent-plugin-geoip-kk.gemspec

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'fluent-plugin-geoip-kk'
+  spec.version       = '1.0.0'
+  spec.authors       = ['kevin197011']
+  spec.email         = ['kevin197011@outlook.com']
+  spec.homepage      = 'https://github.com/kevin197011/fluent-plugin-geoip-kk'
+  spec.summary       = 'Fluentd filter plugin to add geoip'
+  spec.description   = spec.summary
+  spec.license       = 'Apache-2.0'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
+  spec.add_runtime_dependency 'geoip', '>= 0.1.22'
+  spec.add_runtime_dependency 'lru_redux', '>= 1.0.0'
+  spec.add_runtime_dependency 'maxminddb', '>= 1.5.0'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'test-unit' if defined?(RUBY_VERSION) && RUBY_VERSION > '2.2'
+end

data/lib/fluent/plugin/filter_geoip.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'maxminddb'
+require 'lru_redux'
+require 'fluent/plugin/filter'
+
+module Fluent
+  module Plugin
+    class GeoipFilter < Filter
+      Fluent::Plugin.register_filter('geoip', self)
+
+      def initialize
+        @geoip_cache = LruRedux::Cache.new(8192)
+        super
+      end
+
+      config_param :database_path, :string, default: "#{File.dirname(__FILE__)}/../../../data/GeoLite2-City.mmdb"
+      config_param :key_name, :string, default: 'client_ip'
+      config_param :out_key, :string, default: 'geo'
+      config_param :flatten, :bool, default: false
+
+      def configure(conf)
+        super
+        begin
+          @geoip = MaxMindDB.new(@database_path)
+        rescue StandardError => e
+          log.warn 'Failed to configure parser. Use default pattern.', error_class: e.class, error: e.message
+          log.warn_backtrace
+        end
+      end
+
+      def filter(_tag, _time, record)
+        ip_addr = record[@key_name]
+
+        return record if ip_addr.nil? || ip_addr == '-'
+
+        geo_ip = @geoip_cache.getset(ip_addr) { get_geoip(ip_addr) }
+
+        if flatten
+          record.merge! hash_flatten(geo_ip, [@out_key])
+        else
+          record[@out_key] = geo_ip
+        end
+
+        record
+      end
+
+      private
+
+      def get_geoip(ip_addr)
+        geo_ip = @geoip.lookup(ip_addr)
+        data = {}
+        return data if geo_ip.nil? || ip_addr == '-'
+
+        data['coordinates'] = [geo_ip.location.longitude, geo_ip.location.latitude] if geo_ip.location
+        data['country_code'] = geo_ip.country.iso_code if geo_ip.country
+        data['city'] = geo_ip.city.name if geo_ip.city
+        data['region_code'] = geo_ip.subdivisions.first.iso_code if geo_ip.subdivisions.any?
+        data
+      end
+
+      def hash_flatten(a, keys = [])
+        ret = {}
+        a.each do |k, v|
+          ks = keys + [k]
+          if v.instance_of?(Hash)
+            ret.merge!(hash_flatten(v, ks))
+          else
+            ret.merge!({ ks.join('_') => v })
+          end
+        end
+        ret
+      end
+    end
+  end
+end

data/test/bench_geoip_filter.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative 'helper'
+require 'fluent/plugin/filter_geoip'
+
+# setup
+Fluent::Test.setup
+config = %(
+  @type geoip
+  out_key geo
+)
+time = Time.now.to_i
+tag = 'foo.bar'
+driver = Fluent::Test::FilterTestDriver.new(Fluent::GeoipFilter, tag).configure(config, true)
+
+# bench
+require 'benchmark'
+require 'ipaddr'
+n = 100_000
+Benchmark.bm(7) do |x|
+  # 8192 random IP
+  x.report do
+    driver.run do
+      n.times do
+        driver.emit({ 'client_ip' => IPAddr.new(rand(2**13) * 2**19, Socket::AF_INET).to_s }, time)
+      end
+    end
+  end
+end
+
+# Without LRU cache
+#              user     system      total        real
+#         11.410000   2.730000  14.140000 ( 15.431248)
+# With LRU cache(8192) & random  1024 IP
+#              user     system      total        real
+#          1.250000   0.070000   1.320000 (  1.322339)
+# With LRU cache(8192) & random  8192 IP
+#              user     system      total        real
+#          1.890000   0.210000   2.100000 (  2.102445)
+# With LRU cache(8192) & random 16384 IP
+#              user     system      total        real
+#          8.450000   2.070000  10.520000 ( 12.170379)
+# With LRU cache(8192) & random 65536 IP
+#              user     system      total        real
+#         11.890000   2.820000  14.710000 ( 16.051674)
+
+# fluent-plugin-geoip
+# (https://github.com/y-ken/fluent-plugin-geoip)
+#              user     system      total        real
+#         11.540000   0.270000  11.810000 ( 12.685000)

data/test/helper.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  warn e.message
+  warn 'Run `bundle install` to install missing gems'
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval do |_obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  end
+  $log = nulllogger
+end
+require 'fluent/test/helpers'
+require 'fluent/test/driver/filter'
+require 'fluent/plugin/filter_geoip'
+
+module Test
+  module Unit
+    class TestCase
+      include Fluent::Test::Helpers
+    end
+  end
+end

data/test/plugin/test_filter_geoip.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'helper'
+
+class GeoipFilterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %(
+    @type geoip
+    key_name client_ip
+    out_key geo
+  )
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::GeoipFilter).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(CONFIG)
+    assert_equal 'client_ip', d.instance.config['key_name']
+    assert_equal 'geo', d.instance.config['out_key']
+  end
+
+  def test_emit
+    d1 = create_driver(CONFIG)
+    ip_iddr = '93.184.216.34'
+
+    d1.run(default_tag: 'test') do
+      d1.feed({ 'client_ip' => ip_iddr })
+    end
+    emits = d1.filtered
+    assert_equal 1, emits.length
+    geo_object = emits[0][1]['geo']
+    assert_equal [-70.8228, 42.150800000000004], geo_object['coordinates']
+    assert_equal 'US', geo_object['country_code']
+    assert_equal 'Norwell', geo_object['city']
+    assert_equal 'MA', geo_object['region_code']
+  end
+
+  def test_emit_flatten
+    d1 = create_driver(%(
+      @type geoip
+      key_name ip_iddr
+      flatten
+    ))
+    ip_iddr = '93.184.216.34'
+
+    d1.run(default_tag: 'test') do
+      d1.feed({ 'ip_iddr' => ip_iddr })
+    end
+
+    emits = d1.filtered
+    assert_equal 1, emits.length
+    geo_object = emits[0][1]
+    assert_equal [-70.8228, 42.150800000000004], geo_object['geo_coordinates']
+    assert_equal 'US', geo_object['geo_country_code']
+    assert_equal 'Norwell', geo_object['geo_city']
+    assert_equal 'MA', geo_object['geo_region_code']
+  end
+end

data/test.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'maxminddb'
+
+db = MaxMindDB.new('data/GeoLite2-City.mmdb')
+
+ip = '128.101.101.101'
+
+begin
+  result = db.lookup(ip)
+
+  puts "IP: #{ip}"
+  puts "Country: #{result.country.name}"
+  puts "City: #{result.city.name}" if result.city.name
+  puts "Latitude: #{result.location.latitude}" if result.location.latitude
+  puts "Longitude: #{result.location.longitude}" if result.location.longitude
+  puts result.inspect
+rescue MaxMindDB::AddressNotFoundError
+  puts 'IP address not found in the database.'
+end

metadata

@@ -0,0 +1,164 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-geoip-kk
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- kevin197011
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2024-12-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: geoip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.22
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.22
+- !ruby/object:Gem::Dependency
+  name: lru_redux
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: maxminddb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd filter plugin to add geoip
+email:
+- kevin197011@outlook.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitattributes"
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- data/GeoLite2-City.mmdb
+- data/GeoLiteCity.dat
+- fluent-plugin-geoip-kk.gemspec
+- lib/fluent/plugin/filter_geoip.rb
+- test.rb
+- test/bench_geoip_filter.rb
+- test/helper.rb
+- test/plugin/test_filter_geoip.rb
+homepage: https://github.com/kevin197011/fluent-plugin-geoip-kk
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.18
+signing_key: 
+specification_version: 4
+summary: Fluentd filter plugin to add geoip
+test_files:
+- test/bench_geoip_filter.rb
+- test/helper.rb
+- test/plugin/test_filter_geoip.rb