fluent-plugin-geoip-filter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 69bba19629edca4fda5e3e8fab24f2373a3421e5
+  data.tar.gz: 5252200643db4bb6ec7262d64805f8f0c9b56b93
+SHA512:
+  metadata.gz: c589ab0d76dab39b6a2f4b64141e683540d835ad3ded57410e65d6daf0e4d478e464c0f79cc0bf18695ae26593a8fd7f95c479a8ee269cf78040e31c606c0946
+  data.tar.gz: 4d63b2023b783a1175a5b97bd0c8f91c13c7b9673fe9742338dd9a3d1372e76c92f0b946ad60f2ca93ef004a254dbf19891d30b6fa84eecdddf6d70c8f3567d1

data/.gitattributes

@@ -0,0 +1 @@
+*.dat filter=lfs diff=lfs merge=lfs -text

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,11 @@
+language: ruby
+
+rvm:
+  - 2.2
+  - 2.0
+  - 1.9.3
+
+before_script:
+  - wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
+  - gunzip GeoLiteCity.dat.gz
+  - mv GeoLiteCity.dat data/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-geoip-filter.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2015 Yuri UMEZAKI
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,102 @@
+# fluent-plugin-geoip-filter
+
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-geoip-filter.svg)](https://badge.fury.io/rb/fluent-plugin-geoip-filter)
+[![Build Status](https://travis-ci.org/bungoume/fluent-plugin-geoip-filter.svg?branch=master)](https://travis-ci.org/bungoume/fluent-plugin-geoip-filter)
+[![Dependency Status](https://gemnasium.com/bungoume/fluent-plugin-geoip-filter.svg)](https://gemnasium.com/bungoume/fluent-plugin-geoip-filter)
+
+[Fluentd](http://fluentd.org) filter plugin to add geoip.
+
+
+## Installation
+
+```bash
+# for fluentd
+$ gem install fluent-plugin-geoip-filter
+
+# for td-agent2
+$ sudo td-agent-gem install fluent-plugin-geoip-filter
+```
+
+
+## Usage
+
+### Example 1:
+
+```xml
+<filter access.nginx.**>
+  @type geoip
+</filter>
+```
+
+Assuming following inputs are coming:
+
+```json
+access.nginx: {
+  "client_ip":"93.184.216.34",
+  "scheme":"http", "method":"GET", "host":"example.com",
+  "path":"/", "query":"-", "req_bytes":200, "referer":"-",
+  "status":200, "res_bytes":800, "res_body_bytes":600, "taken_time":0.001, "user_agent":"Mozilla/5.0"
+}
+```
+
+then output bocomes as belows:
+
+```json
+access.nginx: {
+  "client_ip":"93.184.216.34",
+  "scheme":"http", "method":"GET", "host":"example.com",
+  "path":"/", "query":"-", "req_bytes":200, "referer":"-",
+  "status":200, "res_bytes":800, "res_body_bytes":600, "taken_time":0.001, "user_agent":"Mozilla/5.0",
+  "geo": {
+    "coordinates": [-70.8228, 42.150800000000004],
+    "country_code": "US",
+    "city": "Norwell",
+    "region_code": "MA",
+  }
+}
+```
+
+
+## Parameters
+- key_name *field_key*
+
+    Target key name. default client_ip.
+
+- out_key *string*
+
+    Output prefix key name. default geo.
+
+- database_path *file_path*
+
+    Database file(GeoIPCity.dat) path.
+    Get from [MaxMind](http://dev.maxmind.com/geoip/legacy/geolite/)
+
+- flatten *bool*
+    join hashed data by '_'. default false.
+
+
+## TODO
+
+* patches welcome!
+
+
+## Contributing
+
+1. Fork it ( https://github.com/bungoume/fluent-plugin-geoip-filter/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+
+## Copyright
+
+Copyright (c) 2015 Yuri Umezaki
+
+
+## License
+
+[Apache License, Version 2.0.](http://www.apache.org/licenses/LICENSE-2.0)
+
+This product includes GeoLite data created by MaxMind, available from 
+[http://www.maxmind.com](http://www.maxmind.com).

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-geoip-filter.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-geoip-filter"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Yuri Umezaki"]
+  spec.email         = ["bungoume@gmail.com"]
+  spec.homepage      = "https://github.com/bungoume/fluent-plugin-geoip-filter"
+  spec.summary       = "Fluentd filter plugin to add geoip"
+  spec.description   = spec.summary
+  spec.license       = "Apache-2.0"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "fluentd", ">= 0.12"
+  spec.add_runtime_dependency "geoip", ">= 1.5.0"
+  spec.add_runtime_dependency "lru_redux", ">= 1.0.0"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  if defined?(RUBY_VERSION) && RUBY_VERSION > '2.2'
+    spec.add_development_dependency "test-unit"
+  end
+end

data/lib/fluent/plugin/filter_geoip.rb

@@ -0,0 +1,69 @@
+require 'geoip'
+require 'lru_redux'
+
+module Fluent
+  class GeoipFilter < Filter
+    Plugin.register_filter('geoip', self)
+
+    def initialize
+      @geoip_cache = LruRedux::Cache.new(10000)
+      super
+    end
+
+    config_param :database_path, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
+    config_param :key_name, :string, :default => 'client_ip'
+    config_param :out_key, :string, :default => 'geo'
+    config_param :flatten, :bool, :default => false
+
+    def configure(conf)
+      super
+      begin
+        @geoip = GeoIP.new(@database_path)
+      rescue => e
+        @geoip = GeoIP.new
+        log.warn "Failed to configure parser. Use default pattern.", :error_class => e.class, :error => e.message
+        log.warn_backtrace
+      end
+    end
+
+    def filter(tag, time, record)
+      ip_addr = record[@key_name]
+      unless ip_addr.nil?
+        geo_ip = @geoip_cache.getset(ip_addr) { get_geoip(ip_addr) }
+        if flatten
+          record.merge! hash_flatten(geo_ip, [@out_key])
+        else
+          record[@out_key] = geo_ip
+        end
+      end
+      record
+    end
+
+    private
+
+    def get_geoip(ip_addr)
+      geo_ip = @geoip.city(ip_addr)
+      data = {}
+      return data if geo_ip.nil?
+      data["coordinates"] = [geo_ip["longitude"], geo_ip["latitude"]]
+      data["country_code"] = geo_ip["country_code2"]
+      data["city"] = geo_ip["city_name"]
+      data["region_code"] = geo_ip["region_name"]
+      data
+    end
+
+    def hash_flatten(a, keys=[])
+      ret = {}
+      a.each{|k,v|
+        ks = keys + [k]
+        if v.class == Hash
+          ret.merge!(hash_flatten(v, ks))
+        else
+          ret.merge!({ks.join('_')=> v})
+        end
+      }
+      ret
+    end
+
+  end if defined?(Filter) # Support only >= v0.12
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/filter_geoip'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_filter_geoip.rb

@@ -0,0 +1,63 @@
+require 'helper'
+
+class GeoipFilterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    type geoip
+    key_name client_ip
+    out_key geo
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::FilterTestDriver.new(Fluent::GeoipFilter, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(CONFIG)
+    assert_equal 'client_ip', d.instance.config['key_name']
+    assert_equal 'geo', d.instance.config['out_key']
+  end
+
+  def test_emit
+    d1 = create_driver(CONFIG)
+    ip_iddr = '93.184.216.34'
+
+    d1.run do
+      d1.emit({'client_ip' => ip_iddr})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 'test', emits[0][0] # tag
+    geo_object = emits[0][2]['geo']
+    assert_equal [-70.8228, 42.150800000000004], geo_object['coordinates']
+    assert_equal 'US', geo_object['country_code']
+    assert_equal 'Norwell', geo_object['city']
+    assert_equal 'MA', geo_object['region_code']
+  end
+
+  def test_emit_flatten
+    d1 =     d1 = create_driver(%[
+      type geoip
+      key_name ip_iddr
+      flatten
+    ], 'test')
+    ip_iddr = '93.184.216.34'
+
+    d1.run do
+      d1.emit({'ip_iddr' => ip_iddr})
+    end
+
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 'test', emits[0][0] # tag
+    geo_object = emits[0][2]
+    assert_equal [-70.8228, 42.150800000000004], geo_object['geo_coordinates']
+    assert_equal 'US', geo_object['geo_country_code']
+    assert_equal 'Norwell', geo_object['geo_city']
+    assert_equal 'MA', geo_object['geo_region_code']
+  end
+
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-geoip-filter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Yuri Umezaki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: geoip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: lru_redux
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd filter plugin to add geoip
+email:
+- bungoume@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitattributes"
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- data/GeoLiteCity.dat
+- fluent-plugin-geoip-filter.gemspec
+- lib/fluent/plugin/filter_geoip.rb
+- test/helper.rb
+- test/plugin/test_filter_geoip.rb
+homepage: https://github.com/bungoume/fluent-plugin-geoip-filter
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Fluentd filter plugin to add geoip
+test_files:
+- test/helper.rb
+- test/plugin/test_filter_geoip.rb