fluent-plugin-filter-geoip 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 05bf2b4b2cdf3db890158b53beead941fb448a2e
-  data.tar.gz: a86f457353eb3a6c16131819559c952ff345e50a
+  metadata.gz: 592c7a6c492a234c331d225cf58de2d35c669a03
+  data.tar.gz: b59b98a1889229052058739831f17ce4516df099
 SHA512:
-  metadata.gz: 8bc1d5235b97701a5d2373d712abdc274cd25bffedd13b035135305bcf5e20b7e59ad0287f28280df1cd8778a9527ef96fae9c7dd1a06b167dd9af220bf74c5c
-  data.tar.gz: 09325fe6bf9b038928a594ad461bbf100ef3e13c9c58e46577141649e7f1728b61c547dfa6d81bb4a8146e6e083194ce8f59734e33baaf2de640a14cbe3d10ed
+  metadata.gz: d723fa0a431a2eb320b59d5012e72153ab6e59786422411c965dbfc7e43a5f566663b437eaf660caa9d30f5ad5798888aa545c04538a7d4054fbf96d0c616292
+  data.tar.gz: 72919a44209008f6fb1d5ca9bf7bf58fadb5f1ce5827dd87f73a89082f318096bde0da9b0409748d9ba42f9167e4cdc271f257b78ad83a1a72d9dc71eb697cf3

data/README.md

@@ -7,7 +7,7 @@ This is a [Fluentd](http://fluentd.org/) filter plugin for adding [GeoIP data](h
 Add this line to your application's Gemfile:
 
 ```
-gem 'fluent-plugin-output-solr'
+gem 'fluent-plugin-output-geoip'
 ```
 
 And then execute:
@@ -16,14 +16,32 @@ And then execute:
 
 Or install it yourself as:
 
-    $ gem install fluent-plugin-output-solr
+    $ gem install fluent-plugin-output-geoip
 
 ## Config parameters
 
-### database_pash
+### download_url
 
 ```
-database_path /path/to/GeoLite2-City.mmdb
+download_url http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz
+```
+
+### md5_url
+
+```
+md5_url http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.md5
+```
+
+### database_path
+
+```
+database_path ./geoip/database/GeoLite2-City.mmdb
+```
+
+### md5_path
+
+```
+md5_path ./geoip/database/GeoLite2-City.md5
 ```
 
 ### lookup_field
@@ -116,7 +134,11 @@ connection_type true
 <filter tail.log>
   @type geoip
 
-  database_path /path/to/GeoLite2-City.mmdb
+  download_url http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz
+  md5_url http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.md5
+  database_path ./geoip/database/GeoLite2-City.mmdb
+  md5_path ./geoip/database/GeoLite2-City.md5
+
   lookup_field host
   field_prefix geoip
   field_delimiter _
@@ -135,6 +157,97 @@ connection_type true
 </filter>
 ```
 
+Assuming following inputs are coming:
+
+```javascript
+{
+  "host":"212.99.123.25",
+  "user":"-",
+  "method":"GET",
+  "path":"/item/sports/4981",
+  "code":"200",
+  "size":"94",
+  "referer":"/category/electronics",
+  "agent":"Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1"
+}
+```
+
+then output bocomes as belows:
+
+```javascript
+{
+  "host":"212.99.123.25",
+  "user":"-",
+  "method":"GET",
+  "path":"/item/sports/4981",
+  "code":"200",
+  "size":"94",
+  "referer":"/category/electronics",
+  "agent":"Mozilla/5.0 (Windows NT 6.0; rv:10.0.1) Gecko/20100101 Firefox/10.0.1",
+  "geoip_continent_code":"EU",
+  "geoip_continent_geoname_id":6255148,
+  "geoip_continent_names_de":"Europa",
+  "geoip_continent_names_en":"Europe",
+  "geoip_continent_names_es":"Europa",
+  "geoip_continent_names_fr":"Europe",
+  "geoip_continent_names_ja":"ヨーロッパ",
+  "geoip_continent_names_pt-BR":"Europa",
+  "geoip_continent_names_ru":"Европа",
+  "geoip_continent_names_zh-CN":"欧洲",
+  "geoip_country_geoname_id":3017382,
+  "geoip_country_iso_code":"FR",
+  "geoip_country_names_de":"Frankreich",
+  "geoip_country_names_en":"France",
+  "geoip_country_names_es":"Francia",
+  "geoip_country_names_fr":"France",
+  "geoip_country_names_ja":"フランス共和国",
+  "geoip_country_names_pt-BR":"França",
+  "geoip_country_names_ru":"Франция",
+  "geoip_country_names_zh-CN":"法国",
+  "geoip_city_geoname_id":3038354,
+  "geoip_city_names_de":"Aix-en-Provence",
+  "geoip_city_names_en":"Aix-en-Provence",
+  "geoip_city_names_es":"Aix-en-Provence",
+  "geoip_city_names_fr":"Aix-en-Provence",
+  "geoip_city_names_ja":"エクス＝アン＝プロヴァンス",
+  "geoip_city_names_pt-BR":"Aix-en-Provence",
+  "geoip_city_names_ru":"Экс-ан-Прованс",
+  "geoip_city_names_zh-CN":"普罗旺斯地区艾克斯",
+  "geoip_location_latitude":43.5283,
+  "geoip_location_longitude":5.4497,
+  "geoip_location_time_zone":"Europe/Paris",
+  "geoip_postal_code":"13090",
+  "geoip_registered_country_geoname_id":3017382,
+  "geoip_registered_country_iso_code":"FR",
+  "geoip_registered_country_names_de":"Frankreich",
+  "geoip_registered_country_names_en":"France",
+  "geoip_registered_country_names_es":"Francia",
+  "geoip_registered_country_names_fr":"France",
+  "geoip_registered_country_names_ja":"フランス共和国",
+  "geoip_registered_country_names_pt-BR":"França",
+  "geoip_registered_country_names_ru":"Франция",
+  "geoip_registered_country_names_zh-CN":"法国",
+  "geoip_subdivisions_0_geoname_id":2985244,
+  "geoip_subdivisions_0_iso_code":"U",
+  "geoip_subdivisions_0_names_de":"Provence-Alpes-Côte d’Azur",
+  "geoip_subdivisions_0_names_en":"Provence-Alpes-Côte d'Azur",
+  "geoip_subdivisions_0_names_es":"Provenza-Alpes-Costa Azul",
+  "geoip_subdivisions_0_names_fr":"Provence-Alpes-Côte d'Azur",
+  "geoip_subdivisions_0_names_ja":"プロヴァンス＝アルプ＝コート・ダジュール地域圏",
+  "geoip_subdivisions_0_names_pt-BR":"Provença-Alpes-Costa Azul",
+  "geoip_subdivisions_0_names_ru":"Прованс — Альпы — Лазурный Берег",
+  "geoip_subdivisions_0_names_zh-CN":"普罗旺斯-阿尔卑斯-蓝色海岸",
+  "geoip_subdivisions_1_geoname_id":3031359,
+  "geoip_subdivisions_1_iso_code":"13",
+  "geoip_subdivisions_1_names_de":"Bouches-du-Rhône",
+  "geoip_subdivisions_1_names_en":"Bouches-du-Rhône",
+  "geoip_subdivisions_1_names_es":"Bocas del Ródano",
+  "geoip_subdivisions_1_names_fr":"Bouches-du-Rhône",
+  "geoip_subdivisions_1_names_pt-BR":"Bocas do Ródano",
+  "geoip_localtion_latlon":"43.5283,5.4497"
+}
+```
+
 ## Development
 
 After checking out the repo, run `bundle install` to install dependencies. Then, run `rake test` to run the tests.

data/fluent-plugin-filter-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-filter-geoip"
-  spec.version       = "0.2.2"
+  spec.version       = "0.2.3"
   spec.authors       = ["Minoru Osuka"]
   spec.email         = ["minoru.osuka@gmail.com"]
 

data/lib/fluent/plugin/filter_geoip.rb

@@ -1,10 +1,17 @@
 require 'maxminddb'
 require 'json'
+require 'fileutils'
+require 'open-uri'
 
 module Fluent
   class GeoIPFilter < Filter
     Fluent::Plugin.register_filter('geoip', self)
 
+    DEFAULT_DOWNLOAD_URL = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz'
+    DEFAULT_MD5_URL = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.md5'
+    DEFAULT_DATABASE_PATH = './geoip/database/GeoLite2-City.mmdb'
+    DEFAULT_MD5_PATH = './geoip/database/GeoLite2-City.md5'
+
     DEFAULT_LOOKUP_FIELD = 'ip'
     DEFAULT_FIELD_PREFIX = 'geoip'
     DEFAULT_FIELD_DELIMITER = '_'
@@ -21,7 +28,16 @@ module Fluent
     DEFAULT_TRAITS = true
     DEFAULT_CONNECTION_TYPE = true
 
-    config_param :database_path, :string, :default => nil,
+    config_param :download_url, :string, :default => DEFAULT_DOWNLOAD_URL,
+                 :desc => ''
+
+    config_param :md5_url, :string, :default => DEFAULT_MD5_URL,
+                 :desc => ''
+
+    config_param :database_path, :string, :default => DEFAULT_DATABASE_PATH,
+                 :desc => ''
+
+    config_param :md5_path, :string, :default => DEFAULT_MD5_PATH,
                  :desc => ''
 
     config_param :lookup_field, :string, :default => DEFAULT_LOOKUP_FIELD,
@@ -73,7 +89,13 @@ module Fluent
     def configure(conf)
       super
 
-      @database_path = conf['database_path']
+      @download_url = conf.has_key?('download_url') ? conf['download_url'] : DEFAULT_DOWNLOAD_URL
+
+      @md5_url = conf.has_key?('md5_url') ? conf['md5_url'] : DEFAULT_MD5_URL
+
+      @database_path = conf.has_key?('database_path') ? conf['database_path'] : DEFAULT_DATABASE_PATH
+
+      @md5_path = conf.has_key?('md5_path') ? conf['md5_path'] : DEFAULT_MD5_PATH
 
       @lookup_field = conf.has_key?('lookup_field') ? conf['lookup_field'] : DEFAULT_LOOKUP_FIELD
 
@@ -103,6 +125,8 @@ module Fluent
 
       @connection_type = conf.has_key?('connection_type') ? to_boolean(conf['connection_type']) : DEFAULT_CONNECTION_TYPE
 
+      download_database @download_url, @md5_url, @database_path, @md5_path
+
       @database = MaxMindDB.new(@database_path)
     end
 
@@ -234,6 +258,75 @@ module Fluent
       else
         return false
       end
-    end    
+    end
+
+    def download_database(download_url, md5_url, database_path, md5_path)
+      # database directory
+      database_dir = File.expand_path(File.dirname(database_path))
+      md5_dir = File.expand_path(File.dirname(md5_path))
+
+      # create database directory if directory does not exist.
+      FileUtils.mkdir_p(database_dir) unless File.exist?(database_dir)
+      FileUtils.mkdir_p(md5_dir) unless File.exist?(md5_dir)
+
+      # create empty md5 file if file does not exist.
+      File.open(md5_path, 'wb').close() unless File.exist?(md5_path)
+
+      # read saved md5
+      saved_md5 = nil
+      begin
+        open(md5_path, 'rb') do |data|
+          saved_md5 = data.read
+        end
+      rescue => e
+        log.warn e.message
+      end
+
+      # fetch md5
+      fetched_md5 = nil
+      begin
+        open(md5_url, 'rb') do |data|
+          fetched_md5 = data.read
+        end
+      rescue => e
+        log.warn e.message
+      end
+
+      # check md5
+      unless saved_md5 == fetched_md5 then
+        # download new database
+        download_path = database_dir + '/' + File.basename(download_url)
+        begin
+          open(download_path, 'wb') do |output|
+            open(download_url, 'rb') do |data|
+              output.write(data.read)
+            end
+          end
+        rescue => e
+          log.warn e.message
+        end
+
+        # unzip new database temporaly
+        tmp_database_path = database_dir + '/tmp_' + File.basename(database_path)
+        begin
+          open(tmp_database_path, 'wb') do |output|
+            Zlib::GzipReader.open(download_path) do |gz|
+              output.write(gz.read)
+            end
+          end
+        rescue => e
+          puts e.message
+        end
+
+        # check mkd5
+        temp_md5 = Digest::MD5.hexdigest(File.open(tmp_database_path, 'rb').read)
+        if fetched_md5 == temp_md5 then
+          FileUtils.mv(tmp_database_path, database_path)
+
+          # record new md5
+          File.write(md5_path, fetched_md5)
+        end
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-filter-geoip
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Minoru Osuka
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-18 00:00:00.000000000 Z
+date: 2016-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd