fluent-plugin-filter-geoip 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 983e1b2b72f93c02932cb24604377948fc452e71
-  data.tar.gz: 96b9b03b2ddd5116b50077ecf339554b128d594d
+  metadata.gz: 6af5d9d9d7027e164b87e91b793baad6f2feecd4
+  data.tar.gz: 1f82bceeab8a2a4111d245645636f704b5bbffc2
 SHA512:
-  metadata.gz: c443452ffe44be24449954a8c41e6f2e3e17a06e0ddcea181c1c017958b1193cc15590251da3cc840eec5dc10813ae284218a58d0d5fd4fa7461d24257edcf97
-  data.tar.gz: 1981eb0d8756276805fb46f23475da8a5ccc80dbf64ea56d2326e58f3ef2b223a2f28f69bdc7a854f9527eff37c760bc8f8b624901ff75ede3a9734e6166d72f
+  metadata.gz: 4796d4b9ebad60ccf9582eda74a0f63ea9f5a5d5060e39ffa479d87574a4b34ed769e3f9fd92d24449cd0e10fe4de4b7def362497c9fc0e8cdc1a3d3fad05a93
+  data.tar.gz: 086f1ac37d6c7bad1634a18c478671746af33d50c078a690ff2476f372c4f5f1ec92a8c7201eb214084f506f1d30de6869d015ec640b987ee1505d09d09951ad

data/README.md

@@ -32,10 +32,16 @@ database_path /path/to/GeoLite2-City.mmdb
 lookup_field host
 ```
 
-### output_field
+### field_prefix
 
 ```
-output_field geoip
+field_prefix geoip
+```
+
+### field_delimiter
+
+```
+field_delimiter _
 ```
 
 ### flatten
@@ -44,6 +50,66 @@ output_field geoip
 flatten true
 ```
 
+### continent
+
+```
+continent true
+```
+
+### country
+
+```
+country true
+```
+
+### city
+
+```
+city true
+```
+
+### location
+
+```
+location true
+```
+
+### postal
+
+```
+postal true
+```
+
+### registered_country
+
+```
+registered_country true
+```
+
+### represented_country
+
+```
+represented_country true
+```
+
+### subdivisions
+
+```
+subdivisions true
+```
+
+### traits
+
+```
+traits true
+```
+
+### connection_type
+
+```
+connection_type true
+```
+
 ## Plugin setup examples
 
 ```
@@ -52,8 +118,20 @@ flatten true
 
   database_path /path/to/GeoLite2-City.mmdb
   lookup_field host
-  output_field geoip
+  field_prefix geoip
+  field_delimiter _
   flatten true
+
+  continent true
+  country true
+  city true
+  location true
+  postal true
+  registered_country true
+  represented_country true
+  subdivisions true
+  traits true
+  connection_type true
 </filter>
 ```
 

data/fluent-plugin-filter-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-filter-geoip"
-  spec.version       = "0.1.0"
+  spec.version       = "0.2.0"
   spec.authors       = ["Minoru Osuka"]
   spec.email         = ["minoru.osuka@gmail.com"]
 

data/lib/fluent/plugin/filter_geoip.rb

@@ -1,25 +1,71 @@
 require 'maxminddb'
+require 'json'
 
 module Fluent
   class GeoIPFilter < Filter
     Fluent::Plugin.register_filter('geoip', self)
 
     DEFAULT_LOOKUP_FIELD = 'ip'
-    DEFAULT_OUTPUT_FIELD = 'geoip'
+    DEFAULT_FIELD_PREFIX = 'geoip'
+    DEFAULT_FIELD_DELIMITER = '_'
     DEFAULT_FLATTEN = false
 
+    DEFAULT_CITY = true
+    DEFAULT_CONTINENT = true
+    DEFAULT_COUNTRY = true
+    DEFAULT_LOCATION = true
+    DEFAULT_POSTAL = true
+    DEFAULT_REGISTERED_COUNTRY = true
+    DEFAULT_REPRESENTED_COUNTRY = true
+    DEFAULT_SUBDIVISIONS = true
+    DEFAULT_TRAITS = true
+    DEFAULT_CONNECTION_TYPE = true
+
     config_param :database_path, :string, :default => nil,
                  :desc => ''
 
     config_param :lookup_field, :string, :default => DEFAULT_LOOKUP_FIELD,
                  :desc => ''
 
-    config_param :output_field, :string, :default => DEFAULT_OUTPUT_FIELD,
+    config_param :field_prefix, :string, :default => DEFAULT_FIELD_PREFIX,
+                 :desc => ''
+
+    config_param :field_delimiter, :string, :default => DEFAULT_FIELD_DELIMITER,
                  :desc => ''
 
     config_param :flatten, :bool, :default => DEFAULT_FLATTEN,
                  :desc => ''
 
+    config_param :continent, :bool, :default => DEFAULT_CONTINENT,
+                 :desc => ''
+
+    config_param :country, :bool, :default => DEFAULT_COUNTRY,
+                 :desc => ''
+
+    config_param :city, :bool, :default => DEFAULT_CITY,
+                 :desc => ''
+
+    config_param :location, :bool, :default => DEFAULT_LOCATION,
+                 :desc => ''
+
+    config_param :postal, :bool, :default => DEFAULT_POSTAL,
+                 :desc => ''
+
+    config_param :registered_country, :bool, :default => DEFAULT_REGISTERED_COUNTRY,
+                 :desc => ''
+
+    config_param :represented_country, :bool, :default => DEFAULT_REPRESENTED_COUNTRY,
+                 :desc => ''
+
+    config_param :subdivisions, :bool, :default => DEFAULT_SUBDIVISIONS,
+                 :desc => ''
+
+    config_param :traits, :bool, :default => DEFAULT_TRAITS,
+                 :desc => ''
+
+    config_param :connection_type, :bool, :default => DEFAULT_CONNECTION_TYPE,
+                 :desc => ''
+
     def initialize
       super
     end
@@ -31,9 +77,31 @@ module Fluent
 
       @lookup_field = conf.has_key?('lookup_field') ? conf['lookup_field'] : DEFAULT_LOOKUP_FIELD
 
-      @output_field = conf.has_key?('output_field') ? conf['output_field'] : DEFAULT_OUTPUT_FIELD
+      @field_prefix = conf.has_key?('field_prefix') ? conf['field_prefix'] : DEFAULT_FIELD_PREFIX
+
+      @field_delimiter = conf.has_key?('field_delimiter') ? conf['field_delimiter'] : DEFAULT_FIELD_DELIMITER
+
+      @flatten = conf.has_key?('flatten') ? to_boolean(conf['flatten']) : DEFAULT_FLATTEN
+
+      @continent = conf.has_key?('continent') ? to_boolean(conf['continent']) : DEFAULT_CONTINENT
+
+      @country = conf.has_key?('country') ? to_boolean(conf['country']) : DEFAULT_COUNTRY
+
+      @city = conf.has_key?('city') ? to_boolean(conf['city']) : DEFAULT_CITY
+
+      @location = conf.has_key?('location') ? to_boolean(conf['location']) : DEFAULT_LOCATION
+
+      @postal = conf.has_key?('postal') ? to_boolean(conf['postal']) : DEFAULT_POSTAL
+
+      @registered_country = conf.has_key?('registered_country') ? to_boolean(conf['registered_country']) : DEFAULT_REGISTERED_COUNTRY
 
-      @flatten = conf.has_key?('flatten') ? conf['flatten'] : DEFAULT_FLATTEN
+      @represented_country = conf.has_key?('represented_country') ? to_boolean(conf['represented_country']) : DEFAULT_REPRESENTED_COUNTRY
+
+      @subdivisions = conf.has_key?('subdivisions') ? to_boolean(conf['subdivisions']) : DEFAULT_SUBDIVISIONS
+
+      @traits = conf.has_key?('traits') ? to_boolean(conf['traits']) : DEFAULT_TRAITS
+
+      @connection_type = conf.has_key?('connection_type') ? to_boolean(conf['connection_type']) : DEFAULT_CONNECTION_TYPE
 
       @database = MaxMindDB.new(@database_path)
     end
@@ -45,12 +113,133 @@ module Fluent
         geoip = @database.lookup(ip)
 
         if geoip.found? then
-          log.info geoip
-          record[@output_field] = geoip.to_hash
+          geoip_hash = geoip.to_hash
+
+          if @continent && geoip_hash.has_key?('continent') then
+            if @flatten then
+              record.merge!(to_flatten(geoip_hash['continent'], [@field_prefix, 'continent'], @field_delimiter))
+            else
+              record[[@field_prefix, 'continent'].join(@field_delimiter)] = geoip_hash['continent'].to_json
+            end
+          end
+
+          if @country && geoip_hash.has_key?('country') then
+            if @flatten then
+              record.merge!(to_flatten(geoip_hash['country'], [@field_prefix, 'country'], @field_delimiter))
+            else
+              record[[@field_prefix, 'country'].join(@field_delimiter)] = geoip_hash['country'].to_json
+            end
+          end
+
+          if @city && geoip_hash.has_key?('city') then
+            if @flatten then
+              record.merge!(to_flatten(geoip_hash['city'], [@field_prefix, 'city'], @field_delimiter))
+            else
+              record[[@field_prefix, 'city'].join(@field_delimiter)] = geoip_hash['city'].to_json
+            end
+          end
+
+          if @location && geoip_hash.has_key?('location') then
+            if @flatten then
+              record.merge!(to_flatten(geoip_hash['location'], [@field_prefix, 'location'], @field_delimiter))
+            else
+              record[[@field_prefix, 'location'].join(@field_delimiter)] = geoip_hash['location'].to_json
+            end
+          end
+
+          if @postal && geoip_hash.has_key?('postal') then
+            if @flatten then
+              record.merge!(to_flatten(geoip_hash['postal'], [@field_prefix, 'postal'], @field_delimiter))
+            else
+              record[[@field_prefix, 'postal'].join(@field_delimiter)] = geoip_hash['postal'].to_json
+            end
+          end
+
+          if @registered_country && geoip_hash.has_key?('registered_country') then
+            if @flatten then
+              record.merge!(to_flatten(geoip_hash['registered_country'], [@field_prefix, 'registered_country'], @field_delimiter))
+            else
+              record[[@field_prefix, 'registered_country'].join(@field_delimiter)] = geoip_hash['registered_country'].to_json
+            end
+          end
+
+          if @represented_country && geoip_hash.has_key?('represented_country') then
+            if @flatten then
+              record.merge!(to_flatten(geoip_hash['represented_country'], [@field_prefix, 'represented_country'], @field_delimiter))
+            else
+              record[[@field_prefix, 'represented_country'].join(@field_delimiter)] = geoip_hash['represented_country'].to_json
+            end
+          end
+
+          if @subdivisions && geoip_hash.has_key?('subdivisions') then
+            if @flatten then
+              record.merge!(to_flatten(geoip_hash['subdivisions'], [@field_prefix, 'subdivisions'], @field_delimiter))
+            else
+              record[[@field_prefix, 'subdivisions'].join(@field_delimiter)] = geoip_hash['subdivisions'].to_json
+            end
+          end
+
+          if @traits && geoip_hash.has_key?('traits') then
+            if @flatten then
+              record.merge!(to_flatten(geoip_hash['traits'], [@field_prefix, 'traits'], @field_delimiter))
+            else
+              record[[@field_prefix, 'traits'].join(@field_delimiter)] = geoip_hash['traits'].to_json
+            end
+          end
+
+          if @connection_type && geoip_hash.has_key?('connection_type') then
+            if @flatten then
+              record.merge!(to_flatten(geoip_hash['connection_type'], [@field_prefix, 'connection_type'], @field_delimiter))
+            else
+              record[[@field_prefix, 'connection_type'].join(@field_delimiter)] = geoip_hash['connection_type'].to_json
+            end
+          end
+
+          log.info "Record: %s" % record.inspect
+        else
+          log.warn "It was not possible to look up the #{ip}."
+        end
+      end
+
+      return record
+    end
+
+    def to_flatten(hash, stack=[], delimiter='/')
+      output = {}
+
+      hash.keys.each do |key|
+        stack.push key
+
+        if hash[key].instance_of?(Hash) then
+          output.merge!(to_flatten(hash[key], stack, delimiter))
+        elsif hash[key].instance_of?(Array) then
+          i = 0
+          hash[key].each do |data|
+            stack.push i
+            if data.instance_of?(Hash) then
+              output.merge!(to_flatten(data, stack, delimiter))
+            end
+            i = i + 1
+            stack.pop
+          end
+        else
+          output[stack.join(delimiter)] = hash[key]
         end
 
+        stack.pop
       end
 
+      return output
     end
+
+    def to_boolean(string)
+      if string== true || string =~ (/(true|t|yes|y|1)$/i) then
+        return true
+      elsif string== false || string.nil? || string =~ (/(false|f|no|n|0)$/i)
+        return false
+      else
+        return false
+      end
+    end    
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-filter-geoip
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Minoru Osuka
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-17 00:00:00.000000000 Z
+date: 2016-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd