logstash-filter-geoip 2.0.3 → 2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e4fabe098f9ced70d73a826e6c9d85e50092b0ec
-  data.tar.gz: d73e071f69b23f2d19ad414a7b01e1d697c8f567
+  metadata.gz: 72506d5a7ea9264ac61e27b0b43fc845fc612d1a
+  data.tar.gz: e0086cf1590ddcf4158e254d3731d96d585ea90c
 SHA512:
-  metadata.gz: 1836cb3e6392c03892624d5c95fbfedeee16f50f99b0eeaeea63d0f1a3bf5b1ac72089902151a2cdfa5e2abd5e1ba907b24aadf6351f519c0ff504b01cfc7423
-  data.tar.gz: d7ca6a1a6e2745b599c5f2470e6ef023fff8d4bef0690d02feba9735a0e766bcea52465f65c3a1658e6cae1f666d2241a510c732d5fae198bb3bab1f4f8d3255
+  metadata.gz: f9903608de6535b87cfae39756fa5d774681ff7ab6ae0d279de1c514648cbec2dd4ca37b3e3dc120553b96efb0e95115f303cd4857f9b82bdc8107b02fd53891
+  data.tar.gz: 150e64d646233afba52aac986f1fcf1e514c26e2611f74d1f08039ced40cb0aaffd85c01f125ad4bd05c597d4611fe047730a25e4b11ca9fb0ff3beb0b19e2e6

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 2.0.4
+ - Refactor GeoIP Struct to hash conversion to minimise repeated manipulation
+
 ## 2.0.3
  - Fix Issue 50, incorrect data returned when geo lookup fails
 

data/lib/logstash/filters/geoip.rb

@@ -120,50 +120,32 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
     LOOKUP_CACHE_INIT_MUTEX.synchronize do
       self.lookup_cache = LOOKUP_CACHES[@geoip_type] ||= LruRedux::ThreadSafeCache.new(1000)
     end
+
+    @no_fields = @fields.nil? || @fields.empty?
   end # def register
 
   public
   def filter(event)
-
-    geo_data = nil
-
-    geo_data = get_geo_data(event)
-
-    # defense against GeoIP code returning something that can't be made a hash
-    return unless geo_data.respond_to?(:to_hash)
-
-    event[@target] = {} if event[@target].nil?
-    geo_data_hash = geo_data.to_hash
-    # don't do anything more if the lookup result is empty
-    if !geo_data_hash.empty?
-      apply_geodata(geo_data_hash, event)
+    geo_data_hash = get_geo_data(event)
+    if apply_geodata(geo_data_hash, event)
       filter_matched(event)
     end
   end # def filter
 
   def apply_geodata(geo_data_hash, event)
-    geo_data_hash.delete(:request)
-    if geo_data_hash.key?(:latitude) && geo_data_hash.key?(:longitude)
-      # If we have latitude and longitude values, add the location field as GeoJSON array
-      geo_data_hash[:location] = [ geo_data_hash[:longitude].to_f, geo_data_hash[:latitude].to_f ]
-    end
+    # don't do anything more if the lookup result is nil?
+    return false if geo_data_hash.nil?
+    # only set the event[@target] if the lookup result is not nil: BWC
+    event[@target] = {} if event[@target].nil?
+    # don't do anything more if the lookup result is empty?
+    return false if geo_data_hash.empty?
     geo_data_hash.each do |key, value|
-      next if value.nil? || (value.is_a?(String) && value.empty?)
-      if @fields.nil? || @fields.empty? || @fields.include?(key.to_s)
-        # convert key to string (normally a Symbol)
-        if value.is_a?(String)
-          # Some strings from GeoIP don't have the correct encoding...
-          value = case value.encoding
-                    # I have found strings coming from GeoIP that are ASCII-8BIT are actually
-                    # ISO-8859-1...
-                    when Encoding::ASCII_8BIT; value.force_encoding(Encoding::ISO_8859_1).encode(Encoding::UTF_8)
-                    when Encoding::ISO_8859_1, Encoding::US_ASCII;  value.encode(Encoding::UTF_8)
-                    else; value.dup
-                  end
-        end
-        event[@target][key.to_s] = value
+      if @no_fields || @fields.include?(key)
+        # can't dup numerics
+        event[@target][key] = value.is_a?(Numeric) ? value : value.dup
       end
     end # geo_data_hash.each
+    true
   end
 
   def get_geo_data(event)
@@ -171,6 +153,7 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
     result = {}
     ip = event[@source]
     ip = ip.first if ip.is_a? Array
+    return nil if ip.nil?
     begin
       result = get_geo_data_for_ip(ip)
     rescue SocketError => e
@@ -187,11 +170,45 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
       cached
     else
       geo_data = Thread.current[threadkey].send(@geoip_type, ip)
-      lookup_cache[ip] = geo_data
-      geo_data
+      converted = prepare_geodata_for_cache(geo_data)
+      lookup_cache[ip] = converted
+      converted
     end
   end
 
+  def prepare_geodata_for_cache(geo_data)
+    # GeoIP returns a nil or a Struct subclass
+    return nil if !geo_data.respond_to?(:each_pair)
+    #lets just do this once before caching
+    result = {}
+    geo_data.each_pair do |k, v|
+      next if v.nil? || k == :request
+      if v.is_a?(String)
+        next if v.empty?
+        # Some strings from GeoIP don't have the correct encoding...
+        result[k.to_s] = case v.encoding
+          # I have found strings coming from GeoIP that are ASCII-8BIT are actually
+          # ISO-8859-1...
+        when Encoding::ASCII_8BIT
+          v.force_encoding(Encoding::ISO_8859_1).encode(Encoding::UTF_8)
+        when Encoding::ISO_8859_1, Encoding::US_ASCII
+          v.encode(Encoding::UTF_8)
+        else
+          v
+        end
+      else
+        result[k.to_s] = v
+      end
+    end
+
+    lat, lng = result.values_at("latitude", "longitude")
+    if lat && lng
+      result["location"] = [ lng.to_f, lat.to_f ]
+    end
+
+    result
+  end
+
   def ensure_database!
     # Use thread-local access to GeoIP. The Ruby GeoIP module forces a mutex
     # around access to the database, which can be overcome with :pread.

data/logstash-filter-geoip.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-geoip'
-  s.version         = '2.0.3'
+  s.version         = '2.0.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "$summary"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/geoip_spec.rb

@@ -264,17 +264,16 @@ describe LogStash::Filters::GeoIP do
 
   describe "returned object identities" do
     let(:plugin) { LogStash::Filters::GeoIP.new("source" => "message") }
-    let(:geo_data) { plugin.get_geo_data_for_ip("8.8.8.8") }
+    let(:event) { LogStash::Event.new("message" => "8.8.8.8") }
+    let(:alt_event) { LogStash::Event.new("message" => "8.8.8.8") }
 
     before do
       plugin.register
     end
 
     it "should dup the objects" do
-      event = { "geoip" => {} }
-      alt_event = { "geoip" => {} }
-      plugin.apply_geodata(geo_data.to_hash, event)
-      plugin.apply_geodata(geo_data.to_hash, alt_event)
+      plugin.apply_geodata(plugin.get_geo_data(event), event)
+      plugin.apply_geodata(plugin.get_geo_data(alt_event), alt_event)
 
       event["geoip"].each do |k,v|
         alt_v = alt_event["geoip"][k]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-geoip
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.4
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-16 00:00:00.000000000 Z
+date: 2015-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -83,8 +83,8 @@ files:
 - vendor/GeoLiteCity-2013-01-18.dat
 - vendor/GeoIPASNum-2014-02-12.dat
 - logstash-filter-geoip.gemspec
-- CHANGELOG.md
 - README.md
+- CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE