logstash-filter-geoip 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4cb9d672a670ac659f61fa7604bc57274eabd759
-  data.tar.gz: d697817443b57ce1ef55e4dbdd19caf657e90cc6
+  metadata.gz: 9d5c90fa4938034e2c5ed20e1351df96ccf0a943
+  data.tar.gz: e69c420b023bc2f13db5d7b5844e6abe65325900
 SHA512:
-  metadata.gz: 95c606eb5c2df267eb7476b91247bf14e9c4758ea1c27b720342fc6d624091473b754c11cf1b42ed13204e321910e288f17bc95e90e61fe4abd3a2e73083c213
-  data.tar.gz: 62ba7bd2e0a890969d6e2ca3c1a0575ab1ccdd8e84232d468bee03bcab0294e4d9850d889160ec5973093313cd6263ab4554897d47fe8c15400b2208bfdc886b
+  metadata.gz: d8a75e6eb497f9d400ba1007c0dd370e35ea3447a7f523cd8719076a0f8b68edb854e58e7bc6716be4974bf10bffb3bf76f2dadd6918e8b28c3e5a18f8bd4403
+  data.tar.gz: a8655cbb27a53d6f4cfe48c0442475a82b7bdbf04ef8af099fc2155f581f534b15fdbf9085ee3c9f6a7b30c8a0ae622e7a65a807a06b4be6094755c67630a445

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+* 1.1.0
+  - Add LRU cache

data/README.md

@@ -1,15 +1,15 @@
 # Logstash Plugin
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
 
 ## Need Help?
 
@@ -89,4 +89,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/filters/geoip.rb

@@ -2,6 +2,8 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 require "tempfile"
+require "lru_redux"
+require "geoip"
 
 # The GeoIP filter adds information about the geographical location of IP addresses,
 # based on data from the Maxmind database.
@@ -11,7 +13,7 @@ require "tempfile"
 # http://geojson.org/geojson-spec.html[GeoJSON] format. Additionally,
 # the default Elasticsearch template provided with the
 # <<plugins-outputs-elasticsearch,`elasticsearch` output>> maps
-# the `[geoip][location]` field to an http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-geo-point-type.html#_mapping_options[Elasticsearch geo_point].
+# the `[geoip][location]` field to an https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-geo-point-type.html#_mapping_options[Elasticsearch geo_point].
 #
 # As this field is a `geo_point` _and_ it is still valid GeoJSON, you get
 # the awesomeness of Elasticsearch's geospatial query, facet and filter functions
@@ -22,6 +24,12 @@ require "tempfile"
 # Maxmind with a CCA-ShareAlike 3.0 license. For more details on GeoLite, see
 # <http://www.maxmind.com/en/geolite>.
 class LogStash::Filters::GeoIP < LogStash::Filters::Base
+  LOOKUP_CACHE_INIT_MUTEX = Mutex.new
+  # Map of lookup caches, keyed by geoip_type
+  LOOKUP_CACHES = {}
+
+  attr_accessor :lookup_cache
+
   config_name "geoip"
 
   # The path to the GeoIP database file which Logstash should use. Country, City, ASN, ISP
@@ -58,9 +66,24 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
   # is still valid GeoJSON.
   config :target, :validate => :string, :default => 'geoip'
 
+  # GeoIP lookup is surprisingly expensive. This filter uses an LRU cache to take advantage of the fact that
+  # IPs agents are often found adjacent to one another in log files and rarely have a random distribution.
+  # The higher you set this the more likely an item is to be in the cache and the faster this filter will run.
+  # However, if you set this too high you can use more memory than desired.
+  #
+  # Experiment with different values for this option to find the best performance for your dataset.
+  #
+  # This MUST be set to a value > 0. There is really no reason to not want this behavior, the overhead is minimal
+  # and the speed gains are large.
+  #
+  # It is important to note that this config value is global to the geoip_type. That is to say all instances of the geoip filter
+  # of the same geoip_type share the same cache. The last declared cache size will 'win'. The reason for this is that there would be no benefit
+  # to having multiple caches for different instances at different points in the pipeline, that would just increase the
+  # number of cache misses and waste memory.
+  config :lru_cache_size, :validate => :number, :default => 1000
+
   public
   def register
-    require "geoip"
     if @database.nil?
       @database = ::Dir.glob(::File.join(::File.expand_path("../../../vendor/", ::File.dirname(__FILE__)),"GeoLiteCity*.dat")).first
       if !File.exists?(@database)
@@ -88,6 +111,11 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
     end
 
     @threadkey = "geoip-#{self.object_id}"
+
+    # This is wrapped in a mutex to make sure the initialization behavior of LOOKUP_CACHES (see def above) doesn't create a dupe
+    LOOKUP_CACHE_INIT_MUTEX.synchronize do
+      self.lookup_cache = LOOKUP_CACHES[@geoip_type] ||= LruRedux::ThreadSafeCache.new(1000)
+    end
   end # def register
 
   public
@@ -104,18 +132,16 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
       Thread.current[@threadkey] = ::GeoIP.new(@database)
     end
 
-    begin
-      ip = event[@source]
-      ip = ip.first if ip.is_a? Array
-      geo_data = Thread.current[@threadkey].send(@geoip_type, ip)
-    rescue SocketError => e
-      @logger.error("IP Field contained invalid IP address or hostname", :field => @source, :event => event)
-    rescue Exception => e
-      @logger.error("Unknown error while looking up GeoIP data", :exception => e, :field => @source, :event => event)
-    end
+    geo_data = get_geo_data(event)
 
     return if geo_data.nil? || !geo_data.respond_to?(:to_hash)
 
+    apply_geodata(geo_data, event)
+
+    filter_matched(event)
+  end # def filter
+
+  def apply_geodata(geo_data,event)
     geo_data_hash = geo_data.to_hash
     geo_data_hash.delete(:request)
     event[@target] = {} if event[@target].nil?
@@ -130,16 +156,36 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
         if value.is_a?(String)
           # Some strings from GeoIP don't have the correct encoding...
           value = case value.encoding
-            # I have found strings coming from GeoIP that are ASCII-8BIT are actually
-            # ISO-8859-1...
-            when Encoding::ASCII_8BIT; value.force_encoding(Encoding::ISO_8859_1).encode(Encoding::UTF_8)
-            when Encoding::ISO_8859_1, Encoding::US_ASCII;  value.encode(Encoding::UTF_8)
-            else; value
-          end
+                    # I have found strings coming from GeoIP that are ASCII-8BIT are actually
+                    # ISO-8859-1...
+                    when Encoding::ASCII_8BIT; value.force_encoding(Encoding::ISO_8859_1).encode(Encoding::UTF_8)
+                    when Encoding::ISO_8859_1, Encoding::US_ASCII;  value.encode(Encoding::UTF_8)
+                    else; value.dup
+                  end
         end
         event[@target][key.to_s] = value
       end
     end # geo_data_hash.each
-    filter_matched(event)
-  end # def filter
+  end
+
+  def get_geo_data(event)
+    ip = event[@source]
+    ip = ip.first if ip.is_a? Array
+
+    get_geo_data_for_ip(ip)
+  rescue SocketError => e
+    @logger.error("IP Field contained invalid IP address or hostname", :field => @source, :event => event)
+  rescue StandardError => e
+    @logger.error("Unknown error while looking up GeoIP data", :exception => e, :field => @source, :event => event)
+  end
+
+  def get_geo_data_for_ip(ip)
+    if (cached = lookup_cache[ip])
+      cached
+    else
+      geo_data = Thread.current[@threadkey].send(@geoip_type, ip)
+      lookup_cache[ip] = geo_data
+      geo_data
+    end
+  end
 end # class LogStash::Filters::GeoIP

data/logstash-filter-geoip.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-geoip'
-  s.version         = '1.0.2'
+  s.version         = '1.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "$summary"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
 
   s.add_runtime_dependency 'geoip', ['>= 1.3.2']
+  s.add_runtime_dependency 'lru_redux', "~> 1.1.0"
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/filters/geoip_spec.rb

@@ -221,6 +221,29 @@ describe LogStash::Filters::GeoIP do
         subject
       end
     end
+  end
+
+  describe "returned object identities" do
+    let(:plugin) { LogStash::Filters::GeoIP.new("source" => "message") }
+    let(:geo_data) { plugin.get_geo_data_for_ip("8.8.8.8") }
+
+    before do
+      plugin.register
+    end
 
+    it "should dup the objects" do
+      event = {}
+      alt_event = {}
+      plugin.apply_geodata(geo_data, event)
+      plugin.apply_geodata(geo_data, alt_event)
+
+      event["geoip"].each do |k,v|
+        alt_v = alt_event["geoip"][k]
+        expect(v).to eql(alt_v)
+        unless v.is_a?(Numeric) # Numeric values can't be mutated, so this isn't an issue, its really for strings
+          expect(v.object_id).not_to eql(alt_v.object_id), "Object Ids for key #{k} and v #{v}"
+        end
+      end
+    end
   end
 end

metadata

@@ -1,83 +1,94 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-geoip
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-30 00:00:00.000000000 Z
+date: 2015-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.4.0
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 2.0.0
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.4.0
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 2.0.0
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.2
   name: geoip
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.2
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.2
+        version: 1.1.0
+  name: lru_redux
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
 - CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
 - README.md
-- Rakefile
 - lib/logstash/filters/geoip.rb
 - logstash-filter-geoip.gemspec
 - spec/filters/geoip_spec.rb
-- vendor.json
-- vendor/GeoLiteCity-2013-01-18.dat
 - vendor/GeoIPASNum-2014-02-12.dat
+- vendor/GeoLiteCity-2013-01-18.dat
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -90,19 +101,20 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
-summary: $summary
+summary: "$summary"
 test_files:
 - spec/filters/geoip_spec.rb
+has_rdoc:

data/.gitignore

@@ -1,4 +0,0 @@
-*.gem
-Gemfile.lock
-.bundle
-vendor

data/Rakefile

@@ -1,10 +0,0 @@
-require 'json'
-
-BASE_PATH = File.expand_path(File.dirname(__FILE__))
-@files = JSON.parse(File.read(File.join(BASE_PATH, 'vendor.json')))
-
-task :default do
-  system("rake -T")
-end
-
-require "logstash/devutils/rake"

data/vendor.json

@@ -1,10 +0,0 @@
-[
-    {
-        "url": "http://logstash.objects.dreamhost.com/maxmind/GeoLiteCity-2013-01-18.dat.gz",
-        "sha1": "15aab9a90ff90c4784b2c48331014d242b86bf82"
-    },
-    {
-        "url": "http://logstash.objects.dreamhost.com/maxmind/GeoIPASNum-2014-02-12.dat.gz",
-        "sha1": "6f33ca0b31e5f233e36d1f66fbeae36909b58f91"
-    }
-]