logstash-filter-dns 2.0.2 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e19a68ccd4a4f4f02f625a725e18ad3c74f5eda3
-  data.tar.gz: 5cabe33a4eed9ee6dcf594ae75c8eb3831607386
+  metadata.gz: 4665d1daf1b87421818c9e6bd11f6efadb74349c
+  data.tar.gz: 4dcb49b8c0e902156fa2ebc3aadd9e97e3b7252f
 SHA512:
-  metadata.gz: 525124b0bd2e5cd58a0637534bf2d9902b1f0b6b7fbaa1d12acd434346d38fc0166748dcc87ec081a524ec2ec245c91077995c19ba5d424538b36dd3d97adec2
-  data.tar.gz: de90a47925204eb2cd4d4bb9e3084a2606dee71d690238b5044f79986359545979e60e19c0e3694698fb9ba8973ccd64f0cf25dacce51f5c06dfc9382a06266a
+  metadata.gz: 87b081e655bd9eb3e2a939cea5a1464328e7225ac655f30ae4fc4a8702b026715bf15576d642baeea467d9801272980f9555c669f5cfbd928e9a3b0abd1d3c13
+  data.tar.gz: 3d18bbc3ae5ae1661763334793b199fea6a07405e2b869ad19614906f1c06ca9382ae7014d0fbb74bbc47f8c380f955b52e6b08b52a5bc7490217e812a895b04

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.1.0
+ - Add caches for failed and successful lookups
+ - Lower default timeout value
+ - Retry a maximum of :max_retries instead of failing immediately
+
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-dns-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-dns-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/filters/dns.rb

@@ -1,6 +1,8 @@
 # encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
+require "lru_redux"
+
 
 # The DNS filter performs a lookup (either an A record/CNAME record lookup
 # or a reverse lookup at the PTR record) on records specified under the
@@ -18,7 +20,7 @@ require "logstash/namespace"
 #     }
 #
 # Caveats: Currently there is no way to specify a timeout in the DNS lookup.
-# 
+#
 # This filter, like all filters, only processes 1 event at a time, so the use
 # of this plugin can significantly slow down your pipeline's throughput if you
 # have a high latency network. By way of example, if each DNS lookup takes 2
@@ -46,7 +48,22 @@ class LogStash::Filters::DNS < LogStash::Filters::Base
   config :nameserver, :validate => :array
 
   # `resolv` calls will be wrapped in a timeout instance
-  config :timeout, :validate => :number, :default => 2
+  config :timeout, :validate => :number, :default => 0.5
+
+  # number of times to retry a failed resolve/reverse
+  config :max_retries, :validate => :number, :default => 2
+
+  # set the size of cache for successful requests
+  config :hit_cache_size, :validate => :number, :default => 0
+
+  # how long to cache successful requests (in seconds)
+  config :hit_cache_ttl, :validate => :number, :default => 60
+
+  # cache size for failed requests (Resolv::
+  config :failed_cache_size, :validate => :number, :default => 0
+
+  # how long to cache failed requests (in seconds)
+  config :failed_cache_ttl, :validate => :number, :default => 5
 
   public
   def register
@@ -58,42 +75,29 @@ class LogStash::Filters::DNS < LogStash::Filters::Base
       @resolv = Resolv.new(resolvers=[::Resolv::Hosts.new, ::Resolv::DNS.new(:nameserver => @nameserver, :search => [], :ndots => 1)])
     end
 
+    if @hit_cache_size > 0
+      @hit_cache = LruRedux::ThreadSafeCache.new(@hit_cache_size, @hit_cache_ttl)
+    end
+
+    if @failed_cache_size > 0
+      @failed_cache = LruRedux::ThreadSafeCache.new(@failed_cache_size, @failed_cache_ttl)
+    end
+
     @ip_validator = Resolv::AddressRegex
   end # def register
 
   public
   def filter(event)
-    
-
-    new_event = event.clone
 
     if @resolve
-      begin
-        status = Timeout::timeout(@timeout) {
-          resolve(new_event)
-        }
-        return if status.nil?
-      rescue Timeout::Error
-        @logger.debug("DNS: resolve action timed out")
-        return
-      end
+      return if resolve(event).nil?
     end
 
     if @reverse
-      begin
-        status = Timeout::timeout(@timeout) {
-          reverse(new_event)
-        }
-        return if status.nil?
-      rescue Timeout::Error
-        @logger.debug("DNS: reverse action timed out")
-        return
-      end
+      return if reverse(event).nil?
     end
 
-    filter_matched(new_event)
-    yield new_event
-    event.cancel
+    filter_matched(event)
   end
 
   private
@@ -111,25 +115,24 @@ class LogStash::Filters::DNS < LogStash::Filters::Base
       end
 
       begin
-        # in JRuby 1.7.11 outputs as US-ASCII
-        address = @resolv.getaddress(raw).force_encoding(Encoding::UTF_8)
+        return if @failed_cache && @failed_cache[raw] # recently failed resolv, skip
+        if @hit_cache
+          address = @hit_cache.getset(raw) { retriable_getaddress(raw) }
+        else
+          address = retriable_getaddress(raw)
+        end
       rescue Resolv::ResolvError
+        @failed_cache[raw] = true if @failed_cache
         @logger.debug("DNS: couldn't resolve the hostname.",
                       :field => field, :value => raw)
         return
-      rescue Resolv::ResolvTimeout
-        @logger.debug("DNS: timeout on resolving the hostname.",
+      rescue Resolv::ResolvTimeout, Timeout::Error
+        @logger.error("DNS: timeout on resolving the hostname.",
                       :field => field, :value => raw)
         return
       rescue SocketError => e
-        @logger.debug("DNS: Encountered SocketError.",
-                      :field => field, :value => raw)
-        return
-      rescue NoMethodError => e
-        # see JRUBY-5647
-        @logger.debug("DNS: couldn't resolve the hostname.",
-                      :field => field, :value => raw,
-                      :extra => "NameError instead of ResolvError")
+        @logger.error("DNS: Encountered SocketError.",
+                      :field => field, :value => raw, :message => e.message)
         return
       end
 
@@ -170,19 +173,24 @@ class LogStash::Filters::DNS < LogStash::Filters::Base
         return
       end
       begin
-        # in JRuby 1.7.11 outputs as US-ASCII
-        hostname = @resolv.getname(raw).force_encoding(Encoding::UTF_8)
+        return if @failed_cache && @failed_cache.key?(raw) # recently failed resolv, skip
+        if @hit_cache
+          hostname = @hit_cache.getset(raw) { retriable_getname(raw) }
+        else
+          hostname = retriable_getname(raw)
+        end
       rescue Resolv::ResolvError
+        @failed_cache[raw] = true if @failed_cache
         @logger.debug("DNS: couldn't resolve the address.",
                       :field => field, :value => raw)
         return
-      rescue Resolv::ResolvTimeout
-        @logger.debug("DNS: timeout on resolving address.",
+      rescue Resolv::ResolvTimeout, Timeout::Error
+        @logger.error("DNS: timeout on resolving address.",
                       :field => field, :value => raw)
         return
       rescue SocketError => e
-        @logger.debug("DNS: Encountered SocketError.",
-                      :field => field, :value => raw)
+        @logger.error("DNS: Encountered SocketError.",
+                      :field => field, :value => raw, :message => e.message)
         return
       end
 
@@ -201,4 +209,45 @@ class LogStash::Filters::DNS < LogStash::Filters::Base
       end
     end
   end
+
+  private
+  def retriable_request(&block)
+    tries = 0
+    begin
+      Timeout::timeout(@timeout) do
+        block.call
+      end
+    rescue Timeout::Error, SocketError
+      if tries < @max_retries
+        tries = tries + 1
+        retry
+      else
+        raise
+      end
+    end
+  end
+
+  private
+  def retriable_getname(address)
+    retriable_request do
+      getname(address)
+    end
+  end
+
+  private
+  def retriable_getaddress(name)
+    retriable_request do
+      getaddress(name)
+    end
+  end
+
+  private
+  def getname(address)
+    @resolv.getname(address).force_encoding(Encoding::UTF_8)
+  end
+
+  private
+  def getaddress(name)
+    @resolv.getaddress(name).force_encoding(Encoding::UTF_8)
+  end
 end # class LogStash::Filters::DNS

data/logstash-filter-dns.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-dns'
-  s.version         = '2.0.2'
+  s.version         = '2.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This filter will resolve any IP addresses from a field of your choosing."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency 'lru_redux', "~> 1.1.0"
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/filters/dns_spec.rb

@@ -89,7 +89,7 @@ describe LogStash::Filters::DNS do
 
     sample("host1" => "carrera.databits.net", "host2" => "nonexistanthostname###.net") do
       insist { subject["tags"] }.nil?
-      insist { subject["host1"] } == "carrera.databits.net"
+      insist { subject["host1"] } == "199.192.228.250"
       insist { subject["host2"] } == "nonexistanthostname###.net"
     end
   end
@@ -126,8 +126,8 @@ describe LogStash::Filters::DNS do
            "ip1" => "127.0.0.1",
            "ip2" => "128.0.0.1") do
       insist { subject["tags"] }.nil?
-      insist { subject["host1"] } == "carrera.databits.net"
-      insist { subject["ip1"] } == "127.0.0.1"
+      insist { subject["host1"] } == "199.192.228.250"
+      insist { subject["ip1"] } == "localhost"
       insist { subject["ip2"] } == "128.0.0.1"
     end
   end
@@ -238,4 +238,124 @@ describe LogStash::Filters::DNS do
       insist { subject["host"] } == "199.192.228.250"
     end
   end
+
+  describe "dns resolve lookup, multiple nameserver fallback" do
+    config <<-CONFIG
+      filter {
+        dns {
+          resolve => ["host"]
+          action => "replace"
+          nameserver => ["127.0.0.99", "8.8.8.8"]
+        }
+      }
+    CONFIG
+
+    sample("host" => "carrera.databits.net") do
+      insist { subject["host"] } == "199.192.228.250"
+    end
+  end
+
+  describe "failed cache" do
+
+    let(:subject) { LogStash::Filters::DNS.new(config) }
+    let(:event1) { LogStash::Event.new("message" => "unkownhost") }
+    let(:event2) { LogStash::Event.new("message" => "unkownhost") }
+
+    before(:each) do
+      allow(subject).to receive(:getaddress).and_raise Resolv::ResolvError
+      subject.register
+    end
+
+    context "when enabled" do
+      let(:config) { { "resolve" => ["message"], "failed_cache_size" => 3 } }
+
+      it "should cache a failed lookup" do
+        expect(subject).to receive(:getaddress).once
+        subject.filter(event1)
+        subject.filter(event2)
+      end
+    end
+
+    context "when disabled" do
+      let(:config) { { "resolve" => ["message"] } }
+
+      it "should not cache a failed lookup" do
+        expect(subject).to receive(:getaddress).twice
+        subject.filter(event1)
+        subject.filter(event2)
+      end
+    end
+  end
+
+  describe "hit cache" do
+
+    let(:subject) { LogStash::Filters::DNS.new(config) }
+    let(:event1) { LogStash::Event.new("message" => "unkownhost") }
+    let(:event2) { LogStash::Event.new("message" => "unkownhost") }
+
+    before(:each) do
+      allow(subject).to receive(:getaddress).and_return("127.0.0.1")
+      subject.register
+    end
+
+    context "when enabled" do
+      let(:config) { { "resolve" => ["message"], "hit_cache_size" => 3 } }
+
+      it "should cache a succesful lookup" do
+        expect(subject).to receive(:getaddress).once
+        subject.filter(event1)
+        subject.filter(event2)
+      end
+    end
+
+    context "when disabled" do
+      let(:config) { { "resolve" => ["message"] } }
+
+      it "should not cache a successful lookup" do
+        expect(subject).to receive(:getaddress).twice
+        subject.filter(event1)
+        subject.filter(event2)
+      end
+    end
+  end
+
+  describe "retries" do
+
+    let(:subject) { LogStash::Filters::DNS.new(config) }
+    let(:event) { LogStash::Event.new("message" => "unkownhost") }
+    let(:max_retries) { 3 }
+    let(:config) { { "resolve" => ["message"], "max_retries" => max_retries } }
+
+    before(:each) { subject.register }
+
+    context "when failing permanently" do
+      before(:each) do
+        allow(subject).to receive(:getaddress).and_raise(Timeout::Error)
+      end
+
+      it "should fail a resolve after max_retries" do
+        expect(subject).to receive(:getaddress).exactly(max_retries+1).times
+        subject.filter(event)
+      end
+    end
+
+    context "when failing temporarily" do
+      before(:each) do
+        allow(subject).to receive(:getaddress) do
+          @try ||= 0
+          if @try < 3
+            @try = @try + 1
+            raise Timeout::Error
+          else
+            return "127.0.0.1"
+          end
+        end
+      end
+
+      it "should resolve before max_retries" do
+        expect(subject).to receive(:getaddress).exactly(3).times
+        subject.filter(event)
+      end
+    end
+  end
 end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-dns
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2016-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,20 +28,36 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: lru_redux
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-  name: logstash-devutils
+        version: 1.1.0
   prerelease: false
-  type: :development
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []