logstash-input-lumberjack 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: da715b62794c6355a47c844a678f452afd568478
-  data.tar.gz: 866e0a9fcfcdecd61a018fdc33c991393aa65216
+  metadata.gz: a24179f3e3ca8a3092c1d692875ea2a88edfa557
+  data.tar.gz: 46e2bde92697647cb0b26e66484124e7e4ae6c11
 SHA512:
-  metadata.gz: c9a7534c7192428a7169eada97f06a6ec40f10bdee769764ab7d059af2eff2f8b12a68c1fe6da3d5148bda4b679715d13422573cf291c19f4b8907145d4289ea
-  data.tar.gz: c6357995a2a4bc630e49ecab1b9d23703264579915b7ffbdc70d9dc858f7e645c7cd0630a5a2e5a196e95f2faf3c1e2845748877fc7667081b5f239c8f739b8e
+  metadata.gz: c21484a7108a55aabe51d75d8877ddc749c1abfe30a033096ec662fa020f46335ffb8a66cba3de79a20b85d24f31f4b7dc51435544263d0f7bdd815ee96387c2
+  data.tar.gz: cc0df5cd9059650ebf13c4d18956ff2953535e8683ea2ecb167d2592a7bdd43eab06353ac4d77e03133ff3106b2ba2f5996f53d7825e0c78756c5c371e90d88f

data/CHANGELOG.md

@@ -0,0 +1,4 @@
+# 0.1.10
+  - Deprecating the `max_clients` option
+  - Use a circuit breaker to start refusing new connection when the queue is blocked for too long.
+  - Add an internal `SizeQueue` with a timeout to drop blocked connections. (https://github.com/logstash-plugins/logstash-input-lumberjack/pull/12)

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/lib/logstash/circuit_breaker.rb

@@ -0,0 +1,96 @@
+require "thread"
+require "cabin"
+
+module LogStash
+  # Largely inspired by Martin's fowler circuit breaker
+  class CircuitBreaker
+    class OpenBreaker < StandardError; end
+
+    # Error threshold before opening the breaker,
+    # if the breaker is open it wont execute the code.
+    DEFAULT_ERROR_THRESHOLD = 5
+
+    # Recover time after the breaker is open to start
+    # executing the method again.
+    DEFAULT_TIME_BEFORE_RETRY = 30
+
+    # Exceptions catched by the circuit breaker,
+    # too much errors and the breaker will trip.
+    DEFAULT_EXCEPTION_RESCUED = [StandardError]
+
+    def initialize(name, options = {}, &block)
+      @exceptions = Array(options.fetch(:exceptions, [StandardError]))
+      @error_threshold = options.fetch(:error_threshold, DEFAULT_ERROR_THRESHOLD)
+      @time_before_retry = options.fetch(:time_before_retry, DEFAULT_TIME_BEFORE_RETRY)
+      @block = block
+      @name = name
+      @mutex = Mutex.new
+      reset
+    end
+
+    def execute(args = nil)
+      case state
+      when :open
+        logger.warn("CircuitBreaker::Open", :name => @name)
+        raise OpenBreaker, "for #{@name}"
+      when :close, :half_open
+        if block_given?
+          yield args
+        else
+          @block.call(args)
+        end
+
+        if state == :half_open
+          logger.warn("CircuitBreaker::Close", :name => @name)
+          reset
+        end
+      end
+    rescue *@exceptions => e
+      logger.warn("CircuitBreaker::rescuing exceptions", :name => @name, :exception => e.class)
+      increment_errors(e)
+    end
+
+    def closed?
+      state == :close || state == :half_open
+    end
+
+    private
+    def logger
+      @logger ||= Cabin::Channel.get(LogStash)
+    end
+
+    def reset
+      @mutex.synchronize do
+        @errors_count = 0
+        @last_failure_time = nil
+      end
+    end
+
+    def increment_errors(exception)
+      @mutex.synchronize do
+        @errors_count += 1
+        @last_failure_time = Time.now
+
+        logger.debug("CircuitBreaker increment errors", 
+                     :errors_count => @errors_count, 
+                     :error_threshold => @error_threshold,
+                     :exception => exception.class,
+                     :message => exception.message) if logger.debug?
+      end
+    end
+
+    def state
+      @mutex.synchronize do
+        if @errors_count >= @error_threshold
+          if Time.now - @last_failure_time > @time_before_retry
+            :half_open
+          else
+            :open
+          end
+        else
+          :close
+        end
+      end
+    end
+  end
+end

data/lib/logstash/inputs/lumberjack.rb

@@ -29,18 +29,23 @@ class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
   # SSL key passphrase to use.
   config :ssl_key_passphrase, :validate => :password
 
-  # Number of maximum clients that the lumberjack input will accept, this allow you
-  # to control the back pressure up to the client and stop logstash to go OOM with 
-  # connection. This settings is a temporary solution and will be deprecated really soon.
-  config :max_clients, :validate => :number, :default => 1000, :deprecated => true
+  # The lumberjack input using a fixed thread pool to do the actual work and
+  # will accept a number of client in a queue, before starting to refuse new
+  # connection. This solve an issue when logstash-forwarder clients are 
+  # trying to connect to logstash which have a blocked pipeline and will 
+  # make logstash crash with an out of memory exception.
+  config :max_clients, :validate => :number, :default => 1000
 
   # TODO(sissel): Add CA to authenticate clients with.
 
-  public
+  BUFFERED_QUEUE_SIZE = 20
+  RECONNECT_BACKOFF_SLEEP = 0.5
+  
   def register
     require "lumberjack/server"
-    require "concurrent"
     require "concurrent/executors"
+    require "logstash/circuit_breaker"
+    require "logstash/sized_queue_timeout"
 
     @logger.info("Starting lumberjack input listener", :address => "#{@host}:#{@port}")
     @lumberjack = Lumberjack::Server.new(:address => @host, :port => @port,
@@ -48,6 +53,7 @@ class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
       :ssl_key_passphrase => @ssl_key_passphrase)
 
     # Limit the number of thread that can be created by the
+    # Limit the number of thread that can be created by the 
     # lumberjack output, if the queue is full the input will 
     # start rejecting new connection and raise an exception
     @threadpool = Concurrent::ThreadPoolExecutor.new(
@@ -56,37 +62,59 @@ class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
       :max_queue => 1, # in concurrent-ruby, bounded queue need to be at least 1.
       fallback_policy: :abort
     )
+    @threadpool = Concurrent::CachedThreadPool.new(:idletime => 15)
+
+    # in 1.5 the main SizeQueue doesnt have the concept of timeout
+    # We are using a small plugin buffer to move events to the internal queue
+    @buffered_queue = LogStash::SizedQueueTimeout.new(BUFFERED_QUEUE_SIZE)
+
+    @circuit_breaker = LogStash::CircuitBreaker.new("Lumberjack input",
+                            :exceptions => [LogStash::SizedQueueTimeout::TimeoutError])
+
   end # def register
 
   def run(output_queue)
+    start_buffer_broker(output_queue)
+
     while true do
-      accept do |connection, codec|
-        invoke(connection, codec) do |_codec, line, fields|
-          _codec.decode(line) do |event|
-            decorate(event)
-            fields.each { |k,v| event[k] = v; v.force_encoding(Encoding::UTF_8) }
-            output_queue << event
+      begin
+        # Wrapping the accept call into a CircuitBreaker
+        if @circuit_breaker.closed?
+          connection = @lumberjack.accept # Blocking call that creates a new connection
+
+          invoke(connection, codec.clone) do |_codec, line, fields|
+            _codec.decode(line) do |event|
+              decorate(event)
+              fields.each { |k,v| event[k] = v; v.force_encoding(Encoding::UTF_8) }
+
+              @circuit_breaker.execute { @buffered_queue << event }
+            end
           end
+        else
+          @logger.warn("Lumberjack input: the pipeline is blocked, temporary refusing new connection.")
+          sleep(RECONNECT_BACKOFF_SLEEP)
         end
+        # When too many errors happen inside the circuit breaker it will throw 
+        # this exception and start refusing connection, we need to catch it but 
+        # it's safe to ignore.
+      rescue LogStash::CircuitBreaker::OpenBreaker => e
       end
     end
+  rescue LogStash::ShutdownSignal
+    @logger.info("Lumberjack input: received ShutdownSignal")
   rescue => e
-    @logger.error("Exception in lumberjack input", :exception => e)
+    @logger.error("Lumberjack input: unhandled exception", :exception => e, :backtrace => e.backtrace)
+  ensure
     shutdown(output_queue)
   end # def run
 
   private
   def accept(&block)
     connection = @lumberjack.accept # Blocking call that creates a new connection
-    
-    if @threadpool.length < @threadpool.max_length
-      block.call(connection, @codec.clone)
-    else
-      @logger.warn("Lumberjack input, maximum connection exceeded, new connection are rejected.", :max_clients => @max_clients)
-      connection.close
-    end
+    block.call(connection, @codec.clone)
   end
 
+  private
   def invoke(connection, codec, &block)
     @threadpool.post do
       begin
@@ -98,4 +126,12 @@ class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
       end
     end
   end
+
+  def start_buffer_broker(output_queue)
+    @threadpool.post do
+      while true
+        output_queue << @buffered_queue.pop_no_timeout
+      end
+    end
+  end
 end # class LogStash::Inputs::Lumberjack

data/lib/logstash/sized_queue_timeout.rb

@@ -0,0 +1,58 @@
+require "concurrent/atomic/condition"
+require "thread"
+
+module LogStash
+  # Minimal subset implement of a SizedQueue supporting
+  # a timeout option on the lock.
+  #
+  # This will be part of the main Logstash's sized queue
+  class SizedQueueTimeout
+    class TimeoutError < StandardError; end
+
+    DEFAULT_TIMEOUT = 2 # in seconds
+
+    def initialize(max_size, options = {})
+      @condition_in = Concurrent::Condition.new
+      @condition_out = Concurrent::Condition.new
+
+      @max_size = max_size
+      @queue = []
+      @mutex = Mutex.new
+    end
+
+    def push(obj, timeout = DEFAULT_TIMEOUT)
+      @mutex.synchronize do
+        while full? # wake up check
+          result = @condition_out.wait(@mutex, timeout) 
+          raise TimeoutError if result.timed_out?
+        end
+
+        @queue << obj
+        @condition_in.signal
+
+        return obj
+      end
+    end
+    alias_method :<<, :push
+
+    def size
+      @mutex.synchronize { @queue.size }
+    end
+
+    def pop_no_timeout
+      @mutex.synchronize do
+        @condition_in.wait(@mutex) while @queue.empty? # Wake up check
+
+        obj = @queue.shift
+        @condition_out.signal
+
+        return obj
+      end
+    end
+
+    private
+    def full?
+      @queue.size == @max_size
+    end
+  end
+end

data/logstash-input-lumberjack.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-lumberjack'
-  s.version         = '0.1.9'
+  s.version         = '0.1.10'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Receive events using the lumberjack protocol."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -29,5 +29,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'stud'
   s.add_development_dependency 'logstash-codec-multiline'
+  s.add_development_dependency "flores"
+  s.add_development_dependency "stud"
 end
 

data/spec/inputs/lumberjack_spec.rb

@@ -6,7 +6,6 @@ require "logstash/codecs/plain"
 require "logstash/codecs/multiline"
 require "logstash/event"
 require "lumberjack/client"
-require_relative "../support/logstash_test"
 
 describe LogStash::Inputs::Lumberjack do
   let(:connection) { double("connection") }
@@ -47,58 +46,4 @@ describe LogStash::Inputs::Lumberjack do
       end
     end
   end
-
-  context "when we have the maximum clients connected" do
-    let(:max_clients) { 1 }
-    let(:window_size) { 1 }
-    let(:config) do 
-      {
-        "port" => port,
-        "ssl_certificate" => certificate.ssl_cert,
-        "ssl_key" => certificate.ssl_key,
-        "type" => "testing",
-        "max_clients" => max_clients
-      }
-    end
-
-    let(:client_options) do
-      {
-        :port => port,
-        :address => "127.0.0.1",
-        :ssl_certificate => certificate.ssl_cert,
-        :window_size => window_size
-      }
-    end
-
-    before do
-      lumberjack.register
-
-      @server = Thread.new do
-        lumberjack.run(queue)
-      end
-
-      sleep(0.1) # wait for the server to correctly accept messages
-    end
-
-    after do
-      @server.raise(LogStash::ShutdownSignal)
-      @server.join
-    end
-
-    it "stops accepting new connection" do
-      client1 = Lumberjack::Socket.new(client_options)
-      
-      # Since the connection is stopped on the other side and OS X and 
-      # linux doesn't behave the same. The client could raise a IOError
-      # or an SSLError. On OSX I had to try to send some data to trip
-      # the error.
-      expect { 
-        client2 = Lumberjack::Socket.new(client_options)
-
-        (window_size + 1).times do
-          client2.write_hash({"line" => "message"}) 
-        end
-      }.to raise_error
-    end
- end
 end

data/spec/logstash/circuit_breaker_spec.rb

@@ -0,0 +1,60 @@
+require "spec_helper"
+require "logstash/circuit_breaker"
+
+class DummyErrorTest < StandardError; end
+
+describe LogStash::CircuitBreaker do
+  let(:error_threshold) { 1 }
+  let(:options) do 
+    {
+      :exceptions => [DummyErrorTest],
+      :error_threshold => error_threshold 
+    }
+  end
+
+  subject { LogStash::CircuitBreaker.new("testing", options) }
+
+
+  it "closed by default" do
+    expect(subject.closed?).to eq(true)
+  end
+
+  context "when having too many errors" do
+    let(:future_time) { Time.now + 3600 }
+    before do
+      subject.execute do
+        raise DummyErrorTest
+      end
+    end
+
+    it "raised an exception if we have too many errors" do
+      expect {
+        subject.execute do
+          raise DummyErrorTest
+        end
+      }.to raise_error(LogStash::CircuitBreaker::OpenBreaker)
+    end
+
+    it "sets the breaker to open" do
+      expect(subject.closed?).to eq(false)
+    end
+
+    it "resets the breaker after the time before retry" do
+      expect(Time).to receive(:now).at_least(1).and_return(future_time)
+      expect(subject.closed?).to eq(true)
+    end
+
+    it "doesnt run the command" do
+      runned = false
+
+      begin
+        subject.execute do
+          runned = true
+        end
+      rescue LogStash::CircuitBreaker::OpenBreaker
+      end
+
+      expect(runned).to eq(false)
+    end
+  end
+end

data/spec/logstash/size_queue_timeout_spec.rb

@@ -0,0 +1,100 @@
+require "spec_helper"
+require "logstash/sized_queue_timeout"
+require "flores/random"
+require "stud/try"
+
+describe "LogStash::SizedQueueTimeout" do
+  let(:max_size) { Flores::Random.integer(2..100) }
+  let(:element) { Flores::Random.text(0..100) }
+
+  subject { LogStash::SizedQueueTimeout.new(max_size) }
+  
+  it "adds element to the queue" do
+    subject << element
+    expect(subject.size).to eq(1)
+  end
+
+  it "allow to pop element from the queue" do
+    subject << element
+    subject << "awesome"
+
+    expect(subject.pop_no_timeout).to eq(element)
+  end
+  
+  context "when the queue is full" do
+    before do
+      max_size.times { subject << element }
+    end
+
+    it "block with a timeout" do
+      expect {
+        subject << element
+      }.to raise_error(LogStash::SizedQueueTimeout::TimeoutError)
+    end
+
+    it "unblock when we pop" do
+      blocked = Thread.new do
+        subject << element
+      end
+      sleep(0.1) until blocked.stop?
+
+      expect(blocked.status).to eq("sleep")
+
+      th = Thread.new do
+        subject.pop_no_timeout
+      end
+      sleep(0.1) until th.stop?
+
+      expect(blocked.status).to eq(false)
+
+      blocked.join
+      th.join
+    end
+  end
+
+  context "when the queue is empty" do
+    it "block on pop" do
+      blocked = Thread.new do
+        subject.pop_no_timeout
+      end
+      sleep(0.1) until blocked.stop?
+
+      expect(blocked.status).to eq("sleep")
+
+      th = Thread.new do
+        subject << element
+      end
+      sleep(0.1) until th.stop?
+
+      expect(blocked.status).to eq(false)
+      th.join
+      blocked.join
+    end
+  end
+
+  context "when the queue is occupied but not full" do
+    before :each do
+      Flores::Random.iterations(0..max_size) { subject << "hurray" } 
+    end
+
+    it "doesnt block on pop" do
+      th = Thread.new do
+        subject.pop_no_timeout
+      end
+      sleep(0.1) until th.stop?
+
+      expect(th.status).to eq(false)
+      th.join
+    end
+
+    it "doesnt block on push" do
+      th = Thread.new do
+        subject << element
+      end
+      sleep(0.1) until th.stop?
+
+      expect(th.status).to eq(false)
+      th.join
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,2 +1,3 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/codecs/plain"
+require_relative "support/logstash_test"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-lumberjack
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-12 00:00:00.000000000 Z
+date: 2015-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -114,6 +114,34 @@ dependencies:
         version: '0'
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: flores
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: stud
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -125,11 +153,16 @@ files:
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
+- NOTICE.TXT
 - README.md
 - Rakefile
+- lib/logstash/circuit_breaker.rb
 - lib/logstash/inputs/lumberjack.rb
+- lib/logstash/sized_queue_timeout.rb
 - logstash-input-lumberjack.gemspec
 - spec/inputs/lumberjack_spec.rb
+- spec/logstash/circuit_breaker_spec.rb
+- spec/logstash/size_queue_timeout_spec.rb
 - spec/spec_helper.rb
 - spec/support/logstash_test.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
@@ -160,5 +193,7 @@ specification_version: 4
 summary: Receive events using the lumberjack protocol.
 test_files:
 - spec/inputs/lumberjack_spec.rb
+- spec/logstash/circuit_breaker_spec.rb
+- spec/logstash/size_queue_timeout_spec.rb
 - spec/spec_helper.rb
 - spec/support/logstash_test.rb