logstash-input-lumberjack 0.1.5 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1e2b2ab7e69541032b8744d37a122375be970525
-  data.tar.gz: 8ff378328f8cd060d1b2b6982c08b449f645721f
+  metadata.gz: 040459673c2faeacdaf421d43d5eae5fa69960d0
+  data.tar.gz: 23b1f847f4bd65ec54890c9869ec9c8c751119d4
 SHA512:
-  metadata.gz: f577d515bdabd959b38f4b527a5432c88a28a6db8bee4a8c14f4402a787c015f8a5940f1c56798f6409a2a93529a1788b5f16d4a439475330df3262ccb59e68f
-  data.tar.gz: 2da25f78ff525a1d1ab76058719ed1404bde0224f84b9607a0d2eabde7acb2214d5781f28da0dc64b647381a37917336619b8ad26f393583872d2e474e3cbb2e
+  metadata.gz: 3971428610a78bf39614ff760100272a8c820814eaff288d773e748f8401213722e73b2cbd32d8a09cdad50c0f2b967af281a673d89188dce2334e19c6873daa
+  data.tar.gz: a79c6bda1d57f5a7aff678bcd63ba67049910706728b8e695b32d4b96e8dc3adc4d4a37bb93eeabeb3309881a5824ab49b2cad67ab35ba7fb64caf0283c06bd1

data/lib/logstash/inputs/lumberjack.rb

@@ -29,19 +29,35 @@ class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
   # SSL key passphrase to use.
   config :ssl_key_passphrase, :validate => :password
 
+  # Number of maximum clients that the lumberjack input will accept, this allow you
+  # to control the back pressure up to the client and stop logstash to go OOM with 
+  # connection. This settings is a temporary solution and will be deprecated really soon.
+  config :max_clients, :validate => :number, :default => 1000, :deprecated => true
+
   # TODO(sissel): Add CA to authenticate clients with.
 
   public
   def register
     require "lumberjack/server"
+    require "concurrent"
+    require "concurrent/executors"
 
     @logger.info("Starting lumberjack input listener", :address => "#{@host}:#{@port}")
     @lumberjack = Lumberjack::Server.new(:address => @host, :port => @port,
       :ssl_certificate => @ssl_certificate, :ssl_key => @ssl_key,
       :ssl_key_passphrase => @ssl_key_passphrase)
+
+    # Limit the number of thread that can be created by the
+    # lumberjack output, if the queue is full the input will 
+    # start rejecting new connection and raise an exception
+    @threadpool = Concurrent::ThreadPoolExecutor.new(
+      :min_threads => 1,
+      :max_threads => @max_clients,
+      :max_queue => 1, # in concurrent-ruby, bounded queue need to be at least 1.
+      fallback_policy: :abort
+    )
   end # def register
 
-  public
   def run(output_queue)
     while true do
       accept do |connection, codec|
@@ -62,16 +78,20 @@ class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
   private
   def accept(&block)
     connection = @lumberjack.accept # Blocking call that creates a new connection
-    block.call(connection, @codec.clone)
+    
+    if @threadpool.length < @threadpool.max_length
+      block.call(connection, @codec.clone)
+    else
+      @logger.warn("Lumberjack input, maximum connection exceeded, new connection are rejected.", :max_clients => @max_clients_queue)
+      connection.close
+    end
   end
 
-  private
   def invoke(connection, codec, &block)
-    Thread.new(connection, codec) do |_connection, _codec|
-      _connection.run do |fields|
-        block.call(_codec, fields.delete("line"), fields)
+    @threadpool.post do
+      connection.run do |fields|
+        block.call(codec, fields.delete("line"), fields)
       end
     end
   end
-
 end # class LogStash::Inputs::Lumberjack

data/logstash-input-lumberjack.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-lumberjack'
-  s.version         = '0.1.5'
+  s.version         = '0.1.7'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Receive events using the lumberjack protocol."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -24,10 +24,10 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'jls-lumberjack', ['>=0.0.20']
+  s.add_runtime_dependency "concurrent-ruby"
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'stud'
   s.add_development_dependency 'logstash-codec-multiline'
-  
 end
 

data/spec/inputs/lumberjack_spec.rb

@@ -4,54 +4,101 @@ require "stud/temporary"
 require 'logstash/inputs/lumberjack'
 require "logstash/codecs/plain"
 require "logstash/codecs/multiline"
+require "logstash/event"
+require "lumberjack/client"
+require_relative "../support/logstash_test"
 
 describe LogStash::Inputs::Lumberjack do
-
-  let(:ssl_cert) { Stud::Temporary.pathname("ssl_certificate") }
-  let(:ssl_key)  { Stud::Temporary.pathname("ssl_key") }
-
+  let(:connection) { double("connection") }
+  let(:certificate) { LogStashTest.certificate }
+  let(:port) { LogStashTest.random_port }
   let(:queue)  { Queue.new }
-  let(:config)   { { "port" => 0, "ssl_certificate" => ssl_cert, "ssl_key" => ssl_key, "type" => "example" } }
+  let(:config)   { { "port" => 0, "ssl_certificate" => certificate.ssl_cert, "ssl_key" => certificate.ssl_key, "type" => "example", "tags" => "lumberjack" } }
 
-
-  before do
-    system("openssl req -x509  -batch -nodes -newkey rsa:2048 -keyout #{ssl_key} -out #{ssl_cert} -subj /CN=localhost > /dev/null 2>&1")
-  end
+  subject(:lumberjack) { LogStash::Inputs::Lumberjack.new(config) }
 
   context "#register" do
-
     it "raise no exception" do
       plugin = LogStash::Inputs::Lumberjack.new(config)
-      expect { plugin.register}.not_to raise_error
+      expect { plugin.register }.not_to raise_error
     end
   end
 
   describe "#processing of events" do
+    let(:lines) { {"line" => "one\ntwo\n  two.2\nthree\n", "tags" => ["syslog"]} }
 
-    context "#codecs" do
+    before do
+      allow(connection).to receive(:run).and_yield(lines)
+      lumberjack.register
+      expect_any_instance_of(Lumberjack::Server).to receive(:accept).and_return(connection)
+    end
 
+    context "#codecs" do
       let(:config) do
-        { "port" => 6969, "ssl_certificate" => ssl_cert, "ssl_key" => ssl_key,
+        { "port" => port, "ssl_certificate" => certificate.ssl_cert, "ssl_key" => certificate.ssl_key,
           "type" => "example", "codec" => codec }
       end
 
       let(:codec) { LogStash::Codecs::Multiline.new("pattern" => '\n', "what" => "previous") }
-      subject!(:lumberjack) { LogStash::Inputs::Lumberjack.new(config) }
-      let(:connection) { double("connection") }
-      let(:lines) { {"line" => "one\ntwo\n  two.2\nthree\n"} }
-
-      before(:each) do
-        allow(connection).to receive(:run).and_yield(lines)
-        lumberjack.register
-      end
-
       it "clone the codec per connection" do
-        expect_any_instance_of(Lumberjack::Server).to receive(:accept).and_return(connection)
         expect(lumberjack.codec).to receive(:clone).once
         expect(lumberjack).to receive(:invoke).and_throw(:msg)
         lumberjack.run(queue)
       end
-
     end
   end
+
+  context "when we have the maximum clients connected" do
+    let(:max_clients) { 1 }
+    let(:window_size) { 1 }
+    let(:config) do 
+      {
+        "port" => port,
+        "ssl_certificate" => certificate.ssl_cert,
+        "ssl_key" => certificate.ssl_key,
+        "type" => "testing",
+        "max_clients" => max_clients
+      }
+    end
+
+    let(:client_options) do
+      {
+        :port => port,
+        :address => "127.0.0.1",
+        :ssl_certificate => certificate.ssl_cert,
+        :window_size => window_size
+      }
+    end
+
+    before do
+      lumberjack.register
+
+      @server = Thread.new do
+        lumberjack.run(queue)
+      end
+
+      sleep(0.1) # wait for the server to correctly accept messages
+    end
+
+    after do
+      @server.raise(LogStash::ShutdownSignal)
+      @server.join
+    end
+
+    it "stops accepting new connection" do
+      client1 = Lumberjack::Socket.new(client_options)
+      
+      # Since the connection is stopped on the other side and OS X and 
+      # linux doesn't behave the same. The client could raise a IOError
+      # or an SSLError. On OSX I had to try to send some data to trip
+      # the error.
+      expect { 
+        client2 = Lumberjack::Socket.new(client_options)
+
+        (window_size + 1).times do
+          client2.write_hash({"line" => "message"}) 
+        end
+      }.to raise_error
+    end
+ end
 end

data/spec/support/logstash_test.rb

@@ -0,0 +1,23 @@
+require "stud/temporary"
+module LogStashTest
+  class Certicate
+    attr_reader :ssl_key, :ssl_cert
+
+    def initialize
+      @ssl_cert = Stud::Temporary.pathname("ssl_certificate")
+      @ssl_key = Stud::Temporary.pathname("ssl_key")
+
+      system("openssl req -x509  -batch -nodes -newkey rsa:2048 -keyout #{ssl_key} -out #{ssl_cert} -subj /CN=localhost > /dev/null 2>&1")
+    end
+  end
+
+  class << self
+    def certificate
+      Certicate.new
+    end
+
+    def random_port
+      rand(2000..10000)
+    end
+  end
+end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-lumberjack
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.7
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-20 00:00:00.000000000 Z
+date: 2015-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,76 +28,92 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jls-lumberjack
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 0.0.20
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: 0.0.20
-  name: jls-lumberjack
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.20
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
-  type: :development
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: stud
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: stud
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-multiline
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-multiline
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -116,6 +130,7 @@ files:
 - logstash-input-lumberjack.gemspec
 - spec/inputs/lumberjack_spec.rb
 - spec/spec_helper.rb
+- spec/support/logstash_test.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -145,3 +160,4 @@ summary: Receive events using the lumberjack protocol.
 test_files:
 - spec/inputs/lumberjack_spec.rb
 - spec/spec_helper.rb
+- spec/support/logstash_test.rb