logstash-input-relp 2.0.1 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bf41bf3e9f571773cd7317ccaa639812b8af4b75
-  data.tar.gz: 5b7b8ade54236ee71f4dd1540a81a831b245c02b
+  metadata.gz: 38af08ca585650125a700ef56813a35c18c2f947
+  data.tar.gz: ee7e0b9bbc59fa5d1cde1d1aee4fa9240b6f4410
 SHA512:
-  metadata.gz: c6724f73a866b45ec55f446c732b1fae0c027fda14bb5cf7516be7fd669a078c82d227db9442fd90467b5ad78d3552873df1ba823573facd37f0807ca17ee548
-  data.tar.gz: e616c70e39a338199012442b962b2b3766417b028d5b3ab10f0d0aefcd9bdf26e3e970714eed99973865ac286ec02b78504b93b6cbdbf620e1ecc5aba662f27f
+  metadata.gz: 86c0e5ea580ebd2dbcaa8a37a76809d5540a3e23cde0a673c2d3936fd348368aaac0d956e0f31a0923f2096fe82940d0b645ce8dc3116f08de10f827a98eb91a
+  data.tar.gz: ac4ac922af4bb5de2b03c2b9245c15f9f2f025d5a7146120bb04965b11734cc0d83014105be1c9345add663544ecff33c421dc9830f6bdde861c4f1ae9e86908

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.0.2
+ - Update the test to play nice within the context of the default
+   plugins LS core integration test.
+
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895

data/lib/logstash/inputs/relp.rb

@@ -3,6 +3,7 @@ require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/util/relp"
 require "logstash/util/socket_peer"
+require "openssl"
 
 # Read RELP events over a TCP socket.
 #
@@ -47,11 +48,16 @@ class LogStash::Inputs::Relp < LogStash::Inputs::Base
   def initialize(*args)
     super(*args)
     @relp_server = nil
+
+    # monkey patch TCPSocket and SSLSocket to include socket peer
+    TCPSocket.module_eval{include ::LogStash::Util::SocketPeer}
+    OpenSSL::SSL::SSLSocket.module_eval{include ::LogStash::Util::SocketPeer}
   end # def initialize
 
   public
   def register
     @logger.info("Starting relp input listener", :address => "#{@host}:#{@port}")
+
     if @ssl_enable
       initialize_ssl_context
       if @ssl_verify == false
@@ -62,13 +68,17 @@ class LogStash::Inputs::Relp < LogStash::Inputs::Base
         ].join("\n")
       end
     end
+
+    # note that since we are opening a socket (through @relp_server) in register,
+    # we must also make sure we close it in the close method even if we also close
+    # it in the stop method since we could have a situation where register is called
+    # but not run & stop.
+
     @relp_server = RelpServer.new(@host, @port,['syslog'], @ssl_context)
   end # def register
 
   private
   def initialize_ssl_context
-    require "openssl"
-
     @ssl_context = OpenSSL::SSL::SSLContext.new
     @ssl_context.cert = OpenSSL::X509::Certificate.new(File.read(@ssl_cert))
     @ssl_context.key = OpenSSL::PKey::RSA.new(File.read(@ssl_key),@ssl_key_passphrase.value)
@@ -110,28 +120,8 @@ class LogStash::Inputs::Relp < LogStash::Inputs::Base
   def run(output_queue)
     while !stop?
       begin
-        # Start a new thread for each connection.
-        Thread.start(@relp_server.accept) do |client|
-            rs = client[0]
-            socket = client[1]
-          begin
-            rs.relp_setup_connection(socket)
-            # monkeypatch a 'peer' method onto the socket.
-            socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
-            peer = socket.peer
-            @logger.debug("Relp Connection to #{peer} created")
-            relp_stream(rs,socket, output_queue, peer)
-          rescue Relp::ConnectionClosed => e
-            @logger.debug("Relp Connection to #{peer} Closed")
-          rescue Relp::RelpError => e
-            @logger.warn('Relp error: '+e.class.to_s+' '+e.message)
-            #TODO: Still not happy with this, are they all warn level?
-            #Will this catch everything I want it to?
-            #Relp spec says to close connection on error, ensure this is the case
-          ensure
-            socket.close rescue nil
-          end
-        end # Thread.start
+        server, socket = @relp_server.accept
+        connection_thread(server, socket, output_queue)
       rescue Relp::InvalidCommand,Relp::InappropriateCommand => e
         @logger.warn('Relp client trying to open connection with something other than open:'+e.message)
       rescue Relp::InsufficientCommands
@@ -140,18 +130,58 @@ class LogStash::Inputs::Relp < LogStash::Inputs::Base
         @logger.debug('Relp Connection closed')
       rescue OpenSSL::SSL::SSLError => ssle
         # NOTE(mrichar1): This doesn't return a useful error message for some reason
-        @logger.error("SSL Error", :exception => ssle,
-                      :backtrace => ssle.backtrace)
-       end
-    end # loop
+        @logger.error("SSL Error", :exception => ssle, :backtrace => ssle.backtrace)
+      rescue => e
+        # if this exception occured while the plugin is stopping
+        # just ignore and exit
+        raise e unless stop?
+      end
+    end
   end # def run
 
   def stop
+    # force close @relp_server which will escape any blocking read with a IO exception
+    # and any thread using them will exit.
+    # catch all rescue nil to discard any close errors or invalid socket
+    if @relp_server
+      @relp_server.shutdown rescue nil
+      @relp_server = nil
+    end
+  end
+
+  def close
+    # see related comment in register: we must make sure to close the @relp_server here
+    # because it is created in the register method and we could be in the context of having
+    # register called but never run & stop, only close.
     if @relp_server
       @relp_server.shutdown rescue nil
       @relp_server = nil
     end
   end
+
+  private
+
+  def connection_thread(server, socket, output_queue)
+    Thread.start(server, socket, output_queue) do |server, socket, output_queue|
+      begin
+        server.relp_setup_connection(socket)
+        peer = socket.peer
+        @logger.debug("Relp Connection to #{peer} created")
+        relp_stream(server,socket, output_queue, peer)
+      rescue Relp::ConnectionClosed => e
+        @logger.debug("Relp Connection to #{peer} Closed")
+      rescue Relp::RelpError => e
+        @logger.warn('Relp error: '+e.class.to_s+' '+e.message)
+        #TODO: Still not happy with this, are they all warn level?
+        #Will this catch everything I want it to?
+        #Relp spec says to close connection on error, ensure this is the case
+      rescue => e
+        # ignore other exceptions
+      ensure
+        socket.close rescue nil
+      end
+    end
+  end
 end # class LogStash::Inputs::Relp
 
 #TODO: structured error logging

data/lib/logstash/util/relp.rb

@@ -15,7 +15,7 @@ class Relp#This isn't much use on its own, but gives RelpServer and RelpClient t
 
   def valid_command?(command)
     valid_commands = Array.new
-    
+
     #Allow anything in the basic protocol for both directions
     valid_commands << 'open'
     valid_commands << 'close'
@@ -102,7 +102,7 @@ class RelpServer < Relp
 
   def initialize(host,port,required_commands=[],ssl_context=nil)
     @logger = Cabin::Channel.get(LogStash)
-    
+
     @server=true
 
     #These are things that are part of the basic protocol, but only valid in one direction (rsp, close etc.)
@@ -197,14 +197,13 @@ class RelpServer < Relp
     frame['command'] = 'serverclose'
     begin
       self.frame_write(socket,frame)
-      socket.close
+      socket.close rescue nil
     rescue ConnectionClosed
     end
   end
 
   def shutdown
-    @server.close
-  rescue Exception#@server might already be down
+    @server.close rescue nil
   end
 
   def ack(socket, txnr)

data/logstash-input-relp.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-relp'
-  s.version         = '2.0.1'
+  s.version         = '2.0.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Read RELP events over a TCP socket."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/relp_spec.rb

@@ -4,6 +4,8 @@ require_relative "../support/ssl"
 
 describe LogStash::Inputs::Relp do
 
+  let!(:helper) { RelpHelpers.new }
+
   before do
     srand(RSpec.configuration.seed)
   end
@@ -13,12 +15,11 @@ describe LogStash::Inputs::Relp do
     it "should register without errors" do
       input = LogStash::Plugin.lookup("input", "relp").new("port" => 1234)
       expect {input.register}.to_not raise_error
+      input.close rescue nil
     end
-
   end
 
   describe "when interrupting the plugin" do
-
     let(:port) { rand(1024..65532) }
 
     it_behaves_like "an interruptible input plugin" do
@@ -28,9 +29,13 @@ describe LogStash::Inputs::Relp do
 
   describe "multiple client connections" do
 
-    let(:nclients) { rand(200) }
+    # (colinsurprenant) don't put number of simultaneous clients too high,
+    # it seems to lock no sure why. the test client code needs a very serious,
+    # refactoring, its too complex and very hard to debug :P
+    let(:nclients) { rand(10) }
+
     let(:nevents)  { 100 }
-    let(:port)     { 5512 }
+    let(:port) { rand(1024..65532) }
 
     let(:conf) do
       <<-CONFIG
@@ -43,15 +48,22 @@ describe LogStash::Inputs::Relp do
       CONFIG
     end
 
-    let(:clients) { setup_clients(nclients, port) }
+    let(:clients) { RelpHelpers.setup_clients(nclients, port) }
 
     let(:events) do
-      input(conf, (nevents*nclients)) do
+      input(conf) do |pipeline, queue|
         nevents.times do |value|
           clients.each_with_index do |client, index|
             client.syslog_write("Hello from client#{index}")
           end
         end
+        (nevents * nclients).times.collect { queue.pop }
+      end
+    end
+
+    after :each do
+      clients.each do |client|
+        client.close rescue nil
       end
     end
 
@@ -64,50 +76,60 @@ describe LogStash::Inputs::Relp do
 
   describe "SSL support" do
 
-    let(:nevents) { 100 }
     let(:certificate) { RelpTest.certificate }
-    let(:port)        { 5513 }
+    let(:port) { rand(1024..65532) }
 
-    let(:conf) do
-      <<-CONFIG
-        input {
-          relp {
-            type => "blah"
-            port => #{port}
-            ssl_enable => true
-            ssl_verify => false
-            ssl_cert => "#{certificate.ssl_cert}"
-            ssl_key  => "#{certificate.ssl_key}"
+    context "events reading" do
+
+      let(:nevents) { 100 }
+
+      let(:conf) do
+        <<-CONFIG
+          input {
+            relp {
+              type => "blah"
+              port => #{port}
+              ssl_enable => true
+              ssl_verify => false
+              ssl_cert => "#{certificate.ssl_cert}"
+              ssl_key  => "#{certificate.ssl_key}"
+           }
          }
-       }
-      CONFIG
-    end
+        CONFIG
+      end
 
-    let(:client) { RelpClient.new("0.0.0.0", port, ["syslog"], {:ssl => true}) }
+      let(:client) { RelpClient.new("0.0.0.0", port, ["syslog"], {:ssl => true}) }
 
-    let(:events) do
-      input(conf, nevents) do
-        nevents.times do
-          client.syslog_write("Hello from client")
+      let!(:events) do
+        input(conf) do |pipeline, queue|
+          nevents.times do
+            client.syslog_write("Hello from client")
+          end
+          nevents.times.collect { queue.pop }
         end
       end
+
+      after :each do
+        client.close rescue nil
+      end
+
+      it "should generated the events as expected" do
+        expect(events).to have(nevents).with("Hello from client")
+      end
     end
 
     context "registration and close" do
 
       it "should register without errors" do
-        input = LogStash::Plugin.lookup("input", "relp").new("port" => 1235, "ssl_enable" => true,
-                                                             "ssl_cert" => certificate.ssl_cert,
-                                                             "ssl_key" => certificate.ssl_key)
+        input = LogStash::Plugin.lookup("input", "relp").new(
+          "port" => port,
+          "ssl_enable" => true,
+          "ssl_cert" => certificate.ssl_cert,
+          "ssl_key" => certificate.ssl_key
+        )
         expect {input.register}.to_not raise_error
+        input.close rescue nil
       end
-
     end
-
-    it "should generated the events as expected" do
-      expect(events).to have(nevents).with("Hello from client")
-    end
-
   end
-
 end

data/spec/spec_helper.rb

@@ -5,62 +5,26 @@ require "logstash/util/relp"
 require "socket"
 require "support/client"
 
-module RelpHelpers
+class RelpHelpers
 
-  def setup_clients(number_of_clients, port)
+  def self.setup_clients(number_of_clients, port)
     number_of_clients.times.inject([]) do |clients|
       clients << RelpClient.new("0.0.0.0", port, ["syslog"])
     end
   end
 
-  def filter(events, message)
+  def self.filter(events, message)
      events.select{|event| event["message"] == message }
   end
-
-  def input(config, size, &block)
-    pipeline = LogStash::Pipeline.new(config)
-    queue = Queue.new
-
-    pipeline.instance_eval do
-      # create closure to capture queue
-      @output_func = lambda { |event| queue << event }
-
-      # output_func is now a method, call closure
-      def output_func(event)
-        @output_func.call(event)
-      end
-    end
-
-    pipeline_thread = Thread.new { pipeline.run }
-    sleep 0.1 while !pipeline.ready?
-
-    block.call
-    sleep 0.1 while queue.size != size
-
-    result = size.times.inject([]) do |acc|
-      acc << queue.pop
-    end
-
-    pipeline.shutdown
-    pipeline_thread.join
-
-    result
-  end # def input
-
-end
-
-RSpec.configure do |c|
-  c.include RelpHelpers
 end
 
 RSpec::Matchers.define :have do |nevents|
 
   match do |events|
-    filter(events, @pattern).size == nevents
+    RelpHelpers.filter(events, @pattern).size == nevents
   end
 
   chain :with do |pattern|
     @pattern = pattern
   end
-
-end
+end

data/spec/support/client.rb

@@ -33,7 +33,7 @@ class RelpClient < Relp
       end
     end
 
-    #This'll start the automatic frame numbering 
+    #This'll start the automatic frame numbering
     @lasttxnr = 0
 
     offer=Hash.new
@@ -56,7 +56,7 @@ class RelpClient < Relp
 
       #subtracting one array from the other checks to see if all elements in @required_relp_commands are present in the offer
     elsif ! (@required_relp_commands - response['commands'].split(',')).empty?
-      #if it can't receive syslog it's useless to us; close the connection 
+      #if it can't receive syslog it's useless to us; close the connection
       self.close()
       raise InsufficientCommands, response['commands'] + ' offered, require '
       + @required_relp_commands.join(',')
@@ -67,7 +67,12 @@ class RelpClient < Relp
     #This thread deals with responses that come back
     reader = Thread.start do
       loop do
-        f = self.frame_read(@socket)
+        begin
+          f = self.frame_read(@socket)
+        rescue
+          # ignore exceptions and quit thread
+          break
+        end
         if f['command'] == 'rsp' && f['message'] == '200 OK'
           @buffer.delete(f['txnr'])
         elsif f['command'] == 'rsp' && f['message'][0,1] == '5'
@@ -88,14 +93,19 @@ class RelpClient < Relp
     Thread.start do
       old_buffer = Hash.new
       loop do
-        #This returns old txnrs that are still present
-        (@buffer.keys & old_buffer.keys).each do |txnr|
-          new_txnr = self.frame_write(@socket, @buffer[txnr])
-          @buffer[new_txnr] = @buffer[txnr]
-          @buffer.delete(txnr)
+        begin
+          #This returns old txnrs that are still present
+          (@buffer.keys & old_buffer.keys).each do |txnr|
+            new_txnr = self.frame_write(@socket, @buffer[txnr])
+            @buffer[new_txnr] = @buffer[txnr]
+            @buffer.delete(txnr)
+          end
+          old_buffer = @buffer
+          sleep @retransmission_timeout
+        rescue
+          # ignore exceptions and quit thread
+          break
         end
-        old_buffer = @buffer
-        sleep @retransmission_timeout
       end
     end
   end
@@ -107,8 +117,9 @@ class RelpClient < Relp
     @close_txnr=self.frame_write(@socket, frame)
     #TODO: ought to properly wait for a reply etc. The serverclose will make it work though
     sleep @retransmission_timeout
-    @socket.close#TODO: shutdown?
     return @buffer
+  ensure
+    @socket.close
   end
 
   def syslog_write(logline)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-relp
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-08 00:00:00.000000000 Z
+date: 2015-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement