log-courier 1.10.0 → 2.7.0

data/lib/log-courier/client_tcp.rb

@@ -1,6 +1,4 @@
-# encoding: utf-8
-
-# Copyright 2014-2019 Jason Woods and Contributors.
+# Copyright 2014-2021 Jason Woods and Contributors.
 #
 # This file is a modification of code from Logstash Forwarder.
 # Copyright 2012-2013 Jordan Sissel and contributors.
@@ -19,42 +17,47 @@
 
 require 'openssl'
 require 'socket'
-require 'thread'
 
 module LogCourier
   # TLS transport implementation
   class ClientTcp
     def initialize(options = {})
       @options = {
-        logger:             nil,
-        transport:         'tls',
-        ssl_ca:             nil,
-        ssl_certificate:    nil,
-        ssl_key:            nil,
+        logger: nil,
+        transport: 'tls',
+        ssl_ca: nil,
+        ssl_certificate: nil,
+        ssl_key: nil,
         ssl_key_passphrase: nil,
+        min_tls_version: 1.2,
+        disable_handshake: false,
       }.merge!(options)
 
       @logger = @options[:logger]
 
       [:port, :ssl_ca].each do |k|
-        fail "output/courier: '#{k}' is required" if @options[k].nil?
+        raise "output/courier: '#{k}' is required" if @options[k].nil?
       end
 
-      if @options[:transport] == 'tls'
-        c = 0
-        [:ssl_certificate, :ssl_key].each do
-          c += 1
-        end
-        fail 'output/courier: \'ssl_certificate\' and \'ssl_key\' must be specified together' if c == 1
+      return unless @options[:transport] == 'tls'
+
+      c = 0
+      [:ssl_certificate, :ssl_key].each do
+        c += 1
       end
+      raise 'output/courier: \'ssl_certificate\' and \'ssl_key\' must be specified together' if c == 1
     end
 
     def connect(io_control)
       loop do
         begin
-          break if tls_connect
+          if tls_connect
+            return unless handshake(io_control)
+
+            break
+          end
         rescue ShutdownSignal
-          raise
+          return
         end
 
         # TODO: Make this configurable
@@ -65,47 +68,44 @@ module LogCourier
       @send_paused = false
 
       @send_thread = Thread.new do
-        begin
-          run_send io_control
-        rescue ShutdownSignal
-        rescue StandardError, NativeException => e
-          @logger.warn e, :hint => 'Unknown write error' unless @logger.nil?
-          io_control << ['F']
-          return
-        end
+        run_send io_control
+      rescue ShutdownSignal
+        # Shutdown
+      rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby
+        @logger&.warn e, hint: 'Unknown write error'
+        io_control << ['F']
       end
       @recv_thread = Thread.new do
-        begin
-          run_recv io_control
-        rescue ShutdownSignal
-        rescue StandardError, NativeException => e
-          @logger.warn e, :hint => 'Unknown read error' unless @logger.nil?
-          io_control << ['F']
-          return
-        end
+        run_recv io_control
+      rescue ShutdownSignal
+        # Shutdown
+      rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby
+        @logger&.warn e, hint: 'Unknown read error'
+        io_control << ['F']
       end
-      return
+      nil
     end
 
     def disconnect
-      @send_thread.raise ShutdownSignal
-      @send_thread.join
-      @recv_thread.raise ShutdownSignal
-      @recv_thread.join
-      return
+      @send_thread&.raise ShutdownSignal
+      @send_thread&.join
+      @recv_thread&.raise ShutdownSignal
+      @recv_thread&.join
+      nil
     end
 
     def send(signature, message)
       # Add to send queue
-      @send_q << [signature, message.length].pack('A4N') + message
-      return
+      @send_q << ([signature, message.length].pack('A4N') + message)
+      nil
     end
 
     def pause_send
       return if @send_paused
+
       @send_paused = true
       @send_q << nil
-      return
+      nil
     end
 
     def send_paused?
@@ -117,11 +117,35 @@ module LogCourier
         @send_paused = false
         @send_q << nil
       end
-      return
+      nil
     end
 
     private
 
+    def handshake(io_control)
+      return true if @options[:disable_handshake]
+
+      @socket.write ['HELO', 8, 0, 2, 7, 0, 'RYLC'].pack('A4NCCCCA4')
+
+      signature, data = receive
+      if signature != 'VERS'
+        raise "Unexpected message during handshake: #{signature}" if signature != '????'
+
+        @vers = Protocol.parse_helo_vers('')
+        @logger&.info 'Remote does not support protocol handshake', server_version: @vers[:client_version]
+        return true
+      end
+
+      @vers = Protocol.parse_helo_vers(data)
+      @logger&.info 'Remote identified', server_version: @vers[:client_version]
+
+      true
+    rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby
+      @logger&.warn e, hint: 'Unknown write error'
+      io_control << ['F']
+      false
+    end
+
     def run_send(io_control)
       # Ask for something to send
       io_control << ['S']
@@ -147,55 +171,52 @@ module LogCourier
           # Ask for more to send while we send this one
           io_control << ['S'] unless paused
 
-          @ssl_client.write message
+          @socket.write message
         end
       end
-      return
     rescue OpenSSL::SSL::SSLError => e
-      @logger.warn 'SSL write error', :error => e.message unless @logger.nil?
+      @logger&.warn 'SSL write error', error: e.message
       io_control << ['F']
-      return
     rescue IOError, Errno::ECONNRESET => e
-      @logger.warn 'Write error', :error => e.message unless @logger.nil?
+      @logger&.warn 'Write error', error: e.message
       io_control << ['F']
-      return
     end
 
     def run_recv(io_control)
       loop do
-        # Grab a header
-        header = @ssl_client.read(8)
-        fail EOFError if header.nil?
-
-        # Decode signature and length
-        signature, length = header.unpack('A4N')
-
-        if length > 1048576
-          # Too big raise error
-          @logger.warn 'Invalid message: data too big', :data_length => length unless @logger.nil?
-          io_control << ['F']
-          break
-        end
-
-        # Read remainder
-        message = @ssl_client.read(length)
+        signature, message = receive
 
         # Pass through to receive
         io_control << ['R', signature, message]
       end
-      return
     rescue OpenSSL::SSL::SSLError => e
-      @logger.warn 'SSL read error', :error => e.message unless @logger.nil?
-      io_control << ['F']
-      return
-    rescue IOError, Errno::ECONNRESET => e
-      @logger.warn 'Read error', :error => e.message unless @logger.nil?
+      @logger&.warn 'SSL read error', error: e.message
       io_control << ['F']
-      return
     rescue EOFError
-      @logger.warn 'Connection closed by server' unless @logger.nil?
+      @logger&.warn 'Connection closed by server'
+      io_control << ['F']
+    rescue IOError, Errno::ECONNRESET => e
+      @logger&.warn 'Read error', error: e.message
       io_control << ['F']
-      return
+    end
+
+    def receive
+      # Grab a header
+      header = @socket.read(8)
+      raise EOFError if header.nil?
+
+      # Decode signature and length
+      signature, length = header.unpack('A4N')
+
+      if length > 1_048_576
+        # Too big raise error
+        raise IOError, 'Invalid message: data too big'
+      end
+
+      # Read remainder
+      message = @socket.read(length)
+
+      [signature, message]
     end
 
     def tls_connect
@@ -203,7 +224,7 @@ module LogCourier
       address = @options[:addresses][0]
       port = @options[:port]
 
-      @logger.info 'Connecting', :address => address, :port => port unless @logger.nil?
+      @logger&.info 'Connecting', address: address, port: port
 
       begin
         tcp_socket = TCPSocket.new(address, port)
@@ -216,8 +237,15 @@ module LogCourier
           ssl.set_params
           # Modify the default options to ensure SSLv2 and SSLv3 is disabled
           # This retains any beneficial options set by default in the current Ruby implementation
-          ssl.options |= OpenSSL::SSL::OP_NO_SSLv2 if defined?(OpenSSL::SSL::OP_NO_SSLv2)
-          ssl.options |= OpenSSL::SSL::OP_NO_SSLv3 if defined?(OpenSSL::SSL::OP_NO_SSLv3)
+          # TODO: https://github.com/jruby/jruby-openssl/pull/215 is fixed in JRuby 9.3.0.0
+          #       As of 7.15 Logstash, JRuby version is still 9.2
+          #       Once 9.3 is in use we can switch to using min_version and max_version
+          ssl.options |= OpenSSL::SSL::OP_NO_SSLv2
+          ssl.options |= OpenSSL::SSL::OP_NO_SSLv3
+          ssl.options |= OpenSSL::SSL::OP_NO_TLSv1 if @options[:min_tls_version] > 1
+          ssl.options |= OpenSSL::SSL::OP_NO_TLSv1_1 if @options[:min_tls_version] > 1.1
+          ssl.options |= OpenSSL::SSL::OP_NO_TLSv1_2 if @options[:min_tls_version] > 1.2
+          raise 'Invalid min_tls_version - max is 1.3' if @options[:min_tls_version] > 1.3
 
           # Set the certificate file
           unless @options[:ssl_certificate].nil?
@@ -230,26 +258,28 @@ module LogCourier
           ssl.cert_store = cert_store
           ssl.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
 
-          @ssl_client = OpenSSL::SSL::SSLSocket.new(tcp_socket, ssl)
-
-          socket = @ssl_client.connect
+          @socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, ssl)
+          @socket.connect
 
           # Verify certificate
-          socket.post_connection_check(address)
+          @socket.post_connection_check(address)
+
+          @logger&.info 'Connected successfully', address: address, port: port, ssl_version: @socket.ssl_version
         else
-          socket = tcp_socket.connect
+          @socket = tcp_socket
+
+          @logger&.info 'Connected successfully', address: address, port: port
         end
 
         # Add extra logging data now we're connected
         @logger['address'] = address
         @logger['port'] = port
 
-        @logger.info 'Connected successfully' unless @logger.nil?
         return true
       rescue OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET => e
-        @logger.warn 'Connection failed', :error => e.message, :address => address, :port => port unless @logger.nil?
-      rescue StandardError, NativeException => e
-        @logger.warn e, :hint => 'Unknown connection failure', :address => address, :port => port unless @logger.nil?
+        @logger&.warn 'Connection failed', error: e.message, address: address, port: port
+      rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby
+        @logger&.warn e, hint: 'Unknown connection failure', address: address, port: port
       end
 
       false

data/lib/log-courier/event_queue.rb

@@ -1,6 +1,4 @@
-# encoding: utf-8
-
-# Copyright 2014-2019 Jason Woods and Contributors.
+# Copyright 2014-2021 Jason Woods and Contributors.
 #
 # This file is a modification of code from Ruby.
 # Ruby is copyrighted free software by Yukihiro Matsumoto <matz@netlab.jp>.
@@ -28,23 +26,24 @@
 # The majority of the code is taken from Ruby's SizedQueue<Queue implementation.
 #
 module LogCourier
+  # EventQueue
   class EventQueue
     #
     # Creates a fixed-length queue with a maximum size of +max+.
     #
     def initialize(max)
-      fail ArgumentError, "queue size must be positive" unless max > 0
+      raise ArgumentError, 'queue size must be positive' unless max.positive?
+
       @max = max
       @enque_cond = ConditionVariable.new
       @num_enqueue_waiting = 0
 
       @que = []
-      @que.taint          # enable tainted communication
+      @que.taint # enable tainted communication
       @num_waiting = 0
-      self.taint
+      taint
       @mutex = Mutex.new
       @cond = ConditionVariable.new
-      return
     end
 
     #
@@ -56,7 +55,7 @@ module LogCourier
     # Sets the maximum size of the queue.
     #
     def max=(max)
-      fail ArgumentError, "queue size must be positive" unless max > 0
+      raise ArgumentError, 'queue size must be positive' unless max.positive?
 
       @mutex.synchronize do
         if max <= @max
@@ -69,7 +68,6 @@ module LogCourier
           end
         end
       end
-      max
     end
 
     #
@@ -77,19 +75,19 @@ module LogCourier
     # until space becomes available, up to a maximum of +timeout+ seconds.
     #
     def push(obj, timeout = nil)
-      unless timeout.nil?
-        start = Time.now
-      end
+      start = Time.now unless timeout.nil?
       @mutex.synchronize do
         loop do
           break if @que.length < @max
+
           @num_enqueue_waiting += 1
           begin
             @enque_cond.wait @mutex, timeout
           ensure
             @num_enqueue_waiting -= 1
           end
-          fail TimeoutError if !timeout.nil? and Time.now - start >= timeout
+
+          raise TimeoutError if !timeout.nil? && Time.now - start >= timeout
         end
 
         @que.push obj
@@ -112,11 +110,9 @@ module LogCourier
     # Retrieves data from the queue and runs a waiting thread, if any.
     #
     def pop(*args)
-      retval = pop_timeout *args
+      retval = pop_timeout(*args)
       @mutex.synchronize do
-        if @que.length < @max
-          @enque_cond.signal
-        end
+        @enque_cond.signal if @que.length < @max
       end
       retval
     end
@@ -182,23 +178,22 @@ module LogCourier
     # raised.
     #
     def pop_timeout(timeout = nil)
-      unless timeout.nil?
-        start = Time.now
-      end
+      start = Time.now unless timeout.nil?
       @mutex.synchronize do
         loop do
           return @que.shift unless @que.empty?
-          fail TimeoutError if timeout == 0
+          raise TimeoutError if !timeout.nil? && timeout.zero?
+
           begin
             @num_waiting += 1
             @cond.wait @mutex, timeout
           ensure
             @num_waiting -= 1
           end
-          fail TimeoutError if !timeout.nil? and Time.now - start >= timeout
+          raise TimeoutError if !timeout.nil? && Time.now - start >= timeout
         end
       end
-      return
+      nil
     end
   end
 end

data/lib/log-courier/server.rb

@@ -1,6 +1,4 @@
-# encoding: utf-8
-
-# Copyright 2014-2019 Jason Woods and Contributors.
+# Copyright 2014-2021 Jason Woods and Contributors.
 #
 # This file is a modification of code from Logstash Forwarder.
 # Copyright 2012-2013 Jordan Sissel and contributors.
@@ -18,44 +16,31 @@
 # limitations under the License.
 
 require 'log-courier/event_queue'
+require 'log-courier/protocol'
 require 'multi_json'
-require 'thread'
 require 'zlib'
 
-class NativeException; end
+# NativeException in case it is missing
+class NativeException
+  def dummy; end
+end
 
 module LogCourier
   class TimeoutError < StandardError; end
+
   class ShutdownSignal < StandardError; end
+
   class ProtocolError < StandardError; end
 
   # Implementation of the server
   class Server
     attr_reader :port
 
-    # TODO(driskell): Consolidate singleton into another file
-    class << self
-      @json_adapter
-      @json_parseerror
-
-      def get_json_adapter
-        @json_adapter = MultiJson.adapter.instance if @json_adapter.nil?
-        @json_adapter
-      end
-
-      def get_json_parseerror
-        if @json_parseerror.nil?
-          @json_parseerror = get_json_adapter.class::ParseError
-        end
-        @json_parseerror
-      end
-    end
-
     def initialize(options = {})
       @options = {
-        logger:     nil,
-        transport:  'tls',
-        raw_events: true,
+        logger: nil,
+        transport: 'tls',
+        disable_handshake: false,
       }.merge!(options)
 
       @logger = @options[:logger]
@@ -64,11 +49,8 @@ module LogCourier
       when 'tcp', 'tls'
         require 'log-courier/server_tcp'
         @server = ServerTcp.new(@options)
-      when 'plainzmq', 'zmq'
-        require 'log-courier/server_zmq'
-        @server = ServerZmq.new(@options)
       else
-        fail 'input/courier: \'transport\' must be tcp, tls, plainzmq or zmq'
+        raise 'input/courier: \'transport\' must be tcp or tls'
       end
 
       # Grab the port back and update the logger context
@@ -91,9 +73,9 @@ module LogCourier
               process_jdat message, comm, @event_queue
             else
               if comm.peer.nil?
-                @logger.warn 'Unknown message received', :from => 'unknown' unless @logger.nil?
+                @logger&.warn 'Unknown message received', from: 'unknown'
               else
-                @logger.warn 'Unknown message received', :from => comm.peer unless @logger.nil?
+                @logger&.warn 'Unknown message received', from: comm.peer
               end
               # Don't kill a client that sends a bad message
               # Just reject it and let it send it again, potentially to another server
@@ -105,6 +87,7 @@ module LogCourier
         loop do
           event = @event_queue.pop
           break if event.nil?
+
           block.call event
         end
       ensure
@@ -114,25 +97,23 @@ module LogCourier
           server_thread.join
         end
       end
-      return
+      nil
     end
 
     def stop
       @event_queue << nil
+      nil
     end
 
     private
 
     def process_ping(message, comm)
       # Size of message should be 0
-      if message.bytesize != 0
-        fail ProtocolError, "unexpected data attached to ping message (#{message.bytesize})"
-      end
+      raise ProtocolError, "unexpected data attached to ping message (#{message.bytesize})" unless message.bytesize.zero?
 
       # PONG!
       # NOTE: comm.send can raise a Timeout::Error of its own
       comm.send 'PONG', ''
-      return
     end
 
     def process_jdat(message, comm, event_queue)
@@ -142,13 +123,11 @@ module LogCourier
       # OK - first is a nonce - we send this back with sequence acks
       # This allows the client to know what is being acknowledged
       # Nonce is 16 so check we have enough
-      if message.bytesize < 17
-        fail ProtocolError, "JDAT message too small (#{message.bytesize})"
-      end
+      raise ProtocolError, "JDAT message too small (#{message.bytesize})" if message.bytesize < 17
 
       nonce = message[0...16]
 
-      if !@logger.nil? && @logger.debug?
+      if @logger&.debug?
         nonce_str = nonce.each_byte.map do |b|
           b.to_s(16).rjust(2, '0')
         end
@@ -162,25 +141,21 @@ module LogCourier
       # We acknowledge them by their 1-index position in the stream
       # A 0 sequence acknowledgement means we haven't processed any yet
       sequence = 0
-      events = []
       length_buf = ''
       data_buf = ''
       loop do
         ret = message.read 4, length_buf
-        if ret.nil?
-          # Finished!
-          break
-        elsif length_buf.bytesize < 4
-          fail ProtocolError, "JDAT length extraction failed (#{ret} #{length_buf.bytesize})"
-        end
+        # Finished?
+        break if ret.nil?
+        raise ProtocolError, "JDAT length extraction failed (#{ret} #{length_buf.bytesize})" if length_buf.bytesize < 4
 
-        length = length_buf.unpack('N').first
+        length = length_buf.unpack1('N')
 
         # Extract message
         ret = message.read length, data_buf
-        if ret.nil? or data_buf.bytesize < length
-          @logger.warn()
-          fail ProtocolError, "JDAT message extraction failed #{ret} #{data_buf.bytesize}"
+        if ret.nil? || data_buf.bytesize < length
+          @logger&.warn()
+          raise ProtocolError, "JDAT message extraction failed #{ret} #{data_buf.bytesize}"
         end
 
         data_buf.force_encoding('utf-8')
@@ -200,9 +175,9 @@ module LogCourier
 
         # Decode the JSON
         begin
-          event = self.class.get_json_adapter.load(data_buf, :raw => @options[:raw_events])
-        rescue self.class.get_json_parseerror => e
-          @logger.warn e, :invalid_encodings => invalid_encodings, :hint => 'JSON parse failure, falling back to plain-text' unless @logger.nil?
+          event = MultiJson.load(data_buf)
+        rescue MultiJson::ParseError => e
+          @logger&.warn e, invalid_encodings: invalid_encodings, hint: 'JSON parse failure, falling back to plain-text'
           event = { 'message' => data_buf }
         end
 
@@ -215,7 +190,7 @@ module LogCourier
         rescue TimeoutError
           # Full pipeline, partial ack
           # NOTE: comm.send can raise a Timeout::Error of its own
-          @logger.debug 'Partially acknowledging message', :nonce => nonce_str.join, :sequence => sequence if !@logger.nil? && @logger.debug?
+          @logger&.debug 'Partially acknowledging message', nonce: nonce_str.join, sequence: sequence if @logger&.debug?
           comm.send 'ACKN', [nonce, sequence].pack('a*N')
           ack_timeout = Time.now.to_i + 5
           retry
@@ -226,9 +201,8 @@ module LogCourier
 
       # Acknowledge the full message
       # NOTE: comm.send can raise a Timeout::Error
-      @logger.debug 'Acknowledging message', :nonce => nonce_str.join, :sequence => sequence if !@logger.nil? && @logger.debug?
+      @logger&.debug 'Acknowledging message', nonce: nonce_str.join, sequence: sequence if @logger&.debug?
       comm.send 'ACKN', [nonce, sequence].pack('A*N')
-      return
     end
   end
 end