log-courier 1.10.0 → 2.7.0

data/lib/log-courier/server_tcp.rb

@@ -1,6 +1,4 @@
-# encoding: utf-8
-
-# Copyright 2014-2019 Jason Woods and Contributors.
+# Copyright 2014-2021 Jason Woods and Contributors.
 #
 # This file is a modification of code from Logstash Forwarder.
 # Copyright 2012-2013 Jordan Sissel and contributors.
@@ -19,7 +17,6 @@
 
 require 'openssl'
 require 'socket'
-require 'thread'
 
 module LogCourier
   # Wrap around TCPServer to grab last error for use in reporting which peer had an error
@@ -42,12 +39,12 @@ module LogCourier
         peer = sock.peeraddr
       end
       @peer = "#{peer[2]}:#{peer[1]}"
-      return sock
+      sock
     end
 
     def reset_peer
       @peer = 'unknown'
-      return
+      nil
     end
   end
 
@@ -58,29 +55,31 @@ module LogCourier
     # Create a new TLS transport endpoint
     def initialize(options = {})
       @options = {
-        logger:                nil,
-        transport:             'tls',
-        port:                  0,
-        address:               '0.0.0.0',
-        ssl_certificate:       nil,
-        ssl_key:               nil,
-        ssl_key_passphrase:    nil,
-        ssl_verify:            false,
+        logger: nil,
+        transport: 'tls',
+        port: 0,
+        address: '0.0.0.0',
+        ssl_certificate: nil,
+        ssl_key: nil,
+        ssl_key_passphrase: nil,
+        ssl_verify: false,
         ssl_verify_default_ca: false,
-        ssl_verify_ca:         nil,
-        max_packet_size:       10_485_760,
-        add_peer_fields:       false,
+        ssl_verify_ca: nil,
+        max_packet_size: 10_485_760,
+        add_peer_fields: false,
+        min_tls_version: 1.2,
+        disable_handshake: false,
       }.merge!(options)
 
       @logger = @options[:logger]
 
       if @options[:transport] == 'tls'
         [:ssl_certificate, :ssl_key].each do |k|
-          fail "input/courier: '#{k}' is required" if @options[k].nil?
+          raise "input/courier: '#{k}' is required" if @options[k].nil?
         end
 
-        if @options[:ssl_verify] and (!@options[:ssl_verify_default_ca] && @options[:ssl_verify_ca].nil?)
-          fail 'input/courier: Either \'ssl_verify_default_ca\' or \'ssl_verify_ca\' must be specified when ssl_verify is true'
+        if @options[:ssl_verify] && (!@options[:ssl_verify_default_ca] && @options[:ssl_verify_ca].nil?)
+          raise 'input/courier: Either \'ssl_verify_default_ca\' or \'ssl_verify_ca\' must be specified when ssl_verify is true'
         end
       end
 
@@ -99,8 +98,15 @@ module LogCourier
           ssl.set_params
           # Modify the default options to ensure SSLv2 and SSLv3 is disabled
           # This retains any beneficial options set by default in the current Ruby implementation
-          ssl.options |= OpenSSL::SSL::OP_NO_SSLv2 if defined?(OpenSSL::SSL::OP_NO_SSLv2)
-          ssl.options |= OpenSSL::SSL::OP_NO_SSLv3 if defined?(OpenSSL::SSL::OP_NO_SSLv3)
+          # TODO: https://github.com/jruby/jruby-openssl/pull/215 is fixed in JRuby 9.3.0.0
+          #       As of 7.15 Logstash, JRuby version is still 9.2
+          #       Once 9.3 is in use we can switch to using min_version and max_version
+          ssl.options |= OpenSSL::SSL::OP_NO_SSLv2
+          ssl.options |= OpenSSL::SSL::OP_NO_SSLv3
+          ssl.options |= OpenSSL::SSL::OP_NO_TLSv1 if @options[:min_tls_version] > 1
+          ssl.options |= OpenSSL::SSL::OP_NO_TLSv1_1 if @options[:min_tls_version] > 1.1
+          ssl.options |= OpenSSL::SSL::OP_NO_TLSv1_2 if @options[:min_tls_version] > 1.2
+          raise 'Invalid min_tls_version - max is 1.3' if @options[:min_tls_version] > 1.3
 
           # Set the certificate file
           ssl.cert = OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate]))
@@ -130,13 +136,11 @@ module LogCourier
           @server = @tcp_server
         end
 
-        if @options[:port] == 0
-          @logger.warn 'Ephemeral port allocated', :transport => @options[:transport], :port => @port unless @logger.nil?
-        end
-      rescue => e
+        @logger&.warn 'Ephemeral port allocated', transport: @options[:transport], port: @port if @options[:port].zero?
+      rescue StandardError, NativeException => e # Until Jruby updated we need NativeException
         raise "input/courier: Failed to initialise: #{e}"
       end
-    end # def initialize
+    end
 
     def run(&block)
       client_threads = {}
@@ -148,14 +152,18 @@ module LogCourier
         client = nil
         begin
           client = @server.accept
-        rescue EOFError, OpenSSL::SSL::SSLError, IOError => e
+        rescue OpenSSL::SSL::SSLError, IOError => e
           # Accept failure or other issue
-          @logger.warn 'Connection failed to accept', :error => e.message, :peer => @tcp_server.peer unless @logger.nil?
-          client.close rescue nil unless client.nil?
+          @logger&.warn 'Connection failed to accept', error: e.message, peer: @tcp_server.peer
+          begin
+            client&.close
+          rescue OpenSSL::SSL::SSLError, IOError
+            # Ignore IO error during close
+          end
           next
         end
 
-    	  @logger.info 'New connection', :peer => @tcp_server.peer unless @logger.nil?
+        @logger&.info 'New connection', peer: @tcp_server.peer
 
         # Clear up finished threads
         client_threads.delete_if do |_, thr|
@@ -167,13 +175,13 @@ module LogCourier
           run_thread client_copy, peer_copy, &block
         end
       end
-      return
+      nil
     rescue ShutdownSignal
-      return
-    rescue StandardError, NativeException => e
+      nil
+    rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby
       # Some other unknown problem
-      @logger.warn e.message, :hint => 'Unknown error, shutting down' unless @logger.nil?
-      return
+      @logger&.warn e.message, hint: 'Unknown error, shutting down'
+      nil
     ensure
       # Raise shutdown in all client threads and join then
       client_threads.each do |_, thr|
@@ -188,25 +196,28 @@ module LogCourier
     private
 
     def run_thread(client, peer, &block)
-      begin
-        # Perform the handshake inside the new thread so we don't block TCP accept
-        if @options[:transport] == 'tls'
+      # Perform the handshake inside the new thread so we don't block TCP accept
+      if @options[:transport] == 'tls'
+        begin
+          client.accept
+        rescue OpenSSL::SSL::SSLError, IOError => e
+          # Handshake failure or other issue
+          @logger&.warn 'Connection failed to initialise', error: e.message, peer: peer
           begin
-            client.accept
-          rescue EOFError, OpenSSL::SSL::SSLError, IOError => e
-            # Handshake failure or other issue
-            @logger.warn 'Connection failed to initialise', :error => e.message, :peer => peer unless @logger.nil?
             client.close
-            return
+          rescue OpenSSL::SSL::SSLError, IOError
+            # Ignore during close
           end
+          return
         end
 
-        ConnectionTcp.new(@logger, client, peer, @options).run(&block)
-      rescue ShutdownSignal
-        # Shutting down
-        @logger.info 'Server shutting down, connection closed', :peer => peer unless @logger.nil?
-        return
+        @logger&.info 'Connection setup successfully', peer: peer, ssl_version: client.ssl_version
       end
+
+      ConnectionTcp.new(@logger, client, peer, @options).run(&block)
+    rescue ShutdownSignal
+      # Shutting down
+      @logger&.info 'Server shutting down, connection closed', peer: peer
     end
   end
 
@@ -214,93 +225,105 @@ module LogCourier
   class ConnectionTcp
     attr_accessor :peer
 
-    def initialize(logger, fd, peer, options)
+    def initialize(logger, sfd, peer, options)
       @logger = logger
-      @fd = fd
+      @fd = sfd
       @peer = peer
       @peer_fields = {}
       @in_progress = false
       @options = options
+      @client = 'Unknown'
+      @major_version = 0
+      @minor_version = 0
+      @patch_version = 0
+      @version = '0.0.0'
+      @client_version = 'Unknown'
 
-      if @options[:add_peer_fields]
-        @peer_fields['peer'] = peer
-        if @options[:transport] == 'tls' && !@fd.peer_cert.nil?
-          @peer_fields['peer_ssl_cn'] = get_cn(@fd.peer_cert)
-        end
-      end
+      return unless @options[:add_peer_fields]
+
+      @peer_fields['peer'] = peer
+      return unless @options[:transport] == 'tls' && !@fd.peer_cert.nil?
+
+      @peer_fields['peer_ssl_cn'] = get_cn(@fd.peer_cert)
     end
 
     def add_fields(event)
-      event.merge! @peer_fields if @peer_fields.length != 0
+      event.merge! @peer_fields unless @peer_fields.empty?
     end
 
     def run(&block)
-      process_messages &block
-    rescue ShutdownSignal
-      # Shutting down
-      @logger.info 'Server shutting down, closing connection', :peer => @peer unless @logger.nil?
-      return
-    rescue StandardError, NativeException => e
-      # Some other unknown problem
-      @logger.warn e.message, :hint => 'Unknown error, connection aborted', :peer => @peer unless @logger.nil?
-      return
-    end
+      handshake(&block)
 
-    def process_messages
       loop do
-        # Read messages
-        # Each message begins with a header
-        # 4 byte signature
-        # 4 byte length
-        # Normally we would not parse this inside transport, but for TLS we have to in order to locate frame boundaries
-        signature, length = recv(8).unpack('A4N')
-
-        # Sanity
-        if length > @options[:max_packet_size]
-          fail ProtocolError, "packet too large (#{length} > #{@options[:max_packet_size]})"
-        end
-
-        # While we're processing, EOF is bad as it may occur during send
-        @in_progress = true
-
-        # Read the message
-        if length == 0
-          data = ''
-        else
-          data = recv(length)
-        end
+        signature, data = receive
 
         # Send for processing
         yield signature, data, self
-
-        # If we EOF next it's a graceful close
-        @in_progress = false
       end
     rescue TimeoutError
       # Timeout of the connection, we were idle too long without a ping/pong
-      @logger.warn 'Connection timed out', :peer => @peer unless @logger.nil?
-      return
+      @logger&.warn 'Connection timed out', peer: @peer
+      nil
     rescue EOFError
       if @in_progress
-        @logger.warn 'Unexpected EOF', :peer => @peer unless @logger.nil?
+        @logger&.warn 'Unexpected EOF', peer: @peer
       else
-        @logger.info 'Connection closed', :peer => @peer unless @logger.nil?
+        @logger&.info 'Connection closed', peer: @peer
       end
-      return
+      nil
     rescue OpenSSL::SSL::SSLError => e
       # Read errors, only action is to shutdown which we'll do in ensure
-      @logger.warn 'SSL error, connection aborted', :error => e.message, :peer => @peer unless @logger.nil?
-      return
+      @logger&.warn 'SSL error, connection aborted', error: e.message, peer: @peer
+      nil
     rescue IOError, Errno::ECONNRESET => e
       # Read errors, only action is to shutdown which we'll do in ensure
-      @logger.warn 'Connection aborted', :error => e.message, :peer => @peer unless @logger.nil?
-      return
+      @logger&.warn 'Connection aborted', error: e.message, peer: @peer
+      nil
     rescue ProtocolError => e
       # Connection abort request due to a protocol error
-      @logger.warn 'Protocol error, connection aborted', :error => e.message, :peer => @peer unless @logger.nil?
-      return
+      @logger&.warn 'Protocol error, connection aborted', error: e.message, peer: @peer
+      nil
+    rescue ShutdownSignal
+      # Shutting down
+      @logger&.info 'Server shutting down, closing connection', peer: @peer
+      nil
+    rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby
+      # Some other unknown problem
+      @logger&.warn e.message, hint: 'Unknown error, connection aborted', peer: @peer
+      nil
     ensure
-      @fd.close rescue nil
+      begin
+        @fd.close
+      rescue OpenSSL::SSL::SSLError, IOError, NativeException
+        # Ignore during close
+      end
+    end
+
+    def receive
+      # Read message
+      # Each message begins with a header
+      # 4 byte signature
+      # 4 byte length
+      # Normally we would not parse this inside transport, but for TLS we have to in order to locate frame boundaries
+      signature, length = recv(8).unpack('A4N')
+
+      # Sanity
+      raise ProtocolError, "packet too large (#{length} > #{@options[:max_packet_size]})" if length > @options[:max_packet_size]
+
+      # While we're processing, EOF is bad as it may occur during send
+      @in_progress = true
+
+      # Read the message
+      data = if length.zero?
+               ''
+             else
+               recv(length)
+             end
+
+      # If we EOF next it's a graceful close
+      @in_progress = false
+
+      [signature, data]
     end
 
     def send(signature, message)
@@ -311,24 +334,55 @@ module LogCourier
         begin
           written = @fd.write_nonblock(data[done...data.length])
         rescue IO::WaitReadable
-          fail TimeoutError if IO.select([@fd], nil, [@fd], @timeout - Time.now.to_i).nil?
+          raise TimeoutError if IO.select([@fd], nil, [@fd], @timeout - Time.now.to_i).nil?
+
           retry
         rescue IO::WaitWritable
-          fail TimeoutError if IO.select(nil, [@fd], [@fd], @timeout - Time.now.to_i).nil?
+          raise TimeoutError if IO.select(nil, [@fd], [@fd], @timeout - Time.now.to_i).nil?
+
           retry
         end
-        fail ProtocolError, "write failure (#{done}/#{data.length})" if written == 0
+        raise ProtocolError, "write failure (#{done}/#{data.length})" if written.zero?
+
         done += written
         break if done >= data.length
       end
-      return
+      nil
     end
 
     private
 
+    def handshake
+      return if @options[:disable_handshake]
+
+      signature, data = receive
+      if signature == 'JDAT'
+        @helo = Protocol.parse_helo_vers('')
+        @logger&.info 'Remote does not support protocol handshake', peer: @peer
+        yield signature, data, self
+        return
+      elsif signature != 'HELO'
+        raise ProtocolError, "unexpected #{signature} message"
+      end
+
+      @helo = Protocol.parse_helo_vers(data)
+      @logger&.info 'Remote identified', peer: @peer, client_version: @helo[:client_version]
+
+      # Flags 1 byte - EVNT flag = 0
+      # (Significant rewrite would be required to support streaming messages as currently we read
+      #  first and then yield for processing. To support EVNT we have to move protocol parsing to
+      #  the connection layer here so we can keep reading until we reach the end of the stream)
+      # Major Version 1 byte
+      # Minor Version 1 byte
+      # Patch Version 1 byte
+      # Client String 4 bytes
+      data = [1, 2, 7, 0, 'RYLC'].pack('CCCCA4')
+      send 'VERS', data
+    end
+
     def get_cn(cert)
       cert.subject.to_a.find do |oid, value|
-        return value if oid == "CN"
+        return value if oid == 'CN'
       end
       nil
     end
@@ -338,20 +392,20 @@ module LogCourier
       have = ''
       loop do
         begin
-       	  buffer = @fd.read_nonblock need - have.length
+          buffer = @fd.read_nonblock need - have.length
         rescue IO::WaitReadable
-          fail TimeoutError if IO.select([@fd], nil, [@fd], @timeout - Time.now.to_i).nil?
+          raise TimeoutError if IO.select([@fd], nil, [@fd], @timeout - Time.now.to_i).nil?
+
           retry
         rescue IO::WaitWritable
-          fail TimeoutError if IO.select(nil, [@fd], [@fd], @timeout - Time.now.to_i).nil?
+          raise TimeoutError if IO.select(nil, [@fd], [@fd], @timeout - Time.now.to_i).nil?
+
           retry
         end
-        if buffer.nil?
-          fail EOFError
-        elsif buffer.length == 0
-          fail ProtocolError, "read failure (#{have.length}/#{need})"
-        end
-        if have.length == 0
+        raise EOFError if buffer.nil?
+        raise ProtocolError, "read failure (#{have.length}/#{need})" if buffer.length.zero?
+
+        if have.length.zero?
           have = buffer
         else
           have << buffer
@@ -364,7 +418,7 @@ module LogCourier
     def reset_timeout
       # TODO: Make configurable
       @timeout = Time.now.to_i + 1_800
-      return
+      nil
     end
   end
 end

metadata

@@ -1,22 +1,22 @@
 --- !ruby/object:Gem::Specification
 name: log-courier
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 2.7.0
 platform: ruby
 authors:
 - Jason Woods
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-08-21 00:00:00.000000000 Z
+date: 2021-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: cabin
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+  name: cabin
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -25,26 +25,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.6'
 - !ruby/object:Gem::Dependency
-  name: ffi-rzmq
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: multi_json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.10'
+  name: multi_json
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -64,13 +50,11 @@ files:
 - lib/log-courier/event_queue.rb
 - lib/log-courier/server.rb
 - lib/log-courier/server_tcp.rb
-- lib/log-courier/server_zmq.rb
-- lib/log-courier/zmq_qpoll.rb
 homepage: https://github.com/driskell/ruby-log-courier
 licenses:
-- Apache
+- Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -85,9 +69,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: nowarning
-rubygems_version: 2.7.7
-signing_key: 
+rubygems_version: 3.0.6
+signing_key:
 specification_version: 4
 summary: Ruby implementation of the Courier protocol
 test_files: []