log-courier 1.0.21.ga82ca4c

data/lib/log-courier/event_queue.rb

@@ -0,0 +1,194 @@
+# encoding: utf-8
+
+# Copyright 2014 Jason Woods.
+#
+# This file is a modification of code from Ruby.
+# Ruby is copyrighted free software by Yukihiro Matsumoto <matz@netlab.jp>.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This is a queue implementation dervied from SizedQueue, but with a timeout.
+#
+# It is significantly faster than using SizedQueue wrapped in Timeout::timeout
+# because it uses mutex.sleep, whereas Timeout::timeout actually starts another
+# thread that waits and then raises exception or has to be stopped on exiting
+# the block.
+#
+# The majority of the code is taken from Ruby's SizedQueue<Queue implementation.
+#
+module LogCourier
+  class EventQueue
+    #
+    # Creates a fixed-length queue with a maximum size of +max+.
+    #
+    def initialize(max)
+      raise ArgumentError, "queue size must be positive" unless max > 0
+      @max = max
+      @enque_cond = ConditionVariable.new
+      @num_enqueue_waiting = 0
+
+      @que = []
+      @que.taint          # enable tainted communication
+      @num_waiting = 0
+      self.taint
+      @mutex = Mutex.new
+      @cond = ConditionVariable.new
+    end
+
+    #
+    # Returns the maximum size of the queue.
+    #
+    def max
+      @max
+    end
+
+    #
+    # Sets the maximum size of the queue.
+    #
+    def max=(max)
+      raise ArgumentError, "queue size must be positive" unless max > 0
+
+      @mutex.synchronize do
+        if max <= @max
+          @max = max
+        else
+          diff = max - @max
+          @max = max
+          diff.times do
+            @enque_cond.signal
+          end
+        end
+      end
+      max
+    end
+
+    #
+    # Pushes +obj+ to the queue.  If there is no space left in the queue, waits
+    # until space becomes available, up to a maximum of +timeout+ seconds.
+    #
+    def push(obj, timeout = nil)
+      unless timeout.nil?
+        start = Time.now
+      end
+      @mutex.synchronize do
+        loop do
+          break if @que.length < @max
+          @num_enqueue_waiting += 1
+          begin
+            @enque_cond.wait @mutex, timeout
+          ensure
+            @num_enqueue_waiting -= 1
+          end
+          raise TimeoutError if !timeout.nil? and Time.now - start >= timeout
+        end
+
+        @que.push obj
+        @cond.signal
+      end
+      self
+    end
+
+    #
+    # Alias of push
+    #
+    alias << push
+
+    #
+    # Alias of push
+    #
+    alias enq push
+
+    #
+    # Retrieves data from the queue and runs a waiting thread, if any.
+    #
+    def pop(*args)
+      retval = _pop_timeout *args
+      @mutex.synchronize do
+        if @que.length < @max
+          @enque_cond.signal
+        end
+      end
+      retval
+    end
+
+    #
+    # Retrieves data from the queue.  If the queue is empty, the calling thread is
+    # suspended until data is pushed onto the queue or, if set, +timeout+ seconds
+    # passes.  If +timeout+ is 0, the thread isn't suspended, and an exception is
+    # raised.
+    #
+    def _pop_timeout(timeout = nil)
+      unless timeout.nil?
+        start = Time.now
+      end
+      @mutex.synchronize do
+        loop do
+          return @que.shift unless @que.empty?
+          raise TimeoutError if timeout == 0
+          begin
+            @num_waiting += 1
+            @cond.wait @mutex, timeout
+          ensure
+            @num_waiting -= 1
+          end
+          raise TimeoutError if !timeout.nil? and Time.now - start >= timeout
+        end
+      end
+    end
+
+    #
+    # Alias of pop
+    #
+    alias shift pop
+
+    #
+    # Alias of pop
+    #
+    alias deq pop
+
+    #
+    # Returns +true+ if the queue is empty.
+    #
+    def empty?
+      @que.empty?
+    end
+
+    #
+    # Removes all objects from the queue.
+    #
+    def clear
+      @que.clear
+      self
+    end
+
+    #
+    # Returns the length of the queue.
+    #
+    def length
+      @que.length
+    end
+
+    #
+    # Alias of length.
+    #
+    alias size length
+
+    #
+    # Returns the number of threads waiting on the queue.
+    #
+    def num_waiting
+      @num_waiting + @num_enqueue_waiting
+    end
+  end
+end

data/lib/log-courier/server.rb

@@ -0,0 +1,185 @@
+# encoding: utf-8
+
+# Copyright 2014 Jason Woods.
+#
+# This file is a modification of code from Logstash Forwarder.
+# Copyright 2012-2013 Jordan Sissel and contributors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'log-courier/event_queue'
+require 'multi_json'
+require 'thread'
+require 'zlib'
+
+module LogCourier
+  class TimeoutError < StandardError; end
+  class ShutdownSignal < StandardError; end
+  class ProtocolError < StandardError; end
+
+  # Implementation of the server
+  class Server
+    attr_reader :port
+
+    def initialize(options = {})
+      @options = {
+        logger:    nil,
+        transport: 'tls'
+      }.merge!(options)
+
+      @logger = @options[:logger]
+
+      case @options[:transport]
+      when 'tcp', 'tls'
+        require 'log-courier/server_tcp'
+        @server = ServerTcp.new(@options)
+      when 'plainzmq', 'zmq'
+        require 'log-courier/server_zmq'
+        @server = ServerZmq.new(@options)
+      else
+        raise '[LogCourierServer] \'transport\' must be tcp, tls, plainzmq or zmq'
+      end
+
+      # Grab the port back
+      @port = @server.port
+
+      # Load the json adapter
+      @json_adapter = MultiJson.adapter.instance
+      @json_options = { raw: true, use_bigdecimal: true }
+    end
+
+    def run(&block)
+      # TODO: Make queue size configurable
+      event_queue = EventQueue.new 1
+      server_thread = nil
+
+      begin
+        server_thread = Thread.new do
+          # Receive messages and process them
+          @server.run do |signature, message, comm|
+            case signature
+            when 'PING'
+              process_ping message, comm
+            when 'JDAT'
+              process_jdat message, comm, event_queue
+            else
+              @logger.warn("[LogCourierServer] Unknown message received from #{comm.peer}") unless @logger.nil?
+              # Don't kill a client that sends a bad message
+              # Just reject it and let it send it again, potentially to another server
+              comm.send '????', ''
+            end
+          end
+        end
+
+        loop do
+          block.call event_queue.pop
+        end
+      ensure
+        # Signal the server thread to stop
+        unless server_thread.nil?
+          server_thread.raise ShutdownSignal
+          server_thread.join
+        end
+      end
+    end
+
+    def process_ping(message, comm)
+      # Size of message should be 0
+      if message.length != 0
+        raise ProtocolError, "unexpected data attached to ping message (#{message.length})"
+      end
+
+      # PONG!
+      # NOTE: comm.send can raise a Timeout::Error of its own
+      comm.send 'PONG', ''
+    end
+
+    def process_jdat(message, comm, event_queue)
+      # Now we have the data, aim to respond within 5 seconds
+      ack_timeout = Time.now.to_i + 5
+
+      # OK - first is a nonce - we send this back with sequence acks
+      # This allows the client to know what is being acknowledged
+      # Nonce is 16 so check we have enough
+      if message.length < 17
+        raise ProtocolError, "JDAT message too small (#{message.length})"
+      end
+
+      nonce = message[0...16]
+
+      # The remainder of the message is the compressed data block
+      message = StringIO.new Zlib::Inflate.inflate(message[16...message.length])
+
+      # Message now contains JSON encoded events
+      # They are aligned as [length][event]... so on
+      # We acknowledge them by their 1-index position in the stream
+      # A 0 sequence acknowledgement means we haven't processed any yet
+      sequence = 0
+      events = []
+      length_buf = ''
+      data_buf = ''
+      loop do
+        ret = message.read 4, length_buf
+        if ret.nil?
+          # Finished!
+          break
+        elsif length_buf.length < 4
+          raise ProtocolError, "JDAT length extraction failed (#{ret} #{length_buf.length})"
+        end
+
+        length = length_buf.unpack('N').first
+
+        # Extract message
+        ret = message.read length, data_buf
+        if ret.nil? or data_buf.length < length
+          @logger.warn()
+          raise ProtocolError, "JDAT message extraction failed #{ret} #{data_buf.length}"
+        end
+
+        data_buf.force_encoding('utf-8')
+
+        # Ensure valid encoding
+        unless data_buf.valid_encoding?
+          data_buf.chars.map do |c|
+            c.valid_encoding? ? c : "\xEF\xBF\xBD"
+          end
+        end
+
+        # Decode the JSON
+        begin
+          event = @json_adapter.load(data_buf, @json_options)
+        rescue MultiJson::ParserError => e
+          @logger.warn("[LogCourierServer] JSON parse failure, falling back to plain-text: #{e}") unless @logger.nil?
+          event = { 'message' => data_buf }
+        end
+
+        # Queue the event
+        begin
+          event_queue.push event, [0, ack_timeout - Time.now.to_i].max
+        rescue TimeoutError
+          # Full pipeline, partial ack
+          # NOTE: comm.send can raise a Timeout::Error of its own
+          comm.send 'ACKN', [nonce, sequence].pack('A*N')
+          ack_timeout = Time.now.to_i + 5
+          retry
+        end
+
+        sequence += 1
+      end
+
+      # Acknowledge the full message
+      # NOTE: comm.send can raise a Timeout::Error
+      comm.send 'ACKN', [nonce, sequence].pack('A*N')
+    end
+  end
+end

data/lib/log-courier/server_tcp.rb

@@ -0,0 +1,275 @@
+# encoding: utf-8
+
+# Copyright 2014 Jason Woods.
+#
+# This file is a modification of code from Logstash Forwarder.
+# Copyright 2012-2013 Jordan Sissel and contributors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'openssl'
+require 'socket'
+require 'thread'
+
+module LogCourier
+  # Wrap around TCPServer to grab last error for use in reporting which peer had an error
+  class ExtendedTCPServer < TCPServer
+    # Yield the peer
+    def accept
+      sock = super
+      peer = sock.peeraddr(:numeric)
+      Thread.current['LogCourierPeer'] = "#{peer[2]}:#{peer[1]}"
+      return sock
+    end
+  end
+
+  # TLS transport implementation for server
+  class ServerTcp
+    attr_reader :port
+
+    # Create a new TLS transport endpoint
+    def initialize(options = {})
+      @options = {
+        logger:                nil,
+        transport:             'tls',
+        port:                  0,
+        address:               '0.0.0.0',
+        ssl_certificate:       nil,
+        ssl_key:               nil,
+        ssl_key_passphrase:    nil,
+        ssl_verify:            false,
+        ssl_verify_default_ca: false,
+        ssl_verify_ca:         nil,
+        max_packet_size:       10_485_760,
+      }.merge!(options)
+
+      @logger = @options[:logger]
+
+      if @options[:transport] == 'tls'
+        [:ssl_certificate, :ssl_key].each do |k|
+          raise "[LogCourierServer] '#{k}' is required" if @options[k].nil?
+        end
+
+        if @options[:ssl_verify] and (!@options[:ssl_verify_default_ca] && @options[:ssl_verify_ca].nil?)
+          raise '[LogCourierServer] Either \'ssl_verify_default_ca\' or \'ssl_verify_ca\' must be specified when ssl_verify is true'
+        end
+      end
+
+      begin
+        @tcp_server = ExtendedTCPServer.new(@options[:address], @options[:port])
+
+        # Query the port in case the port number is '0'
+        # TCPServer#addr == [ address_family, port, address, address ]
+        @port = @tcp_server.addr[1]
+
+        if @options[:transport] == 'tls'
+          ssl = OpenSSL::SSL::SSLContext.new
+          ssl.cert = OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate]))
+          ssl.key = OpenSSL::PKey::RSA.new(File.read(@options[:ssl_key]), @options[:ssl_key_passphrase])
+
+          if @options[:ssl_verify]
+            cert_store = OpenSSL::X509::Store.new
+
+            # Load the system default certificate path to the store
+            cert_store.set_default_paths if @options[:ssl_verify_default_ca]
+
+            if File.directory?(@options[:ssl_verify_ca])
+              cert_store.add_path(@options[:ssl_verify_ca])
+            else
+              cert_store.add_file(@options[:ssl_verify_ca])
+            end
+
+            ssl.cert_store = cert_store
+
+            ssl.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
+          end
+
+          @server = OpenSSL::SSL::SSLServer.new(@tcp_server, ssl)
+        else
+          @server = @tcp_server
+        end
+
+        if @options[:port] == 0
+          @logger.warn '[LogCourierServer] Transport ' + @options[:transport] + ' is listening on ephemeral port ' + @port.to_s
+        end
+      rescue => e
+        raise "[LogCourierServer] Failed to initialise: #{e}"
+      end
+    end # def initialize
+
+    def run(&block)
+      client_threads = {}
+
+      loop do
+        # This means ssl accepting is single-threaded.
+        begin
+          client = @server.accept
+        rescue EOFError, OpenSSL::SSL::SSLError, IOError => e
+          # Handshake failure or other issue
+          peer = Thread.current['LogCourierPeer'] || 'unknown'
+          @logger.warn "[LogCourierServer] Connection from #{peer} failed to initialise: #{e}" unless @logger.nil?
+          client.close rescue nil
+          next
+        end
+
+        peer = Thread.current['LogCourierPeer'] || 'unknown'
+
+    	  @logger.info "[LogCourierServer] New connection from #{peer}" unless @logger.nil?
+
+        # Clear up finished threads
+        client_threads.delete_if do |_, thr|
+          !thr.alive?
+        end
+
+        # Start a new connection thread
+        client_threads[client] = Thread.new(client, peer) do |client_copy, peer_copy|
+          ConnectionTcp.new(@logger, client_copy, peer_copy, @options).run(&block)
+        end
+      end
+    rescue ShutdownSignal
+      # Capture shutting down signal
+      0
+    ensure
+      # Raise shutdown in all client threads and join then
+      client_threads.each do |_, thr|
+        thr.raise ShutdownSignal
+      end
+
+      client_threads.each(&:join)
+
+      @tcp_server.close
+    end
+  end
+
+  # Representation of a single connected client
+  class ConnectionTcp
+    attr_accessor :peer
+
+    def initialize(logger, fd, peer, options)
+      @logger = logger
+      @fd = fd
+      @peer = peer
+      @in_progress = false
+      @options = options
+    end
+
+    def run
+      loop do
+        # Read messages
+        # Each message begins with a header
+        # 4 byte signature
+        # 4 byte length
+        # Normally we would not parse this inside transport, but for TLS we have to in order to locate frame boundaries
+        signature, length = recv(8).unpack('A4N')
+
+        # Sanity
+        if length > @options[:max_packet_size]
+          raise ProtocolError, "packet too large (#{length} > #{@options[:max_packet_size]})"
+        end
+
+        # While we're processing, EOF is bad as it may occur during send
+        @in_progress = true
+
+        # Read the message
+        if length == 0
+          data = ''
+        else
+          data = recv(length)
+        end
+
+        # Send for processing
+        yield signature, data, self
+
+        # If we EOF next it's a graceful close
+        @in_progress = false
+      end
+    rescue TimeoutError
+      # Timeout of the connection, we were idle too long without a ping/pong
+      @logger.warn("[LogCourierServer] Connection from #{@peer} timed out") unless @logger.nil?
+    rescue EOFError
+      if @in_progress
+        @logger.warn("[LogCourierServer] Premature connection close on connection from #{@peer}") unless @logger.nil?
+      else
+        @logger.info("[LogCourierServer] Connection from #{@peer} closed") unless @logger.nil?
+      end
+    rescue OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET => e
+      # Read errors, only action is to shutdown which we'll do in ensure
+      @logger.warn("[LogCourierServer] SSL error on connection from #{@peer}: #{e}") unless @logger.nil?
+    rescue ProtocolError => e
+      # Connection abort request due to a protocol error
+      @logger.warn("[LogCourierServer] Protocol error on connection from #{@peer}: #{e}") unless @logger.nil?
+    rescue ShutdownSignal
+      # Shutting down
+      @logger.warn("[LogCourierServer] Closing connecting from #{@peer}: server shutting down") unless @logger.nil?
+    rescue => e
+      # Some other unknown problem
+      @logger.warn("[LogCourierServer] Unknown error on connection from #{@peer}: #{e}") unless @logger.nil?
+      @logger.warn("[LogCourierServer] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil?
+    ensure
+      @fd.close rescue nil
+    end
+
+    def recv(need)
+      reset_timeout
+      have = ''
+      loop do
+        begin
+       	  buffer = @fd.read_nonblock need - have.length
+        rescue IO::WaitReadable
+          raise TimeoutError if IO.select([@fd], nil, [@fd], @timeout - Time.now.to_i).nil?
+          retry
+        rescue IO::WaitWritable
+          raise TimeoutError if IO.select(nil, [@fd], [@fd], @timeout - Time.now.to_i).nil?
+          retry
+        end
+        if buffer.nil?
+          raise EOFError
+        elsif buffer.length == 0
+          raise ProtocolError, "read failure (#{have.length}/#{need})"
+        end
+        if have.length == 0
+          have = buffer
+        else
+          have << buffer
+        end
+        break if have.length >= need
+      end
+      have
+    end
+
+    def send(signature, message)
+      reset_timeout
+      data = signature + [message.length].pack('N') + message
+      done = 0
+      loop do
+        begin
+          written = @fd.write_nonblock(data[done...data.length])
+        rescue IO::WaitReadable
+          raise TimeoutError if IO.select([@fd], nil, [@fd], @timeout - Time.now.to_i).nil?
+          retry
+        rescue IO::WaitWritable
+          raise TimeoutError if IO.select(nil, [@fd], [@fd], @timeout - Time.now.to_i).nil?
+          retry
+        end
+        raise ProtocolError, "write failure (#{done}/#{data.length})" if written == 0
+        done += written
+        break if done >= data.length
+      end
+    end
+
+    def reset_timeout
+      # TODO: Make configurable
+      @timeout = Time.now.to_i + 1_800
+    end
+  end
+end