log-courier 1.0.21.ga82ca4c

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 480d6e6a5958b6f307727dfa1713e8a9c0a10d29
+  data.tar.gz: 2f2de7aeedd2e539a16412dd40f8321ecc9cff20
+SHA512:
+  metadata.gz: 3d2d449239c58e05a7654ad0f77148b2fd8ac5ebf0094f50b85c7311cb904650cc426a55aa2918a9b9334c70c283ca1d7aa47cb47c94b0999011bfc0949270b7
+  data.tar.gz: 8e51f6507f0b12a7ed56a17fb28213ea9a7f71b5257544c8191ad19bb70e91f1c6a8e940d37203fab587f3c564deb0551426fc8996081a7101faaf707d014748

data/lib/log-courier/client.rb

@@ -0,0 +1,352 @@
+# encoding: utf-8
+
+# Copyright 2014 Jason Woods.
+#
+# This file is a modification of code from Logstash Forwarder.
+# Copyright 2012-2013 Jordan Sissel and contributors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'log-courier/event_queue'
+require 'multi_json'
+require 'thread'
+require 'zlib'
+
+module LogCourier
+  # TODO: Make these shared
+  class ClientShutdownSignal < StandardError; end
+  class ClientProtocolError < StandardError; end
+
+  # Describes a pending payload
+  class PendingPayload
+    attr_accessor :ack_events
+    attr_accessor :events
+    attr_accessor :nonce
+    attr_accessor :data
+
+    attr_accessor :previous
+    attr_accessor :next
+
+    def initialize(options = {})
+      @ack_events = 0
+
+      options.each do |k, v|
+        raise ArgumentError unless self.respond_to?(k)
+        instance_variable_set "@#{k}", v
+      end
+    end
+  end
+
+  # Implementation of a single client connection
+  class Client
+    def initialize(options = {})
+      @options = {
+        logger:       nil,
+        spool_size:   1024,
+        idle_timeout: 5
+      }.merge!(options)
+
+      @logger = @options[:logger]
+
+      require 'log-courier/client_tls'
+      @client = ClientTls.new(@options)
+
+      # Load the json adapter
+      @json_adapter = MultiJson.adapter.instance
+
+      @event_queue = EventQueue.new @options[:spool_size]
+      @pending_payloads = {}
+      @first_payload = nil
+      @last_payload = nil
+
+      # Start the spooler which will collect events into chunks
+      @send_ready = false
+      @send_mutex = Mutex.new
+      @send_cond = ConditionVariable.new
+      @spooler_thread = Thread.new do
+        run_spooler
+      end
+
+      @pending_ping = false
+
+      # Start the IO thread
+      @io_control = EventQueue.new 1
+      @io_thread = Thread.new do
+        run_io
+      end
+    end
+
+    def publish(event)
+      # Pass the event into the spooler
+      @event_queue << event
+    end
+
+    def shutdown
+      # Raise a shutdown signal in the spooler and wait for it
+      @spooler_thread.raise ClientShutdownSignal
+      @io_thread.raise ClientShutdownSignal
+      @spooler_thread.join
+      @io_thread.join
+    end
+
+    def run_spooler
+      loop do
+        spooled = []
+        next_flush = Time.now.to_i + @options[:idle_timeout]
+
+        # The spooler loop
+        begin
+          loop do
+            event = @event_queue.pop next_flush - Time.now.to_i
+            spooled.push(event)
+            break if spooled.length >= @options[:spool_size]
+          end
+        rescue TimeoutError
+          # Hit timeout but no events, keep waiting
+          next if spooled.length == 0
+        end
+
+        # Pass through to io_control but only if we're ready to send
+        @send_mutex.synchronize do
+          @send_cond.wait(@send_mutex) unless @send_ready
+          @send_ready = false
+          @io_control << ['E', spooled]
+        end
+      end
+    rescue ClientShutdownSignal
+      # Just shutdown
+      0
+    end
+
+    def run_io
+      # TODO: Make keepalive configurable?
+      @keepalive_timeout = 1800
+
+      # TODO: Make pending payload max configurable?
+      max_pending_payloads = 100
+
+      retry_payload = nil
+
+      can_send = true
+
+      loop do
+        # Reconnect loop
+        @client.connect @io_control
+
+        reset_keepalive
+
+        # Capture send exceptions
+        begin
+          # IO loop
+          loop do
+            catch :keepalive do
+              begin
+                action = @io_control.pop @keepalive_next - Time.now.to_i
+
+                # Process the action
+                case action[0]
+                when 'S'
+                  # If we're flushing through the pending, pick from there
+                  unless retry_payload.nil?
+                    # Regenerate data if we need to
+                    retry_payload.data = buffer_jdat_data(retry_payload.events, retry_payload.nonce) if retry_payload.data == nil
+
+                    # Send and move onto next
+                    @client.send 'JDAT', retry_payload.data
+
+                    retry_payload = retry_payload.next
+                    throw :keepalive
+                  end
+
+                  # Ready to send, allow spooler to pass us something
+                  @send_mutex.synchronize do
+                    @send_ready = true
+                    @send_cond.signal
+                  end
+
+                  can_send = true
+                when 'E'
+                  # If we have too many pending payloads, pause the IO
+                  if @pending_payloads.length + 1 >= max_pending_payloads
+                    @client.pause_send
+                  end
+
+                  # Received some events - send them
+                  send_jdat action[1]
+
+                  # The send action will trigger another "S" if we have more send buffer
+                  can_send = false
+                when 'R'
+                  # Received a message
+                  signature, message = action[1..2]
+                  case signature
+                  when 'PONG'
+                    process_pong message
+                  when 'ACKN'
+                    process_ackn message
+                  else
+                    # Unknown message - only listener is allowed to respond with a "????" message
+                    # TODO: What should we do? Just ignore for now and let timeouts conquer
+                  end
+                when 'F'
+                  # Reconnect, an error occurred
+                  break
+                end
+              rescue TimeoutError
+                # Keepalive timeout hit, send a PING unless we were awaiting a PONG
+                if @pending_ping
+                  # Timed out, break into reconnect
+                  raise TimeoutError
+                end
+
+                # Is send full? can_send will be false if so
+                # We should've started receiving ACK by now so time out
+                raise TimeoutError unless can_send
+
+                # Send PING
+                send_ping
+
+                # We may have filled send buffer
+                can_send = false
+              end
+            end
+
+            # Reset keepalive timeout
+            reset_keepalive
+          end
+        rescue ClientProtocolError => e
+          # Reconnect required due to a protocol error
+          @logger.warn("[LogCourierClient] Protocol error: #{e}") unless @logger.nil?
+        rescue TimeoutError
+          # Reconnect due to timeout
+          @logger.warn('[LogCourierClient] Timeout occurred') unless @logger.nil?
+        rescue ClientShutdownSignal
+          # Shutdown, break out
+          break
+        rescue => e
+          # Unknown error occurred
+          @logger.warn("[LogCourierClient] Unknown error: #{e}") unless @logger.nil?
+          @logger.warn("[LogCourierClient] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil?
+        end
+
+        # Disconnect and retry payloads
+        @client.disconnect
+        retry_payload = @first_payload
+
+        # TODO: Make reconnect time configurable?
+        sleep 5
+      end
+
+      @client.disconnect
+    end
+
+    def reset_keepalive
+      @keepalive_next = Time.now.to_i + @keepalive_timeout
+    end
+
+    def generate_nonce
+      (0...16).map { rand(256).chr }.join("")
+    end
+
+    def send_ping
+      # Send it
+      @client.send 'PING', ''
+    end
+
+    def send_jdat(events)
+      # Generate the JSON payload and compress it
+      nonce = generate_nonce
+      data = buffer_jdat_data(events, nonce)
+
+      # Save the pending payload
+      payload = PendingPayload.new(
+        :events => events,
+        :nonce  => nonce,
+        :data   => data
+      )
+
+      @pending_payloads[nonce] = payload
+
+      if @first_payload.nil?
+        @first_payload = payload
+        @last_payload = payload
+      else
+        @last_payload.next = payload
+        @last_payload = payload
+      end
+
+      # Send it
+      @client.send 'JDAT', payload.data
+    end
+
+    def buffer_jdat_data(events, nonce)
+      buffer = Zlib::Deflate.new
+
+      # Write each event in JSON format
+      events.each do |event|
+        buffer_jdat_data_event(buffer, event)
+      end
+
+      # Generate and return the message
+      nonce + buffer.flush(Zlib::FINISH)
+    end
+
+    def buffer_jdat_data_event(buffer, event)
+      json_data = @json_adapter.dump(event)
+
+      # Add length and then the data
+      buffer << [json_data.length].pack('N') << json_data
+    end
+
+    def process_pong(message)
+      # Sanity
+      if message.length != 0
+        raise ClientProtocolError, "Unexpected data attached to pong message (#{message.length})"
+      end
+
+      # No longer pending a PONG
+      @ping_pending = false
+    end
+
+    def process_ackn(message)
+      # Sanity
+      if message.length != 20
+        raise ClientProtocolError, "ACKN message size invalid (#{message.length})"
+      end
+
+      # Grab nonce
+      sequence, nonce = message[0...4].unpack('N').first, message[4..-1]
+
+      # Find the payload - skip if we couldn't as it will just a duplicated ACK
+      return unless @pending_payloads.key?(nonce)
+
+      payload = @pending_payloads[nonce]
+
+      # Full ACK?
+      # TODO: protocol error if sequence too large?
+      if sequence >= payload.events.length
+        @client.resume_send if @client.send_paused?
+
+        @pending_payloads.delete nonce
+        payload.previous.next = payload.next
+      else
+        # Partial ACK - only process if something was actually processed
+        if sequence > payload.ack_events
+          payload.ack_events = sequence
+          payload.events = payload.events[0...sequence]
+          payload.data = nil
+        end
+      end
+    end
+  end
+end

data/lib/log-courier/client_tls.rb

@@ -0,0 +1,217 @@
+# encoding: utf-8
+
+# Copyright 2014 Jason Woods.
+#
+# This file is a modification of code from Logstash Forwarder.
+# Copyright 2012-2013 Jordan Sissel and contributors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'openssl'
+require 'socket'
+require 'thread'
+
+module LogCourier
+  # TLS transport implementation
+  class ClientTls
+    def initialize(options = {})
+      @options = {
+        logger:             nil,
+        port:               nil,
+        addresses:          [],
+        ssl_ca:             nil,
+        ssl_certificate:    nil,
+        ssl_key:            nil,
+        ssl_key_passphrase: nil
+      }.merge!(options)
+
+      @logger = @options[:logger]
+
+      [:port, :ssl_ca].each do |k|
+        raise "[LogCourierClient] '#{k}' is required" if @options[k].nil?
+      end
+
+      raise '[LogCourierClient] \'addresses\' must contain at least one address' if @options[:addresses].empty?
+
+      c = 0
+      [:ssl_certificate, :ssl_key].each do
+        c += 1
+      end
+
+      raise '[LogCourierClient] \'ssl_certificate\' and \'ssl_key\' must be specified together' if c == 1
+    end
+
+    def connect(io_control)
+      begin
+        tls_connect
+      rescue ClientShutdownSignal
+        raise
+      rescue
+        # TODO: Make this configurable
+        sleep 5
+        retry
+      end
+
+      @send_q = SizedQueue.new 1
+      @send_paused = false
+
+      @send_thread = Thread.new do
+        run_send io_control
+      end
+      @recv_thread = Thread.new do
+        run_recv io_control
+      end
+    end
+
+    def disconnect
+      @send_thread.raise ClientShutdownSignal
+      @send_thread.join
+      @recv_thread.raise ClientShutdownSignal
+      @recv_thread.join
+    end
+
+    def run_send(io_control)
+      # Ask for something to send
+      io_control << ['S']
+
+      # If paused, we still accept message to send, but we don't release "S" to ask for more
+      # As soon as we resume we then release "S" to ask for more
+      paused = false
+
+      loop do
+        # Wait for data and send when we get it
+        message = @send_q.pop
+
+        # A nil is a pause/resume
+        if message.nil?
+          if paused
+            paused = false
+            io_control << ['S']
+          else
+            paused = true
+            next
+          end
+        else
+          # Ask for more to send while we send this one
+          io_control << ['S'] unless paused
+
+          @ssl_client.write message
+        end
+      end
+    rescue OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET => e
+      @logger.warn("[LogCourierClient] SSL write error: #{e}") unless @logger.nil?
+      io_control << ['F']
+    rescue ClientShutdownSignal
+      # Just shutdown
+    rescue => e
+      @logger.warn("[LogCourierClient] Unknown SSL write error: #{e}") unless @logger.nil?
+      @logger.warn("[LogCourierClient] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil?
+      io_control << ['F']
+    end
+
+    def run_recv(io_control)
+      loop do
+        # Grab a header
+        header = @ssl_client.read(8)
+        raise EOFError if header.nil?
+
+        # Decode signature and length
+        signature, length = header.unpack('A4N')
+
+        if length > 1048576
+          # Too big raise error
+          @logger.warn("[LogCourierClient] Invalid message: data too big (#{length})") unless @logger.nil?
+          io_control << ['F']
+          break
+        end
+
+        # Read remainder
+        message = @ssl_client.read(length)
+
+        # Pass through to receive
+        io_control << ['R', signature, message]
+      end
+    rescue OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET => e
+      @logger.warn("[LogCourierClient] SSL read error: #{e}") unless @logger.nil?
+      io_control << ['F']
+    rescue EOFError
+      @logger.warn("[LogCourierClient] Connection closed by server") unless @logger.nil?
+      io_control << ['F']
+    rescue ClientShutdownSignal
+      # Just shutdown
+    rescue => e
+      @logger.warn("[LogCourierClient] Unknown SSL read error: #{e}") unless @logger.nil?
+      @logger.warn("[LogCourierClient] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil?
+      io_control << ['F']
+    end
+
+    def send(signature, message)
+      # Add to send queue
+      @send_q << [signature, message.length].pack('A4N') + message
+    end
+
+    def pause_send
+      return if @send_paused
+      @send_paused = true
+      @send_q << nil
+    end
+
+    def send_paused
+      @send_paused
+    end
+
+    def resume_send
+      if @send_paused
+        @send_paused = false
+        @send_q << nil
+      end
+    end
+
+    def tls_connect
+      # TODO: Implement random selection - and don't use separate :port - remember to update post_connection_check too
+      @logger.info("[LogCourierClient] Connecting to #{@options[:addresses][0]}:#{@options[:port]}") unless @logger.nil?
+      tcp_socket = TCPSocket.new(@options[:addresses][0], @options[:port])
+
+      ssl = OpenSSL::SSL::SSLContext.new
+
+      unless @options[:ssl_certificate].nil?
+        ssl.cert = OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate]))
+        ssl.key = OpenSSL::PKey::RSA.new(File.read(@options[:ssl_key]), @options[:ssl_key_passphrase])
+      end
+
+      cert_store = OpenSSL::X509::Store.new
+      cert_store.add_file(@options[:ssl_ca])
+      #ssl.cert_store = cert_store
+      ssl.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
+
+      @ssl_client = OpenSSL::SSL::SSLSocket.new(tcp_socket)
+
+      socket = @ssl_client.connect
+
+      # Verify certificate
+      socket.post_connection_check(@options[:addresses][0])
+
+      @logger.info("[LogCourierClient] Connected successfully") unless @logger.nil?
+
+      socket
+    rescue OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET => e
+      @logger.warn("[LogCourierClient] Connection to #{@options[:addresses][0]}:#{@options[:port]} failed: #{e}") unless @logger.nil?
+    rescue ClientShutdownSignal
+      # Just shutdown
+      0
+    rescue => e
+      @logger.warn("[LogCourierClient] Unknown connection failure to #{@options[:addresses][0]}:#{@options[:port]}: #{e}") unless @logger.nil?
+      @logger.warn("[LogCourierClient] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil?
+    end
+  end
+end