log-courier 1.10.0 → 2.7.3

data/lib/log-courier/rspec/spec_helper.rb

@@ -0,0 +1,217 @@
+# Copyright 2014-2021 Jason Woods.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'cabin'
+require 'log-courier/client'
+require 'log-courier/server'
+
+TEMP_PATH = File.join(File.dirname(__FILE__), 'tmp')
+EVENT_WAIT_COUNT = 50
+EVENT_WAIT_TIME = 0.5
+
+# Common helpers for testing both ruby client and the courier
+shared_context 'LogCourier' do
+  before :all do
+    Thread.abort_on_exception = true
+
+    FileUtils.rm_r(TEMP_PATH) if File.directory?(TEMP_PATH)
+    Dir.mkdir(TEMP_PATH)
+
+    @ssl_cert = File.open(File.join(TEMP_PATH, 'ssl_cert'), 'w')
+    @ssl_key = File.open(File.join(TEMP_PATH, 'ssl_key'), 'w')
+    @ssl_csr = File.open(File.join(TEMP_PATH, 'ssl_csr'), 'w')
+
+    # Generate the ssl key
+    cnf_path = "#{File.dirname(__FILE__)}/openssl.cnf"
+    system("openssl req -config #{cnf_path} -new -batch -keyout #{@ssl_key.path} -out #{@ssl_csr.path}")
+    system(
+      "openssl x509 -extfile #{cnf_path} -extensions extensions_section -req -days 365 -in #{@ssl_csr.path}" \
+      " -signkey #{@ssl_key.path} -out #{@ssl_cert.path}",
+    )
+  end
+
+  after :all do
+    FileUtils.rm_r(TEMP_PATH) if File.directory?(TEMP_PATH)
+  end
+
+  before :each do
+    @event_queue = SizedQueue.new 10_000
+
+    @clients = {}
+    @servers = {}
+    @server_counts = {}
+    @server_threads = {}
+  end
+
+  after :each do
+    unless @servers.length.zero?
+      id, = @servers.first
+      raise "Server was not shutdown: #{id}"
+    end
+    unless @clients.length.zero?
+      id, = @clients.first
+      raise "Client was not shutdown: #{id}"
+    end
+  end
+
+  def start_client(**args)
+    args = {
+      id: '__default__',
+      transport: 'tls',
+      addresses: ['127.0.0.1'],
+    }.merge!(**args)
+
+    args[:ssl_ca] = @ssl_cert.path if args[:transport] == 'tls'
+
+    id = args[:id]
+    args[:port] = server_port(id) unless args.key?(:port)
+
+    logger = Cabin::Channel.new
+    logger.subscribe $stdout
+    logger['instance'] = "Client #{id}"
+    logger.level = :debug
+
+    # Reset server for each test
+    @clients[id] = LogCourier::Client.new(
+      logger: logger,
+      **args,
+    )
+  end
+
+  def shutdown_client(which = nil)
+    which = if which.nil?
+              @clients.keys
+            else
+              [which]
+            end
+    which.each do |id|
+      @clients[id].shutdown
+      @clients.delete id
+    end
+    nil
+  end
+
+  def start_server(**args)
+    args = {
+      id: '__default__',
+      transport: 'tls',
+    }.merge!(**args)
+
+    if args[:transport] == 'tls'
+      args[:ssl_certificate] = @ssl_cert.path
+      args[:ssl_key] = @ssl_key.path
+    end
+
+    id = args[:id]
+
+    logger = Cabin::Channel.new
+    logger.subscribe $stdout
+    logger['instance'] = "Server #{id}"
+    logger.level = :debug
+
+    raise 'Server already initialised' if @servers.key?(id)
+
+    # Reset server for each test
+    @servers[id] = LogCourier::Server.new(
+      logger: logger,
+      **args,
+    )
+
+    @server_counts[id] = 0
+    @server_threads[id] = Thread.new do
+      @servers[id].run do |event|
+        @server_counts[id] += 1
+        @event_queue << event
+      end
+    rescue LogCourier::ShutdownSignal
+      0
+    end
+    @servers[id]
+  end
+
+  # A helper to shutdown a Log Courier server
+  def shutdown_server(which = nil)
+    which = if which.nil?
+              @servers.keys
+            else
+              [which]
+            end
+    which.each do |id|
+      @server_threads[id].raise LogCourier::ShutdownSignal
+      @server_threads[id].join
+      @server_threads.delete id
+      @server_counts.delete id
+      @servers.delete id
+    end
+    nil
+  end
+
+  # A helper to get the port a server is bound to
+  def server_port(id = '__default__')
+    @servers[id].port
+  end
+
+  # A helper to get number of events received on the server
+  def server_count(id = '__default__')
+    @server_counts[id]
+  end
+
+  def receive_and_check(args = {})
+    args = {
+      total: nil,
+      check: true,
+      check_file: true,
+      check_order: true,
+      host: nil,
+    }.merge!(args)
+
+    # Quick check of the total events we are expecting - but allow time to receive them
+    total = if args[:total].nil?
+              @files.reduce(0) do |sum, f|
+                sum + f.count
+              end
+            else
+              args[:total]
+            end
+
+    args.delete_if do |_, v|
+      v.nil?
+    end
+
+    orig_total = total
+    check = args[:check]
+
+    waited = 0
+    while total.positive? && waited <= EVENT_WAIT_COUNT
+      if @event_queue.length.zero?
+        sleep(EVENT_WAIT_TIME)
+        waited += 1
+        next
+      end
+
+      waited = 0
+      until @event_queue.length.zero?
+        e = @event_queue.pop
+        total -= 1
+        next unless check
+
+        yield e
+      end
+    end
+
+    # Fancy calculation to give a nice "expected" output of expected num of events
+    expect(orig_total - total).to eq orig_total
+    nil
+  end
+end

data/lib/log-courier/server.rb

@@ -1,6 +1,4 @@
-# encoding: utf-8
-
-# Copyright 2014-2019 Jason Woods and Contributors.
+# Copyright 2014-2021 Jason Woods and Contributors.
 #
 # This file is a modification of code from Logstash Forwarder.
 # Copyright 2012-2013 Jordan Sissel and contributors.
@@ -18,44 +16,27 @@
 # limitations under the License.
 
 require 'log-courier/event_queue'
+require 'log-courier/protocol'
+require 'log-courier/version'
 require 'multi_json'
-require 'thread'
 require 'zlib'
 
-class NativeException; end
-
 module LogCourier
   class TimeoutError < StandardError; end
+
   class ShutdownSignal < StandardError; end
+
   class ProtocolError < StandardError; end
 
   # Implementation of the server
   class Server
     attr_reader :port
 
-    # TODO(driskell): Consolidate singleton into another file
-    class << self
-      @json_adapter
-      @json_parseerror
-
-      def get_json_adapter
-        @json_adapter = MultiJson.adapter.instance if @json_adapter.nil?
-        @json_adapter
-      end
-
-      def get_json_parseerror
-        if @json_parseerror.nil?
-          @json_parseerror = get_json_adapter.class::ParseError
-        end
-        @json_parseerror
-      end
-    end
-
     def initialize(options = {})
       @options = {
-        logger:     nil,
-        transport:  'tls',
-        raw_events: true,
+        logger: nil,
+        transport: 'tls',
+        disable_handshake: false,
       }.merge!(options)
 
       @logger = @options[:logger]
@@ -64,11 +45,8 @@ module LogCourier
       when 'tcp', 'tls'
         require 'log-courier/server_tcp'
         @server = ServerTcp.new(@options)
-      when 'plainzmq', 'zmq'
-        require 'log-courier/server_zmq'
-        @server = ServerZmq.new(@options)
       else
-        fail 'input/courier: \'transport\' must be tcp, tls, plainzmq or zmq'
+        raise 'input/courier: \'transport\' must be tcp or tls'
       end
 
       # Grab the port back and update the logger context
@@ -91,9 +69,9 @@ module LogCourier
               process_jdat message, comm, @event_queue
             else
               if comm.peer.nil?
-                @logger.warn 'Unknown message received', :from => 'unknown' unless @logger.nil?
+                @logger&.warn 'Unknown message received', from: 'unknown'
               else
-                @logger.warn 'Unknown message received', :from => comm.peer unless @logger.nil?
+                @logger&.warn 'Unknown message received', from: comm.peer
               end
               # Don't kill a client that sends a bad message
               # Just reject it and let it send it again, potentially to another server
@@ -105,6 +83,7 @@ module LogCourier
         loop do
           event = @event_queue.pop
           break if event.nil?
+
           block.call event
         end
       ensure
@@ -114,25 +93,23 @@ module LogCourier
           server_thread.join
         end
       end
-      return
+      nil
     end
 
     def stop
       @event_queue << nil
+      nil
     end
 
     private
 
     def process_ping(message, comm)
       # Size of message should be 0
-      if message.bytesize != 0
-        fail ProtocolError, "unexpected data attached to ping message (#{message.bytesize})"
-      end
+      raise ProtocolError, "unexpected data attached to ping message (#{message.bytesize})" unless message.bytesize.zero?
 
       # PONG!
       # NOTE: comm.send can raise a Timeout::Error of its own
       comm.send 'PONG', ''
-      return
     end
 
     def process_jdat(message, comm, event_queue)
@@ -142,13 +119,11 @@ module LogCourier
       # OK - first is a nonce - we send this back with sequence acks
       # This allows the client to know what is being acknowledged
       # Nonce is 16 so check we have enough
-      if message.bytesize < 17
-        fail ProtocolError, "JDAT message too small (#{message.bytesize})"
-      end
+      raise ProtocolError, "JDAT message too small (#{message.bytesize})" if message.bytesize < 17
 
       nonce = message[0...16]
 
-      if !@logger.nil? && @logger.debug?
+      if @logger&.debug?
         nonce_str = nonce.each_byte.map do |b|
           b.to_s(16).rjust(2, '0')
         end
@@ -162,25 +137,21 @@ module LogCourier
       # We acknowledge them by their 1-index position in the stream
       # A 0 sequence acknowledgement means we haven't processed any yet
       sequence = 0
-      events = []
       length_buf = ''
       data_buf = ''
       loop do
         ret = message.read 4, length_buf
-        if ret.nil?
-          # Finished!
-          break
-        elsif length_buf.bytesize < 4
-          fail ProtocolError, "JDAT length extraction failed (#{ret} #{length_buf.bytesize})"
-        end
+        # Finished?
+        break if ret.nil?
+        raise ProtocolError, "JDAT length extraction failed (#{ret} #{length_buf.bytesize})" if length_buf.bytesize < 4
 
-        length = length_buf.unpack('N').first
+        length = length_buf.unpack1('N')
 
         # Extract message
         ret = message.read length, data_buf
-        if ret.nil? or data_buf.bytesize < length
-          @logger.warn()
-          fail ProtocolError, "JDAT message extraction failed #{ret} #{data_buf.bytesize}"
+        if ret.nil? || data_buf.bytesize < length
+          @logger&.warn()
+          raise ProtocolError, "JDAT message extraction failed #{ret} #{data_buf.bytesize}"
         end
 
         data_buf.force_encoding('utf-8')
@@ -200,9 +171,9 @@ module LogCourier
 
         # Decode the JSON
         begin
-          event = self.class.get_json_adapter.load(data_buf, :raw => @options[:raw_events])
-        rescue self.class.get_json_parseerror => e
-          @logger.warn e, :invalid_encodings => invalid_encodings, :hint => 'JSON parse failure, falling back to plain-text' unless @logger.nil?
+          event = MultiJson.load(data_buf)
+        rescue MultiJson::ParseError => e
+          @logger&.warn e, invalid_encodings: invalid_encodings, hint: 'JSON parse failure, falling back to plain-text'
           event = { 'message' => data_buf }
         end
 
@@ -215,7 +186,7 @@ module LogCourier
         rescue TimeoutError
           # Full pipeline, partial ack
           # NOTE: comm.send can raise a Timeout::Error of its own
-          @logger.debug 'Partially acknowledging message', :nonce => nonce_str.join, :sequence => sequence if !@logger.nil? && @logger.debug?
+          @logger&.debug 'Partially acknowledging message', nonce: nonce_str.join, sequence: sequence if @logger&.debug?
           comm.send 'ACKN', [nonce, sequence].pack('a*N')
           ack_timeout = Time.now.to_i + 5
           retry
@@ -226,9 +197,8 @@ module LogCourier
 
       # Acknowledge the full message
       # NOTE: comm.send can raise a Timeout::Error
-      @logger.debug 'Acknowledging message', :nonce => nonce_str.join, :sequence => sequence if !@logger.nil? && @logger.debug?
+      @logger&.debug 'Acknowledging message', nonce: nonce_str.join, sequence: sequence if @logger&.debug?
       comm.send 'ACKN', [nonce, sequence].pack('A*N')
-      return
     end
   end
 end