jls-lumberjack-logzio 0.0.26

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a1e528d32753cc4cf001f41c1b2a8333d6404c73fb6d46fd9958dbd530cb233d
+  data.tar.gz: d9bf8d7e486345f5ea41f70bb9f4f4f202d6b5026f7888f5c556dbe82eda560a
+SHA512:
+  metadata.gz: acb9f7b86989d76046fe8d886c972975eaa9a8510e27edc069ef8bdaafd8ed311927c1853fa58493edd5c486a7c55a6a6ffd977399c2943f04ee803711de9eb4
+  data.tar.gz: 26e5c8b67f70640144cf1c4859d9eb4d661eaa9bf684bee4df7e8ae688a3af42a1b033368693e3de4a9cc1fc30d299281306b81f189215a84c7161dc8eb6cec3

data/lib/lumberjack/client.rb

@@ -0,0 +1,217 @@
+# encoding: utf-8
+require "lumberjack"
+require "socket"
+require "thread"
+require "openssl"
+require "zlib"
+
+module Lumberjack
+  class Client
+    def initialize(opts={})
+      @opts = {
+        :port => 0,
+        :addresses => [],
+        :ssl_certificate => nil,
+        :ssl => true,
+        :json => false,
+      }.merge(opts)
+
+      @opts[:addresses] = [@opts[:addresses]] if @opts[:addresses].class == String
+      raise "Must set a port." if @opts[:port] == 0
+      raise "Must set atleast one address" if @opts[:addresses].empty? == 0
+      raise "Must set a ssl certificate or path" if @opts[:ssl_certificate].nil? && @opts[:ssl]
+
+      @socket = connect
+    end
+
+    private
+    def connect
+      addrs = @opts[:addresses].shuffle
+      begin
+        raise "Could not connect to any hosts" if addrs.empty?
+        opts = @opts
+        opts[:address] = addrs.pop
+        Lumberjack::Socket.new(opts)
+      rescue *[Errno::ECONNREFUSED,SocketError]
+        retry
+      end
+    end
+
+    public
+    def write(elements, opts={})
+      @socket.write_sync(elements, opts)
+    end
+
+    public
+    def host
+      @socket.host
+    end
+  end
+
+  class Socket
+    # Create a new Lumberjack Socket.
+    #
+    # - options is a hash. Valid options are:
+    #
+    # * :port - the port to listen on
+    # * :address - the host/address to bind to
+    # * :ssl - enable/disable ssl support
+    # * :ssl_certificate - the path to the ssl cert to use.
+    #                      If ssl_certificate is not set, a plain tcp connection
+    #                      will be used.
+    attr_reader :sequence
+    attr_reader :host
+    def initialize(opts={})
+      @sequence = 0
+      @last_ack = 0
+      @opts = {
+        :port => 0,
+        :address => "127.0.0.1",
+        :ssl_certificate => nil,
+        :ssl => true,
+        :json => false,
+      }.merge(opts)
+      @host = @opts[:address]
+
+      connection_start(opts)
+    end
+
+    private
+    def connection_start(opts)
+      tcp_socket = TCPSocket.new(opts[:address], opts[:port])
+      if !opts[:ssl]
+        @socket = tcp_socket
+      else
+        certificate_store = OpenSSL::X509::Store.new
+        certificate_store.add_file(opts[:ssl_certificate])
+
+        ssl_context = OpenSSL::SSL::SSLContext.new
+        ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        ssl_context.cert_store = certificate_store
+
+        @socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, ssl_context)
+        @socket.connect
+      end
+    end
+
+    private 
+    def inc
+      @sequence = 0 if @sequence + 1 > Lumberjack::SEQUENCE_MAX
+      @sequence = @sequence + 1
+    end
+
+    private
+    def send_window_size(size)
+      @socket.syswrite(["1", "W", size].pack("AAN"))
+    end
+
+    private
+    def send_payload(payload)
+      # SSLSocket has a limit of 16k per message
+      # execute multiple writes if needed
+      bytes_written = 0
+      while bytes_written < payload.bytesize
+        bytes_written += @socket.syswrite(payload.byteslice(bytes_written..-1))
+      end
+    end
+
+    public
+    def write_sync(elements, opts={})
+      options = {
+        :json => @opts[:json],
+      }.merge(opts)
+
+      elements = [elements] if elements.is_a?(Hash)
+      send_window_size(elements.size)
+
+      encoder = options[:json] ? JsonEncoder : FrameEncoder
+      payload = elements.map { |element| encoder.to_frame(element, inc) }.join
+      compress = compress_payload(payload)
+      send_payload(compress)
+
+      ack(elements.size)
+    end
+
+    private 
+    def compress_payload(payload)
+      compress = Zlib::Deflate.deflate(payload)
+      ["1", "C", compress.bytesize, compress].pack("AANA*")
+    end
+
+    private
+    def ack(size)
+      _, type = read_version_and_type
+      raise "Whoa we shouldn't get this frame: #{type}" if type != "A"
+      @last_ack = read_last_ack
+    end
+
+    private
+    def unacked_sequence_size
+      sequence - (@last_ack + 1)
+    end
+
+    private
+    def read_version_and_type
+      version = @socket.read(1)
+      type    = @socket.read(1)
+      [version, type]
+    end
+
+    private
+    def read_last_ack
+      @socket.read(4).unpack("N").first
+    end
+  end
+
+  module JsonEncoder
+    def self.to_frame(hash, sequence)
+      json = Lumberjack::json.dump(hash)
+      json_length = json.bytesize
+      pack = "AANNA#{json_length}"
+      frame = ["1", "J", sequence, json_length, json]
+      frame.pack(pack)
+    end
+  end # JsonEncoder
+
+  module FrameEncoder
+    def self.to_frame(hash, sequence)
+      frame = ["1", "D", sequence]
+      pack = "AAN"
+      keys = deep_keys(hash)
+      frame << keys.length
+      pack << "N"
+      keys.each do |k|
+        val = deep_get(hash,k)
+        key_length = k.bytesize
+        val_length = val.bytesize
+        frame << key_length
+        pack << "N"
+        frame << k
+        pack << "A#{key_length}"
+        frame << val_length
+        pack << "N"
+        frame << val
+        pack << "A#{val_length}"
+      end
+      frame.pack(pack)
+    end
+
+    private
+    def self.deep_get(hash, key="")
+      return hash if key.nil?
+      deep_get(
+        hash[key.split('.').first],
+        key[key.split('.').first.length+1..key.length]
+      )
+    end
+    private
+    def self.deep_keys(hash, prefix="")
+      keys = []
+      hash.each do |k,v|
+        keys << "#{prefix}#{k}" if v.class == String
+        keys << deep_keys(hash[k], "#{k}.") if v.class == Hash
+      end
+      keys.flatten
+    end
+  end # module Encoder
+end

data/lib/lumberjack/server.rb

@@ -0,0 +1,433 @@
+# encoding: utf-8
+require "lumberjack"
+require "socket"
+require "thread"
+require "openssl"
+require "zlib"
+require "json"
+require "concurrent"
+
+module Lumberjack
+  class Server
+    SOCKET_TIMEOUT = 1 # seconds
+
+    attr_reader :port
+
+    # Create a new Lumberjack server.
+    #
+    # - options is a hash. Valid options are:
+    #
+    # * :port - the port to listen on
+    # * :address - the host/address to bind to
+    # * :ssl_certificate - the path to the ssl cert to use
+    # * :ssl_key - the path to the ssl key to use
+    # * :ssl_key_passphrase - the key passphrase (optional)
+    def initialize(options={})
+      @options = {
+        :port => 0,
+        :address => "0.0.0.0",
+        :ssl => true,
+        :ssl_certificate => nil,
+        :ssl_key => nil,
+        :ssl_key_passphrase => nil
+      }.merge(options)
+
+      if @options[:ssl]
+        [:ssl_certificate, :ssl_key].each do |k|
+          if @options[k].nil?
+            raise "You must specify #{k} in Lumberjack::Server.new(...)"
+          end
+        end
+      end
+
+      @server = TCPServer.new(@options[:address], @options[:port])
+
+      @close = Concurrent::AtomicBoolean.new
+
+      # Query the port in case the port number is '0'
+      # TCPServer#addr == [ address_family, port, address, address ]
+      @port = @server.addr[1]
+
+      if @options[:ssl]
+        # load SSL certificate
+        @ssl = OpenSSL::SSL::SSLContext.new
+        @ssl.cert = OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate]))
+        @ssl.key = OpenSSL::PKey::RSA.new(File.read(@options[:ssl_key]),
+          @options[:ssl_key_passphrase])
+      end
+    end # def initialize
+
+    def run(&block)
+      while !closed?
+        connection = accept
+
+        # Some exception may occur in the accept loop
+        # we will try again in the next iteration
+        # unless the server is closing
+        next unless connection
+
+        Thread.new(connection) do |connection|
+          connection.run(&block)
+        end
+      end
+    end # def run
+
+    def ssl?
+      @ssl
+    end
+
+    def accept(&block)
+      begin
+        socket = @server.accept_nonblock
+        # update the socket with a SSL layer
+        socket = accept_ssl(socket) if ssl?
+
+        if block_given?
+          block.call(socket, self)
+        else
+          return Connection.new(socket, self)
+        end
+      rescue OpenSSL::SSL::SSLError, IOError, EOFError, Errno::EBADF
+        socket.close rescue nil
+        retry unless closed?
+      rescue IO::WaitReadable, Errno::EAGAIN # Resource not ready yet, so lets try again
+        begin
+          IO.select([@server], nil, nil, SOCKET_TIMEOUT)
+          retry unless closed?
+        rescue IOError, Errno::EBADF => e # we currently closing
+          raise e unless closed?
+        end
+      end
+    end
+
+    def accept_ssl(tcp_socket)
+      ssl_socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, @ssl)
+      ssl_socket.sync_close
+
+      begin
+        ssl_socket.accept_nonblock
+
+        return ssl_socket
+      rescue IO::WaitReadable # handshake
+        IO.select([ssl_socket], nil, nil, SOCKET_TIMEOUT)
+        retry unless closed?
+      rescue IO::WaitWritable # handshake
+        IO.select(nil, [ssl_socket], nil, SOCKET_TIMEOUT)
+        retry unless closed?
+      end
+    end
+
+    def closed?
+      @close.value
+    end
+
+    def close
+      @close.make_true
+      @server.close unless @server.closed?
+    end
+  end # class Server
+
+  class Parser
+    PROTOCOL_VERSION_1 = "1".ord
+    PROTOCOL_VERSION_2 = "2".ord
+
+    SUPPORTED_PROTOCOLS = [PROTOCOL_VERSION_1, PROTOCOL_VERSION_2]
+
+    def initialize
+      @buffer_offset = 0
+      @buffer = ""
+      @buffer.force_encoding("BINARY")
+      transition(:header, 2)
+    end # def initialize
+
+    def transition(state, next_length)
+      @state = state
+      #puts :transition => state
+      # TODO(sissel): Assert this self.respond_to?(state)
+      # TODO(sissel): Assert state is in STATES
+      # TODO(sissel): Assert next_length is a number
+      need(next_length)
+    end # def transition
+
+    # Feed data to this parser.
+    # 
+    # Currently, it will return the raw payload of websocket messages.
+    # Otherwise, it returns nil if no complete message has yet been consumed.
+    #
+    # @param [String] the string data to feed into the parser. 
+    # @return [String, nil] the websocket message payload, if any, nil otherwise.
+    def feed(data, &block)
+      @buffer << data
+      #p :need => @need
+      while have?(@need)
+        send(@state, &block) 
+        #case @state
+        #when :header; header(&block)
+        #when :window_size; window_size(&block)
+        #when :data_lead; data_lead(&block)
+        #when :data_field_key_len; data_field_key_len(&block)
+        #when :data_field_key; data_field_key(&block)
+        #when :data_field_value_len; data_field_value_len(&block)
+        #when :data_field_value; data_field_value(&block)
+        #when :data_field_value; data_field_value(&block)
+        #when :compressed_lead; compressed_lead(&block)
+        #when :compressed_payload; compressed_payload(&block)
+        #end # case @state
+      end
+      return nil
+    end # def <<
+
+    # Do we have at least 'length' bytes in the buffer?
+    def have?(length)
+      return length <= (@buffer.size - @buffer_offset)
+    end # def have?
+
+    # Get 'length' string from the buffer.
+    def get(length=nil)
+      length = @need if length.nil?
+      data = @buffer[@buffer_offset ... @buffer_offset + length]
+      @buffer_offset += length
+      if @buffer_offset > 16384
+        @buffer = @buffer[@buffer_offset  .. -1]
+        @buffer_offset = 0
+      end
+      return data
+    end # def get
+
+    # Set the minimum number of bytes we need in the buffer for the next read.
+    def need(length)
+      @need = length
+    end # def need
+
+    FRAME_WINDOW = "W".ord
+    FRAME_DATA = "D".ord
+    FRAME_JSON_DATA = "J".ord
+    FRAME_COMPRESSED = "C".ord
+    def header(&block)
+      version, frame_type = get.bytes.to_a[0..1]
+      version ||= PROTOCOL_VERSION_1
+
+      handle_version(version, &block)
+
+      case frame_type
+      when FRAME_WINDOW; transition(:window_size, 4)
+      when FRAME_DATA; transition(:data_lead, 8)
+      when FRAME_JSON_DATA; transition(:json_data_lead, 8)
+      when FRAME_COMPRESSED; transition(:compressed_lead, 4)
+      else; raise "Unknown frame type: `#{frame_type}`"
+      end
+    end
+
+    def handle_version(version, &block)
+      if supported_protocol?(version)
+        yield :version, version
+      else
+        raise "unsupported protocol #{version}"
+      end
+    end
+
+    def supported_protocol?(version)
+      SUPPORTED_PROTOCOLS.include?(version)
+    end
+
+    def window_size(&block)
+      @window_size = get.unpack("N").first
+      transition(:header, 2)
+      yield :window_size, @window_size
+    end # def window_size
+
+    def json_data_lead(&block)
+      @sequence, payload_size = get.unpack("NN")
+      transition(:json_data_payload, payload_size)
+    end
+
+    def json_data_payload(&block)
+      payload = get
+      yield :json, @sequence, Lumberjack::json.load(payload)
+      transition(:header, 2)
+    end
+
+    def data_lead(&block)
+      @sequence, @data_count = get.unpack("NN")
+      @data = {}
+      transition(:data_field_key_len, 4)
+    end
+
+    def data_field_key_len(&block)
+      key_len = get.unpack("N").first
+      transition(:data_field_key, key_len)
+    end
+
+    def data_field_key(&block)
+      @key = get
+      transition(:data_field_value_len, 4)
+    end
+
+    def data_field_value_len(&block)
+      transition(:data_field_value, get.unpack("N").first)
+    end
+
+    def data_field_value(&block)
+      @value = get
+
+      @data_count -= 1
+      @data[@key] = @value
+
+      if @data_count > 0
+        transition(:data_field_key_len, 4)
+      else
+        # emit the whole map now that we found the end of the data fields list.
+        yield :data, @sequence, @data
+        transition(:header, 2)
+      end
+
+    end # def data_field_value
+
+    def compressed_lead(&block)
+      length = get.unpack("N").first
+      transition(:compressed_payload, length)
+    end
+
+    def compressed_payload(&block)
+      original = Zlib::Inflate.inflate(get)
+      transition(:header, 2)
+
+      # Parse the uncompressed payload.
+      feed(original, &block)
+    end
+  end # class Parser
+
+  class Connection
+    READ_SIZE = 16384
+
+    attr_accessor :server
+
+    def initialize(fd, server)
+      @parser = Parser.new
+      @fd = fd
+
+      @server = server
+      @ack_handler = nil
+    end
+
+    def run(&block)
+      while !server.closed?
+        read_socket(&block)
+      end
+    rescue EOFError, OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET
+      # EOF or other read errors, only action is to shutdown which we'll do in
+      # 'ensure'
+    ensure
+      close rescue 'Already closed stream'
+    end # def run
+
+    def read_socket(&block)
+      # TODO(sissel): Ack on idle.
+      # X: - if any unacked, IO.select
+      # X:   - on timeout, ack all.
+      # X: Doing so will prevent slow streams from retransmitting
+      # X: too many events after errors.
+      @parser.feed(@fd.sysread(READ_SIZE)) do |event, *args|
+        case event
+        when :version
+          version(*args)
+        when :window_size
+          reset_next_ack(*args)
+        when :data
+          sequence, map = args
+          ack_if_needed(sequence) { data(map, &block) }
+        when :json
+          # If the payload is an array of items we will emit multiple events
+          # this behavior was moved from the plugin to the library.
+          # see this commit: https://github.com/logstash-plugins/logstash-input-lumberjack/pull/57/files#diff-1b9590423b15f04f215635164e7376ecR158
+          sequence, map = args
+
+          ack_if_needed(sequence) do
+            if map.is_a?(Array)
+              map.each { |e| data(e, &block) }
+            else
+              data(map, &block)
+            end
+          end
+        end
+      end
+    end
+
+    def version(version)
+      @version = version
+    end
+
+    def ack_if_needed(sequence, &block)
+      block.call
+      send_ack(sequence) if @ack_handler.ack?(sequence)
+    end
+
+    def close
+      @fd.close unless @fd.closed?
+    end
+
+    def data(map, &block)
+      block.call(map) if block_given?
+    end
+
+    def reset_next_ack(window_size)
+      klass = (@version == Parser::PROTOCOL_VERSION_1) ? AckingProtocolV1 : AckingProtocolV2
+      @ack_handler = klass.new(window_size)
+    end
+
+    def send_ack(sequence)
+      @fd.syswrite(@ack_handler.ack_frame(sequence))
+    end
+  end # class Connection
+
+  class AckingProtocolV1
+    def initialize(window_size)
+      @next_ack = nil
+      @window_size = window_size
+    end
+
+    def ack?(sequence)
+      # The first encoded event will contain the sequence number 
+      # this is needed to know when we should ack.
+      @next_ack = compute_next_ack(sequence) if @next_ack.nil?
+      sequence == @next_ack
+    end
+
+    def ack_frame(sequence)
+      ["1A", sequence].pack("A*N")
+    end
+
+    private
+    def compute_next_ack(sequence)
+      (sequence + @window_size - 1) % SEQUENCE_MAX
+    end
+  end
+
+  # Allow lumberjack to send partial ack back to the producer
+  # only V2 client support partial Acks
+  #
+  # Send Ack on every 20% of the data, so with default settings every 200 events
+  # This should reduce the congestion on retransmit.
+  class AckingProtocolV2
+    ACK_RATIO = 5
+
+    def initialize(window_size)
+      @window_size = window_size
+      @every = (window_size / ACK_RATIO).round
+    end
+
+    def ack?(sequence)
+      if @window_size == sequence
+        true
+      elsif sequence % @every == 0
+        true
+      else
+        false
+      end
+    end
+
+    def ack_frame(sequence)
+      ["2A", sequence].pack("A*N")
+    end
+  end
+end # module Lumberjack

data/lib/lumberjack.rb

@@ -0,0 +1,22 @@
+require "json"
+
+module Lumberjack
+  SEQUENCE_MAX = (2**32-1).freeze
+
+  @@json = Class.new do
+    def self.load(blob)
+      JSON.parse(blob)
+    end
+    def self.dump(v)
+      v.to_json
+    end
+  end
+
+  def self.json
+    @@json
+  end
+
+  def self.json=(j)
+    @@json = j
+  end
+end

data/spec/integration_spec.rb

@@ -0,0 +1,210 @@
+# encoding: utf-8
+require "lumberjack/client"
+require "lumberjack/server"
+require "stud/temporary"
+require "flores/pki"
+require "fileutils"
+require "thread"
+require "spec_helper"
+
+Thread.abort_on_exception = true
+describe "A client" do
+  let(:certificate) { Flores::PKI.generate }
+  let(:certificate_file_crt) { "certificate.crt" }
+  let(:certificate_file_key) { "certificate.key" }
+  let(:port) { Flores::Random.integer(1024..65335) }
+  let(:tcp_port) { port + 1 }
+  let(:host) { "127.0.0.1" }
+  let(:queue) { [] }
+
+  before do
+    expect(File).to receive(:read).at_least(1).with(certificate_file_crt) { certificate.first.to_s }
+    expect(File).to receive(:read).at_least(1).with(certificate_file_key) { certificate.last.to_s }
+
+    tcp_server = Lumberjack::Server.new(:port => tcp_port, :address => host, :ssl => false)
+
+    ssl_server = Lumberjack::Server.new(:port => port,
+                                        :address => host,
+                                        :ssl_certificate => certificate_file_crt,
+                                        :ssl_key => certificate_file_key)
+
+    @tcp_server = Thread.new do
+      while true
+        tcp_server.accept do |socket|
+          con = Lumberjack::Connection.new(socket, tcp_server)
+          begin
+            con.run { |data| queue << data }
+          rescue
+            # Close connection on failure. For example SSL client will make
+            # parser for TCP based server trip.
+            # Connection is closed by Server connection object
+          end
+        end
+      end
+    end
+
+    @ssl_server = Thread.new do
+      ssl_server.run { |data| queue << data }
+    end
+  end
+
+  shared_examples "send payload" do
+    it "supports single element" do
+      (1..random_number_of_events).each do |n|
+        expect(client.write(payload)).to eq(sequence_start + n)
+      end
+      sleep(0.5) # give time to the server to read the events
+      expect(queue.size).to eq(random_number_of_events)
+    end
+
+    it "support sending multiple elements in one payload" do
+      expect(client.write(batch_payload)).to eq(sequence_start + batch_size)
+      sleep(0.5)
+
+      expect(queue.size).to eq(batch_size)
+      expect(queue).to match_array(batch_payload)
+    end
+  end
+
+  shared_examples "transmit payloads" do
+    let(:random_number_of_events) { Flores::Random.integer(2..10) }
+    let(:payload) { { "line" => "foobar" } }
+    let(:batch_size) { Flores::Random.integer(1..1024) }
+    let(:batch_payload) do
+      batch = []
+      batch_size.times do |n|
+        batch <<  { "line" => "foobar #{n}" }
+      end
+      batch
+    end
+
+    context "when sequence start at 0" do
+      let(:sequence_start) { 0 }
+
+      include_examples "send payload"
+    end
+
+    context "when sequence doesn't start at zero" do
+      let(:sequence_start) { Flores::Random.integer(1..2000) }
+
+      before do
+        client.instance_variable_get(:@socket).instance_variable_set(:@sequence, sequence_start)
+      end
+
+      include_examples "send payload"
+    end
+
+    context "when the sequence rollover" do
+      let(:batch_size) { 100 }
+      let(:sequence_start) { Lumberjack::SEQUENCE_MAX - batch_size / 2 }
+
+      before do
+        client.instance_variable_get(:@socket).instance_variable_set(:@sequence, sequence_start)
+      end
+
+      it "adjusts the ack" do
+        expect(client.write(batch_payload)).to eq(batch_size / 2)
+        sleep(0.5)
+        expect(queue.size).to eq(batch_size)
+        expect(queue).to match_array(batch_payload)
+      end
+    end
+  end
+
+  context "using plain tcp connection" do
+    it "should successfully connect to tcp server if ssl explicitely disabled" do
+      expect {
+        Lumberjack::Client.new(:port => tcp_port, :host => host, :addresses => host, :ssl => false)
+      }.not_to raise_error
+    end
+
+    it "should fail to connect to tcp server if ssl not explicitely disabled" do
+      expect {
+        Lumberjack::Client.new(:port => tcp_port, :host => host, :addresses => host)
+      }.to raise_error(RuntimeError, /Must set a ssl certificate/)
+    end
+
+    it "should fail to communicate to ssl based server" do
+      expect {
+        client = Lumberjack::Client.new(:port => port,
+                                        :host => host,
+                                        :addresses => host,
+                                        :ssl => false)
+        client.write({ "line" => "foobar" })
+      }.to raise_error(RuntimeError)
+    end
+
+    context "When transmitting a payload" do
+      let(:options) { {:port => tcp_port, :host => host, :addresses => host, :ssl => false } }
+      let(:client) { Lumberjack::Client.new(options) }
+
+      context "json" do
+        let(:options) { super.merge({ :json => true }) }
+        include_examples "transmit payloads"
+      end
+
+      context "v1 frame" do
+        include_examples "transmit payloads"
+      end
+    end
+  end
+
+  context "using ssl encrypted connection" do
+    context "with a valid certificate" do
+      it "successfully connect to the server" do
+        expect { 
+          Lumberjack::Client.new(:port => port,
+                                 :host => host,
+                                 :addresses => host,
+                                 :ssl_certificate => certificate_file_crt)
+        }.not_to raise_error
+      end
+
+      it "should fail connecting to plain tcp server" do
+        expect { 
+          Lumberjack::Client.new(:port => tcp_port,
+                                 :host => host,
+                                 :addresses => host,
+                                 :ssl_certificate => certificate_file_crt)
+        }.to raise_error(OpenSSL::SSL::SSLError)
+      end
+    end
+
+    context "with an invalid certificate" do
+      let(:invalid_certificate) { Flores::PKI.generate }
+      let(:invalid_certificate_file) { "invalid.crt" }
+
+      before do
+        expect(File).to receive(:read).with(invalid_certificate_file) { invalid_certificate.first.to_s }
+      end
+
+      it "should refuse to connect" do
+        expect {
+          Lumberjack::Client.new(:port => port,
+                                 :host => host,
+                                 :addresses => host,
+                                 :ssl_certificate => invalid_certificate_file)
+
+        }.to raise_error(OpenSSL::SSL::SSLError, /certificate verify failed/)
+      end
+    end
+
+    context "When transmitting a payload" do
+      let(:client) do
+        Lumberjack::Client.new(:port => port,
+                               :host => host,
+                               :addresses => host,
+                               :ssl_certificate => certificate_file_crt)
+      end
+
+      context "json" do
+        let(:options) { super.merge({ :json => true }) }
+        include_examples "transmit payloads"
+      end
+
+      context "v1 frame" do
+        include_examples "transmit payloads"
+      end
+    end
+  end
+end

data/spec/lumberjack/acking_protocol_v1_spec.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require "lumberjack/server"
+require "flores/random"
+
+describe Lumberjack::AckingProtocolV1 do
+  let(:number_of_events) { Flores::Random.integer(100..1024) }
+
+  subject { Lumberjack::AckingProtocolV1.new(number_of_events) }
+
+  it "should return true only once" do
+    results = []
+    number_of_events.times { |n| results << subject.ack?(n) }
+    expect(results.size).to eq(number_of_events)
+    expect(results.count(true)).to eq(1)
+  end
+end

data/spec/lumberjack/acking_protocol_v2_spec.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+require "lumberjack/server"
+require "flores/random"
+
+describe Lumberjack::AckingProtocolV2 do
+  let(:results) { [] }
+  subject { Lumberjack::AckingProtocolV2.new(number_of_events) }
+  before { 1.upto(number_of_events) { |n| results << subject.ack?(n) } }
+
+  context "with multiples events" do
+    let(:number_of_events) { Flores::Random.integer(100..1024) }
+
+    it "should return multiples partial acks" do
+      expect(results.size).to eq(number_of_events)
+      expect(results.count(true)).to be_within(1).of((number_of_events / number_of_events * Lumberjack::AckingProtocolV2::ACK_RATIO).ceil)
+    end
+
+    it "last ack should be true" do
+      expect(results.last).to be_truthy
+    end
+  end
+
+  context "with only one event" do
+    let(:number_of_events) { 1 }
+
+    it "should return true only once" do
+      expect(results.size).to eq(number_of_events)
+      expect(results.count(true)).to eq(1)
+    end
+  end
+end

data/spec/lumberjack/client_spec.rb

@@ -0,0 +1,106 @@
+# encoding: utf-8
+require 'spec_helper'
+require 'lumberjack/client'
+require 'lumberjack/server'
+require "socket"
+require "thread"
+require "openssl"
+require "zlib"
+
+describe "Lumberjack::Client" do
+  describe "Lumberjack::Socket" do
+    let(:port)   { 5000 }
+
+    subject(:socket) { Lumberjack::Socket.new(:port => port, :ssl_certificate => "" ) }
+
+    before do
+      allow_any_instance_of(Lumberjack::Socket).to receive(:connection_start).and_return(true)
+      # mock any network call
+      allow(socket).to receive(:send_window_size).with(kind_of(Integer)).and_return(true)
+      allow(socket).to receive(:send_payload).with(kind_of(String)).and_return(true)
+    end
+
+    context "sequence" do
+     let(:hash)   { {:a => 1, :b => 2}}
+     let(:max_unsigned_int) { (2**32)-1 }
+
+      before(:each) do
+        allow(socket).to receive(:ack).and_return(true)
+      end
+
+      it "force sequence to be an unsigned 32 bits int" do
+        socket.instance_variable_set(:@sequence, max_unsigned_int)
+        socket.write_sync(hash)
+        expect(socket.sequence).to eq(1)
+      end
+    end
+  end
+
+  describe Lumberjack::FrameEncoder do
+    it 'should creates frames without truncating accentued characters' do
+      content = {
+        "message" => "Le Canadien de Montréal est la meilleure équipe au monde!",
+        "other" => "éléphant"
+      }
+      parser = Lumberjack::Parser.new
+      parser.feed(Lumberjack::FrameEncoder.to_frame(content, 0)) do |code, sequence, data|
+        if code == :data
+          expect(data["message"].force_encoding('UTF-8')).to eq(content["message"])
+          expect(data["other"].force_encoding('UTF-8')).to eq(content["other"])
+        end
+      end
+    end
+
+    it 'should creates frames without dropping multibytes characters' do
+      content = {
+        "message" => "国際ホッケー連盟" # International Hockey Federation
+      }
+      parser = Lumberjack::Parser.new
+      parser.feed(Lumberjack::FrameEncoder.to_frame(content, 0)) do |code, sequence, data|
+        expect(data["message"].force_encoding('UTF-8')).to eq(content["message"]) if code == :data
+      end
+    end
+  end
+
+  describe Lumberjack::JsonEncoder do
+    it 'should create frames from nested hash' do
+      content = {
+        "number" => 1,
+        "string" => "hello world",
+        "array" => [1,2,3],
+        "sub" => {
+          "a" => 1
+        }
+      }
+      parser = Lumberjack::Parser.new
+      frame = Lumberjack::JsonEncoder.to_frame(content, 0)
+      parser.feed(frame) do |code, sequence, data|
+        expect(data).to eq(content) if code == :json
+      end
+    end
+
+    it 'should creates frames without truncating accentued characters' do
+      content = {
+        "message" => "Le Canadien de Montréal est la meilleure équipe au monde!",
+        "other" => "éléphant"
+      }
+      parser = Lumberjack::Parser.new
+      parser.feed(Lumberjack::JsonEncoder.to_frame(content, 0)) do |code, sequence, data|
+        if code == :json
+          expect(data["message"]).to eq(content["message"])
+          expect(data["other"]).to eq(content["other"])
+        end
+      end
+    end
+
+    it 'should creates frames without dropping multibytes characters' do
+      content = {
+        "message" => "国際ホッケー連盟" # International Hockey Federation
+      }
+      parser = Lumberjack::Parser.new
+      parser.feed(Lumberjack::JsonEncoder.to_frame(content, 0)) do |code, sequence, data|
+        expect(data["message"]).to eq(content["message"]) if code == :json
+      end
+    end
+  end
+end

data/spec/lumberjack/connection_spec.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require "lumberjack/server"
+require "spec_helper"
+require "flores/random"
+
+describe "Connnection" do
+  let(:server) { double("server", :closed? => false) }
+  let(:socket) { double("socket", :closed? => false) }
+  let(:connection) { Lumberjack::Connection.new(socket, server) }
+  let(:payload) { {"line" => "foobar" } }
+  let(:start_sequence) { Flores::Random.integer(0..2000) }
+  let(:random_number_of_events) { Flores::Random.integer(2..200) }
+
+  context "when the server is running" do
+    before do
+      expect(socket).to receive(:sysread).at_least(:once).with(Lumberjack::Connection::READ_SIZE).and_return("")
+      allow(socket).to receive(:syswrite).with(anything).and_return(true)
+      allow(socket).to receive(:close)
+
+
+      expectation = receive(:feed)
+        .with("")
+        .and_yield(:version, Lumberjack::Parser::PROTOCOL_VERSION_1)
+        .and_yield(:window_size, random_number_of_events)
+
+      random_number_of_events.times { |n| expectation.and_yield(:data, start_sequence + n + 1, payload) }
+
+      expect_any_instance_of(Lumberjack::Parser).to expectation
+    end
+
+    it "should ack the end of a sequence" do
+      expect(socket).to receive(:syswrite).with(["1A", random_number_of_events + start_sequence].pack("A*N"))
+      connection.read_socket
+    end
+  end
+
+  context "when the server stop" do
+    let(:server) { double("server", :closed? => true) }
+    before do
+      expect(socket).to receive(:close).and_return(true)
+    end
+
+    it "stop reading from the socket" do
+      expect { |b| connection.run(&b) }.not_to yield_control
+    end
+  end
+end

data/spec/lumberjack/server_spec.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+require "lumberjack/client"
+require "lumberjack/server"
+require "flores/random"
+require "flores/pki"
+require "spec_helper"
+
+Thread.abort_on_exception = true
+
+describe "Server" do
+  let(:certificate) { Flores::PKI.generate }
+  let(:certificate_file_crt) { "certificate.crt" }
+  let(:certificate_file_key) { "certificate.key" }
+  let(:port) { Flores::Random.integer(1024..65335) }
+  let(:tcp_port) { port + 1 }
+  let(:host) { "127.0.0.1" }
+  let(:queue) { [] }
+
+  before do
+    expect(File).to receive(:read).at_least(1).with(certificate_file_crt) { certificate.first.to_s }
+    expect(File).to receive(:read).at_least(1).with(certificate_file_key) { certificate.last.to_s }
+  end
+
+  subject do
+    Lumberjack::Server.new(:port => port,
+                           :address => host,
+                           :ssl_certificate => certificate_file_crt,
+                           :ssl_key => certificate_file_key)
+  end
+
+  it "should not block when closing the server" do
+    thread = Thread.new do
+      subject.run do |event|
+        queue << event
+      end
+    end
+
+    sleep(1) while thread.status != "run"
+    subject.close
+    wait_for { thread.status }.to be_falsey
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require 'rspec'
+require 'rspec/mocks'
+require 'rspec/wait'
+
+$: << File.realpath(File.join(File.dirname(__FILE__), "..", "lib"))
+
+RSpec.configure do |config|
+  config.order = :rand
+end

metadata

@@ -0,0 +1,145 @@
+--- !ruby/object:Gem::Specification
+name: jls-lumberjack-logzio
+version: !ruby/object:Gem::Version
+  version: 0.0.26
+platform: ruby
+authors:
+- Jordan Sissel
+- Roi Rav-Hon
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-05-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: flores
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.6
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: stud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-wait
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: lumberjack log transport library
+email:
+- jls@semicomplete.com
+- roi@logz.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/lumberjack.rb
+- lib/lumberjack/client.rb
+- lib/lumberjack/server.rb
+- spec/integration_spec.rb
+- spec/lumberjack/acking_protocol_v1_spec.rb
+- spec/lumberjack/acking_protocol_v2_spec.rb
+- spec/lumberjack/client_spec.rb
+- spec/lumberjack/connection_spec.rb
+- spec/lumberjack/server_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/jordansissel/lumberjack
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: lumberjack log transport library
+test_files:
+- spec/spec_helper.rb
+- spec/integration_spec.rb
+- spec/lumberjack/acking_protocol_v1_spec.rb
+- spec/lumberjack/client_spec.rb
+- spec/lumberjack/server_spec.rb
+- spec/lumberjack/acking_protocol_v2_spec.rb
+- spec/lumberjack/connection_spec.rb