jls-lumberjack 0.0.24 → 0.0.25.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d58912ce6afda9045d016510618a04363349628d
-  data.tar.gz: faeaba187853930185d78673f4b69c16c5b107bb
+  metadata.gz: e2b2959f02e7f417cf30eb4d840508950edf0808
+  data.tar.gz: a1f37209be258dbcc97617b8eed7e5c61ca66db5
 SHA512:
-  metadata.gz: e40270de1f12efc47375be7a5a9cd9c83fae6ce2b615fe50885037746ecf074e1a26bf2dd4b2bf8a2ebdd1816c95f6d36b56e84de32e3fc5f73a225be7bc6aa7
-  data.tar.gz: 6224d5a7b5475195b0afd5168eea3c37a8c704cef432cf90b01cb8fcd520ea3293f1503e89b6bd4e0eaf2e84f24afeab78b7681ab9be4999e2720b2987291dd8
+  metadata.gz: a87220c6b45a3c53f4bf685c2309c37cdd953c06d489f56cb442eb9ebc482357067b3fcad526b7e44c28455117be4c189226815184d65049bb6b429326ef5d42
+  data.tar.gz: 8d326476bf7fe3d76873fc998dae45fbaf94cbe776de1a9e276fe8d4fadcfe6243a3ac2db7c90568056998c8296dabe44424fc3b03dfb1a7f91f7d87a5df41ae

data/lib/lumberjack.rb

@@ -1,3 +1,22 @@
+require "json"
+
 module Lumberjack
   SEQUENCE_MAX = (2**32-1).freeze
+
+  @@json = Class.new do
+    def self.load(blob)
+      JSON.parse(blob)
+    end
+    def self.dump(v)
+      v.to_json
+    end
+  end
+
+  def self.json
+    @@json
+  end
+
+  def self.json=(j)
+    @@json = j
+  end
 end

data/lib/lumberjack/client.rb

@@ -12,15 +12,16 @@ module Lumberjack
         :port => 0,
         :addresses => [],
         :ssl_certificate => nil,
+        :ssl => true,
+        :json => false,
       }.merge(opts)
 
       @opts[:addresses] = [@opts[:addresses]] if @opts[:addresses].class == String
       raise "Must set a port." if @opts[:port] == 0
       raise "Must set atleast one address" if @opts[:addresses].empty? == 0
-      raise "Must set a ssl certificate or path" if @opts[:ssl_certificate].nil?
+      raise "Must set a ssl certificate or path" if @opts[:ssl_certificate].nil? && @opts[:ssl]
 
       @socket = connect
-
     end
 
     private
@@ -37,8 +38,8 @@ module Lumberjack
     end
 
     public
-    def write(elements)
-      @socket.write_sync(elements)
+    def write(elements, opts={})
+      @socket.write_sync(elements, opts)
     end
 
     public
@@ -54,7 +55,10 @@ module Lumberjack
     #
     # * :port - the port to listen on
     # * :address - the host/address to bind to
-    # * :ssl_certificate - the path to the ssl cert to use
+    # * :ssl - enable/disable ssl support
+    # * :ssl_certificate - the path to the ssl cert to use.
+    #                      If ssl_certificate is not set, a plain tcp connection
+    #                      will be used.
     attr_reader :sequence
     attr_reader :host
     def initialize(opts={})
@@ -64,6 +68,8 @@ module Lumberjack
         :port => 0,
         :address => "127.0.0.1",
         :ssl_certificate => nil,
+        :ssl => true,
+        :json => false,
       }.merge(opts)
       @host = @opts[:address]
 
@@ -73,18 +79,21 @@ module Lumberjack
     private
     def connection_start(opts)
       tcp_socket = TCPSocket.new(opts[:address], opts[:port])
+      if !opts[:ssl]
+        @socket = tcp_socket
+      else
+        certificate = OpenSSL::X509::Certificate.new(File.read(opts[:ssl_certificate]))
 
-      certificate = OpenSSL::X509::Certificate.new(File.read(opts[:ssl_certificate]))
-
-      certificate_store = OpenSSL::X509::Store.new
-      certificate_store.add_cert(certificate)
+        certificate_store = OpenSSL::X509::Store.new
+        certificate_store.add_cert(certificate)
 
-      ssl_context = OpenSSL::SSL::SSLContext.new
-      ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
-      ssl_context.cert_store = certificate_store
+        ssl_context = OpenSSL::SSL::SSLContext.new
+        ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        ssl_context.cert_store = certificate_store
 
-      @socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, ssl_context)
-      @socket.connect
+        @socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, ssl_context)
+        @socket.connect
+      end
     end
 
     private 
@@ -109,11 +118,16 @@ module Lumberjack
     end
 
     public
-    def write_sync(elements)
+    def write_sync(elements, opts={})
+      options = {
+        :json => @opts[:json],
+      }.merge(opts)
+
       elements = [elements] if elements.is_a?(Hash)
       send_window_size(elements.size)
 
-      payload = elements.map { |element| Encoder.to_frame(element, inc) }.join
+      encoder = options[:json] ? JsonEncoder : FrameEncoder
+      payload = elements.map { |element| encoder.to_frame(element, inc) }.join
       compress = compress_payload(payload)
       send_payload(compress)
 
@@ -151,7 +165,17 @@ module Lumberjack
     end
   end
 
-  module Encoder
+  module JsonEncoder
+    def self.to_frame(hash, sequence)
+      json = Lumberjack::json.dump(hash)
+      json_length = json.bytesize
+      pack = "AANNA#{json_length}"
+      frame = ["1", "J", sequence, json_length, json]
+      frame.pack(pack)
+    end
+  end # JsonEncoder
+
+  module FrameEncoder
     def self.to_frame(hash, sequence)
       frame = ["1", "D", sequence]
       pack = "AAN"

data/lib/lumberjack/server.rb

@@ -4,9 +4,13 @@ require "socket"
 require "thread"
 require "openssl"
 require "zlib"
+require "json"
+require "concurrent"
 
 module Lumberjack
   class Server
+    SOCKET_TIMEOUT = 1 # seconds
+
     attr_reader :port
 
     # Create a new Lumberjack server.
@@ -22,55 +26,113 @@ module Lumberjack
       @options = {
         :port => 0,
         :address => "0.0.0.0",
+        :ssl => true,
         :ssl_certificate => nil,
         :ssl_key => nil,
         :ssl_key_passphrase => nil
       }.merge(options)
 
-      [:ssl_certificate, :ssl_key].each do |k|
-        if @options[k].nil?
-          raise "You must specify #{k} in Lumberjack::Server.new(...)"
+      if @options[:ssl]
+        [:ssl_certificate, :ssl_key].each do |k|
+          if @options[k].nil?
+            raise "You must specify #{k} in Lumberjack::Server.new(...)"
+          end
         end
       end
 
-      @tcp_server = TCPServer.new(@options[:address], @options[:port])
+      @server = TCPServer.new(@options[:address], @options[:port])
+
+      @close = Concurrent::AtomicBoolean.new
 
       # Query the port in case the port number is '0'
       # TCPServer#addr == [ address_family, port, address, address ]
-      @port = @tcp_server.addr[1]
-      @ssl = OpenSSL::SSL::SSLContext.new
-      @ssl.cert = OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate]))
-      @ssl.key = OpenSSL::PKey::RSA.new(File.read(@options[:ssl_key]),
-                                        @options[:ssl_key_passphrase])
-      @ssl_server = OpenSSL::SSL::SSLServer.new(@tcp_server, @ssl)
+      @port = @server.addr[1]
+
+      if @options[:ssl]
+        # load SSL certificate
+        @ssl = OpenSSL::SSL::SSLContext.new
+        @ssl.cert = OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate]))
+        @ssl.key = OpenSSL::PKey::RSA.new(File.read(@options[:ssl_key]),
+          @options[:ssl_key_passphrase])
+      end
     end # def initialize
 
     def run(&block)
-      while true
+      while !closed?
         connection = accept
+
+        # Some exception may occur in the accept loop
+        # we will try again in the next iteration
+        # unless the server is closing
+        next unless connection
+
         Thread.new(connection) do |connection|
           connection.run(&block)
         end
       end
     end # def run
 
+    def ssl?
+      @ssl
+    end
+
     def accept(&block)
       begin
-        fd = @ssl_server.accept
-      rescue EOFError, OpenSSL::SSL::SSLError, IOError
-        # ssl handshake or other accept-related failure.
-        # TODO(sissel): Make it possible to log this.
-        retry
+        socket = @server.accept_nonblock
+        # update the socket with a SSL layer
+        socket = accept_ssl(socket) if ssl?
+
+        if block_given?
+          block.call(socket, self)
+        else
+          return Connection.new(socket, self)
+        end
+      rescue OpenSSL::SSL::SSLError, IOError, EOFError, Errno::EBADF
+        socket.close rescue nil
+        retry unless closed?
+      rescue IO::WaitReadable, Errno::EAGAIN # Resource not ready yet, so lets try again
+        begin
+          IO.select([@server], nil, nil, SOCKET_TIMEOUT)
+          retry unless closed?
+        rescue IOError, Errno::EBADF => e # we currently closing
+          raise e unless closed?
+        end
       end
-      if block_given?
-        block.call(fd)
-      else
-        Connection.new(fd)
+    end
+
+    def accept_ssl(tcp_socket)
+      ssl_socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, @ssl)
+      ssl_socket.sync_close
+
+      begin
+        ssl_socket.accept_nonblock
+
+        return ssl_socket
+      rescue IO::WaitReadable # handshake
+        IO.select([ssl_socket], nil, nil, SOCKET_TIMEOUT)
+        retry unless closed?
+      rescue IO::WaitWritable # handshake
+        IO.select(nil, [ssl_socket], nil, SOCKET_TIMEOUT)
+        retry unless closed?
       end
     end
+
+    def closed?
+      @close.value
+    end
+
+    def close
+      @close.make_true
+      @server.close unless @server.closed?
+    end
   end # class Server
 
   class Parser
+    PROTOCOL_VERSION_1 = "1".ord
+    PROTOCOL_VERSION_2 = "2".ord
+
+    SUPPORTED_PROTOCOLS = [PROTOCOL_VERSION_1, PROTOCOL_VERSION_2]
+
     def initialize
       @buffer_offset = 0
       @buffer = ""
@@ -139,24 +201,52 @@ module Lumberjack
 
     FRAME_WINDOW = "W".ord
     FRAME_DATA = "D".ord
+    FRAME_JSON_DATA = "J".ord
     FRAME_COMPRESSED = "C".ord
     def header(&block)
       version, frame_type = get.bytes.to_a[0..1]
+      version ||= PROTOCOL_VERSION_1
+
+      handle_version(version, &block)
 
       case frame_type
       when FRAME_WINDOW; transition(:window_size, 4)
       when FRAME_DATA; transition(:data_lead, 8)
+      when FRAME_JSON_DATA; transition(:json_data_lead, 8)
       when FRAME_COMPRESSED; transition(:compressed_lead, 4)
-      else; raise "Unknown frame type: #{frame_type}"
+      else; raise "Unknown frame type: `#{frame_type}`"
       end
     end
 
+    def handle_version(version, &block)
+      if supported_protocol?(version)
+        yield :version, version
+      else
+        raise "unsupported protocol #{version}"
+      end
+    end
+
+    def supported_protocol?(version)
+      SUPPORTED_PROTOCOLS.include?(version)
+    end
+
     def window_size(&block)
       @window_size = get.unpack("N").first
       transition(:header, 2)
       yield :window_size, @window_size
     end # def window_size
 
+    def json_data_lead(&block)
+      @sequence, payload_size = get.unpack("NN")
+      transition(:json_data_payload, payload_size)
+    end
+
+    def json_data_payload(&block)
+      payload = get
+      yield :json, @sequence, Lumberjack::json.load(payload)
+      transition(:header, 2)
+    end
+
     def data_lead(&block)
       @sequence, @data_count = get.unpack("NN")
       @data = {}
@@ -210,17 +300,18 @@ module Lumberjack
   class Connection
     READ_SIZE = 16384
 
-    def initialize(fd)
-      super()
+    attr_accessor :server
+
+    def initialize(fd, server)
       @parser = Parser.new
       @fd = fd
 
-      # a safe default until we are told by the client what window size to use
-      @window_size = 1 
+      @server = server
+      @ack_handler = nil
     end
 
     def run(&block)
-      while true
+      while !server.closed?
         read_socket(&block)
       end
     rescue EOFError, OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET
@@ -238,42 +329,97 @@ module Lumberjack
       # X: too many events after errors.
       @parser.feed(@fd.sysread(READ_SIZE)) do |event, *args|
         case event
+        when :version
+          version(*args)
         when :window_size
-          # We receive a new payload
-          window_size(*args)
-          reset_next_ack
+          reset_next_ack(*args)
         when :data
-          data(*args, &block)
+          sequence, map = args
+          ack_if_needed(sequence) { data(map, &block) }
+        when :json
+          # If the payload is an array of items we will emit multiple events
+          # this behavior was moved from the plugin to the library.
+          # see this commit: https://github.com/logstash-plugins/logstash-input-lumberjack/pull/57/files#diff-1b9590423b15f04f215635164e7376ecR158
+          sequence, map = args
+
+          ack_if_needed(sequence) do
+            if map.is_a?(Array)
+              map.each { |e| data(e, &block) }
+            else
+              data(map, &block)
+            end
+          end
         end
       end
     end
 
-    def close
-      @fd.close
+    def version(version)
+      @version = version
     end
 
-    def window_size(size)
-      @window_size = size
+    def ack_if_needed(sequence, &block)
+      block.call
+      send_ack(sequence) if @ack_handler.ack?(sequence)
     end
 
-    def reset_next_ack
-      @next_ack = nil
+    def close
+      @fd.close unless @fd.closed?
     end
 
-    def data(sequence, map, &block)
+    def data(map, &block)
       block.call(map) if block_given?
-      ack_if_needed(sequence)
     end
-    
-    def compute_next_ack(sequence)
-      (sequence + @window_size - 1) % SEQUENCE_MAX
+
+    def reset_next_ack(window_size)
+      klass = (@version == Parser::PROTOCOL_VERSION_1) ? AckingProtocolV1 : AckingProtocolV2
+      @ack_handler = klass.new(window_size)
+    end
+
+    def send_ack(sequence)
+      @fd.syswrite(["1A", sequence].pack("A*N"))
+    end
+  end # class Connection
+
+  class AckingProtocolV1
+    def initialize(window_size)
+      @next_ack = nil
+      @window_size = window_size
     end
 
-    def ack_if_needed(sequence)
+    def ack?(sequence)
       # The first encoded event will contain the sequence number 
       # this is needed to know when we should ack.
       @next_ack = compute_next_ack(sequence) if @next_ack.nil?
-      @fd.syswrite(["1A", sequence].pack("A*N")) if sequence == @next_ack
+      sequence == @next_ack
     end
-  end # class Connection
+
+    private
+    def compute_next_ack(sequence)
+      (sequence + @window_size - 1) % SEQUENCE_MAX
+    end
+  end
+
+  # Allow lumberjack to send partial ack back to the producer
+  # only V2 client support partial Acks
+  #
+  # Send Ack on every 20% of the data, so with default settings every 200 events
+  # This should reduce the congestion on retransmit.
+  class AckingProtocolV2
+    ACK_RATIO = 5
+
+    def initialize(window_size)
+      @window_size = window_size
+      @every = (window_size / ACK_RATIO).round
+    end
+
+    def ack?(sequence)
+      if @window_size == sequence
+        true
+      elsif sequence % @every == 0
+        true
+      else
+        false
+      end
+    end
+  end
 end # module Lumberjack

data/spec/integration_spec.rb

@@ -8,60 +8,46 @@ require "thread"
 require "spec_helper"
 
 Thread.abort_on_exception = true
-
 describe "A client" do
   let(:certificate) { Flores::PKI.generate }
   let(:certificate_file_crt) { "certificate.crt" }
   let(:certificate_file_key) { "certificate.key" }
   let(:port) { Flores::Random.integer(1024..65335) }
+  let(:tcp_port) { port + 1 }
   let(:host) { "127.0.0.1" }
   let(:queue) { [] }
 
   before do
     expect(File).to receive(:read).at_least(1).with(certificate_file_crt) { certificate.first.to_s }
     expect(File).to receive(:read).at_least(1).with(certificate_file_key) { certificate.last.to_s }
-    
-    server = Lumberjack::Server.new(:port => port,
-                                    :address => host,
-                                    :ssl_certificate => certificate_file_crt,
-                                    :ssl_key => certificate_file_key)
-
-    @server = Thread.new do
-      server.run { |data| queue << data }
-    end
-  end
-
-  context "with a valid certificate" do
-    it "successfully connect to the server" do
-      expect { 
-        Lumberjack::Client.new(:port => port, 
-                               :host => host,
-                               :addresses => host,
-                               :ssl_certificate => certificate_file_crt)
 
-      }.not_to raise_error
-    end
-  end
-
-  context "with an invalid certificate" do
-    let(:invalid_certificate) { Flores::PKI.generate }
-    let(:invalid_certificate_file) { "invalid.crt" }
-
-    before do
-      expect(File).to receive(:read).with(invalid_certificate_file) { invalid_certificate.first.to_s }
+    tcp_server = Lumberjack::Server.new(:port => tcp_port, :address => host, :ssl => false)
+
+    ssl_server = Lumberjack::Server.new(:port => port,
+                                        :address => host,
+                                        :ssl_certificate => certificate_file_crt,
+                                        :ssl_key => certificate_file_key)
+
+    @tcp_server = Thread.new do
+      while true
+        tcp_server.accept do |socket|
+          con = Lumberjack::Connection.new(socket, tcp_server)
+          begin
+            con.run { |data| queue << data }
+          rescue
+            # Close connection on failure. For example SSL client will make
+            # parser for TCP based server trip.
+            # Connection is closed by Server connection object
+          end
+        end
+      end
     end
 
-    it "should refuse to connect" do
-      expect { 
-        Lumberjack::Client.new(:port => port, 
-                               :host => host,
-                               :addresses => host,
-                               :ssl_certificate => invalid_certificate_file)
-
-      }.to raise_error(OpenSSL::SSL::SSLError, /certificate verify failed/)
+    @ssl_server = Thread.new do
+      ssl_server.run { |data| queue << data }
     end
   end
-  
+
   shared_examples "send payload" do
     it "supports single element" do
       (1..random_number_of_events).each do |n|
@@ -80,15 +66,9 @@ describe "A client" do
     end
   end
 
-  context "When transmitting a payload" do
+  shared_examples "transmit payloads" do
     let(:random_number_of_events) { Flores::Random.integer(2..10) }
     let(:payload) { { "line" => "foobar" } }
-    let(:client) do
-      Lumberjack::Client.new(:port => port, 
-                             :host => host,
-                             :addresses => host,
-                             :ssl_certificate => certificate_file_crt)
-    end
     let(:batch_size) { Flores::Random.integer(1..1024) }
     let(:batch_payload) do
       batch = []
@@ -130,4 +110,101 @@ describe "A client" do
       end
     end
   end
+
+  context "using plain tcp connection" do
+    it "should successfully connect to tcp server if ssl explicitely disabled" do
+      expect {
+        Lumberjack::Client.new(:port => tcp_port, :host => host, :addresses => host, :ssl => false)
+      }.not_to raise_error
+    end
+
+    it "should fail to connect to tcp server if ssl not explicitely disabled" do
+      expect {
+        Lumberjack::Client.new(:port => tcp_port, :host => host, :addresses => host)
+      }.to raise_error(RuntimeError, /Must set a ssl certificate/)
+    end
+
+    it "should fail to communicate to ssl based server" do
+      expect {
+        client = Lumberjack::Client.new(:port => port,
+                                        :host => host,
+                                        :addresses => host,
+                                        :ssl => false)
+        client.write({ "line" => "foobar" })
+      }.to raise_error(RuntimeError)
+    end
+
+    context "When transmitting a payload" do
+      let(:options) { {:port => tcp_port, :host => host, :addresses => host, :ssl => false } }
+      let(:client) { Lumberjack::Client.new(options) }
+
+      context "json" do
+        let(:options) { super.merge({ :json => true }) }
+        include_examples "transmit payloads"
+      end
+
+      context "v1 frame" do
+        include_examples "transmit payloads"
+      end
+    end
+  end
+
+  context "using ssl encrypted connection" do
+    context "with a valid certificate" do
+      it "successfully connect to the server" do
+        expect { 
+          Lumberjack::Client.new(:port => port,
+                                 :host => host,
+                                 :addresses => host,
+                                 :ssl_certificate => certificate_file_crt)
+        }.not_to raise_error
+      end
+
+      it "should fail connecting to plain tcp server" do
+        expect { 
+          Lumberjack::Client.new(:port => tcp_port,
+                                 :host => host,
+                                 :addresses => host,
+                                 :ssl_certificate => certificate_file_crt)
+        }.to raise_error(OpenSSL::SSL::SSLError)
+      end
+    end
+
+    context "with an invalid certificate" do
+      let(:invalid_certificate) { Flores::PKI.generate }
+      let(:invalid_certificate_file) { "invalid.crt" }
+
+      before do
+        expect(File).to receive(:read).with(invalid_certificate_file) { invalid_certificate.first.to_s }
+      end
+
+      it "should refuse to connect" do
+        expect {
+          Lumberjack::Client.new(:port => port,
+                                 :host => host,
+                                 :addresses => host,
+                                 :ssl_certificate => invalid_certificate_file)
+
+        }.to raise_error(OpenSSL::SSL::SSLError, /certificate verify failed/)
+      end
+    end
+
+    context "When transmitting a payload" do
+      let(:client) do
+        Lumberjack::Client.new(:port => port,
+                               :host => host,
+                               :addresses => host,
+                               :ssl_certificate => certificate_file_crt)
+      end
+
+      context "json" do
+        let(:options) { super.merge({ :json => true }) }
+        include_examples "transmit payloads"
+      end
+
+      context "v1 frame" do
+        include_examples "transmit payloads"
+      end
+    end
+  end
 end

data/spec/lumberjack/acking_protocol_v1_spec.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require "lumberjack/server"
+require "flores/random"
+
+describe Lumberjack::AckingProtocolV1 do
+  let(:number_of_events) { Flores::Random.integer(100..1024) }
+
+  subject { Lumberjack::AckingProtocolV1.new(number_of_events) }
+
+  it "should return true only once" do
+    results = []
+    number_of_events.times { |n| results << subject.ack?(n) }
+    expect(results.size).to eq(number_of_events)
+    expect(results.count(true)).to eq(1)
+  end
+end

data/spec/lumberjack/acking_protocol_v2_spec.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+require "lumberjack/server"
+require "flores/random"
+
+describe Lumberjack::AckingProtocolV2 do
+  let(:results) { [] }
+  subject { Lumberjack::AckingProtocolV2.new(number_of_events) }
+  before { 1.upto(number_of_events) { |n| results << subject.ack?(n) } }
+
+  context "with multiples events" do
+    let(:number_of_events) { Flores::Random.integer(100..1024) }
+
+    it "should return multiples partial acks" do
+      expect(results.size).to eq(number_of_events)
+      expect(results.count(true)).to eq((number_of_events / number_of_events * Lumberjack::AckingProtocolV2::ACK_RATIO).ceil + 1)
+    end
+
+    it "last ack should be true" do
+      expect(results.last).to be_truthy
+    end
+  end
+
+  context "with only one event" do
+    let(:number_of_events) { 1 }
+
+    it "should return true only once" do
+      expect(results.size).to eq(number_of_events)
+      expect(results.count(true)).to eq(1)
+    end
+  end
+end

data/spec/lumberjack/client_spec.rb

@@ -8,9 +8,7 @@ require "openssl"
 require "zlib"
 
 describe "Lumberjack::Client" do
-
   describe "Lumberjack::Socket" do
-
     let(:port)   { 5000 }
 
     subject(:socket) { Lumberjack::Socket.new(:port => port, :ssl_certificate => "" ) }
@@ -38,16 +36,60 @@ describe "Lumberjack::Client" do
     end
   end
 
-  describe Lumberjack::Encoder do
+  describe Lumberjack::FrameEncoder do
+    it 'should creates frames without truncating accentued characters' do
+      content = {
+        "message" => "Le Canadien de Montréal est la meilleure équipe au monde!",
+        "other" => "éléphant"
+      }
+      parser = Lumberjack::Parser.new
+      parser.feed(Lumberjack::FrameEncoder.to_frame(content, 0)) do |code, sequence, data|
+        if code == :data
+          expect(data["message"].force_encoding('UTF-8')).to eq(content["message"])
+          expect(data["other"].force_encoding('UTF-8')).to eq(content["other"])
+        end
+      end
+    end
+
+    it 'should creates frames without dropping multibytes characters' do
+      content = {
+        "message" => "国際ホッケー連盟" # International Hockey Federation
+      }
+      parser = Lumberjack::Parser.new
+      parser.feed(Lumberjack::FrameEncoder.to_frame(content, 0)) do |code, sequence, data|
+        expect(data["message"].force_encoding('UTF-8')).to eq(content["message"]) if code == :data
+      end
+    end
+  end
+
+  describe Lumberjack::JsonEncoder do
+    it 'should create frames from nested hash' do
+      content = {
+        "number" => 1,
+        "string" => "hello world",
+        "array" => [1,2,3],
+        "sub" => {
+          "a" => 1
+        }
+      }
+      parser = Lumberjack::Parser.new
+      frame = Lumberjack::JsonEncoder.to_frame(content, 0)
+      parser.feed(frame) do |code, sequence, data|
+        expect(data).to eq(content) if code == :json
+      end
+    end
+
     it 'should creates frames without truncating accentued characters' do
       content = {
         "message" => "Le Canadien de Montréal est la meilleure équipe au monde!",
         "other" => "éléphant"
       }
       parser = Lumberjack::Parser.new
-      parser.feed(Lumberjack::Encoder.to_frame(content, 0)) do |code, sequence, data|
-        expect(data["message"].force_encoding('UTF-8')).to eq(content["message"])
-        expect(data["other"].force_encoding('UTF-8')).to eq(content["other"])
+      parser.feed(Lumberjack::JsonEncoder.to_frame(content, 0)) do |code, sequence, data|
+        if code == :json
+          expect(data["message"]).to eq(content["message"])
+          expect(data["other"]).to eq(content["other"])
+        end
       end
     end
 
@@ -56,8 +98,8 @@ describe "Lumberjack::Client" do
         "message" => "国際ホッケー連盟" # International Hockey Federation
       }
       parser = Lumberjack::Parser.new
-      parser.feed(Lumberjack::Encoder.to_frame(content, 0)) do |code, sequence, data|
-        expect(data["message"].force_encoding('UTF-8')).to eq(content["message"])
+      parser.feed(Lumberjack::JsonEncoder.to_frame(content, 0)) do |code, sequence, data|
+        expect(data["message"]).to eq(content["message"]) if code == :json
       end
     end
   end

data/spec/lumberjack/connection_spec.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require "lumberjack/server"
+require "spec_helper"
+require "flores/random"
+
+describe "Connnection" do
+  let(:server) { double("server", :closed? => false) }
+  let(:socket) { double("socket", :closed? => false) }
+  let(:connection) { Lumberjack::Connection.new(socket, server) }
+  let(:payload) { {"line" => "foobar" } }
+  let(:start_sequence) { Flores::Random.integer(0..2000) }
+  let(:random_number_of_events) { Flores::Random.integer(2..200) }
+
+  context "when the server is running" do
+    before do
+      expect(socket).to receive(:sysread).at_least(:once).with(Lumberjack::Connection::READ_SIZE).and_return("")
+      allow(socket).to receive(:syswrite).with(anything).and_return(true)
+      allow(socket).to receive(:close)
+
+
+      expectation = receive(:feed)
+        .with("")
+        .and_yield(:version, Lumberjack::Parser::PROTOCOL_VERSION_1)
+        .and_yield(:window_size, random_number_of_events)
+
+      random_number_of_events.times { |n| expectation.and_yield(:data, start_sequence + n + 1, payload) }
+
+      expect_any_instance_of(Lumberjack::Parser).to expectation
+    end
+
+    it "should ack the end of a sequence" do
+      expect(socket).to receive(:syswrite).with(["1A", random_number_of_events + start_sequence].pack("A*N"))
+      connection.read_socket
+    end
+  end
+
+  context "when the server stop" do
+    let(:server) { double("server", :closed? => true) }
+    before do
+      expect(socket).to receive(:close).and_return(true)
+    end
+
+    it "stop reading from the socket" do
+      expect { |b| connection.run(&b) }.not_to yield_control
+    end
+  end
+end

data/spec/lumberjack/server_spec.rb

@@ -1,33 +1,42 @@
 # encoding: utf-8
+require "lumberjack/client"
 require "lumberjack/server"
-require "spec_helper"
 require "flores/random"
+require "flores/pki"
+require "spec_helper"
+
+Thread.abort_on_exception = true
 
 describe "Server" do
-  let(:socket) { double("socket") }
-  let(:connection) { Lumberjack::Connection.new(socket) }
-  let(:payload) { {"line" => "foobar" } }
-  let(:start_sequence) { Flores::Random.integer(0..2000) }
-  let(:random_number_of_events) { Flores::Random.integer(2..200) }
+  let(:certificate) { Flores::PKI.generate }
+  let(:certificate_file_crt) { "certificate.crt" }
+  let(:certificate_file_key) { "certificate.key" }
+  let(:port) { Flores::Random.integer(1024..65335) }
+  let(:tcp_port) { port + 1 }
+  let(:host) { "127.0.0.1" }
+  let(:queue) { [] }
 
   before do
-    expect(socket).to receive(:sysread).at_least(:once).with(Lumberjack::Connection::READ_SIZE).and_return("")
-    allow(socket).to receive(:syswrite).with(anything).and_return(true)
-    allow(socket).to receive(:close)
-
-    expectation = receive(:feed)
-      .with("")
-      .and_yield(:window_size, random_number_of_events)
-
-    random_number_of_events.times { |n| expectation.and_yield(:data, start_sequence + n + 1, payload) }
+    expect(File).to receive(:read).at_least(1).with(certificate_file_crt) { certificate.first.to_s }
+    expect(File).to receive(:read).at_least(1).with(certificate_file_key) { certificate.last.to_s }
+  end
 
-    expect_any_instance_of(Lumberjack::Parser).to expectation
+  subject do
+    Lumberjack::Server.new(:port => port,
+                           :address => host,
+                           :ssl_certificate => certificate_file_crt,
+                           :ssl_key => certificate_file_key)
   end
 
-  describe "Connnection" do
-    it "should ack the end of a sequence" do
-      expect(socket).to receive(:syswrite).with(["1A", random_number_of_events + start_sequence].pack("A*N"))
-      connection.read_socket
+  it "should not block when closing the server" do
+    thread = Thread.new do
+      subject.run do |event|
+        queue << event
+      end
     end
+
+    sleep(1) while thread.status != "run"
+    subject.close
+    wait_for { thread.status }.to be_falsey
   end
 end

data/spec/spec_helper.rb

@@ -1,6 +1,8 @@
 # encoding: utf-8
 require 'rspec'
 require 'rspec/mocks'
+require 'rspec/wait'
+
 $: << File.realpath(File.join(File.dirname(__FILE__), "..", "lib"))
 
 RSpec.configure do |config|

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: jls-lumberjack
 version: !ruby/object:Gem::Version
-  version: 0.0.24
+  version: 0.0.25.beta1
 platform: ruby
 authors:
 - Jordan Sissel
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-17 00:00:00.000000000 Z
+date: 2015-10-17 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: concurrent-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -66,6 +80,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec-wait
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: lumberjack log transport library
 email:
 - jls@semicomplete.com
@@ -77,7 +105,10 @@ files:
 - lib/lumberjack/client.rb
 - lib/lumberjack/server.rb
 - spec/integration_spec.rb
+- spec/lumberjack/acking_protocol_v1_spec.rb
+- spec/lumberjack/acking_protocol_v2_spec.rb
 - spec/lumberjack/client_spec.rb
+- spec/lumberjack/connection_spec.rb
 - spec/lumberjack/server_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/jordansissel/lumberjack
@@ -94,9 +125,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.4.8
@@ -106,5 +137,8 @@ summary: lumberjack log transport library
 test_files:
 - spec/integration_spec.rb
 - spec/spec_helper.rb
+- spec/lumberjack/acking_protocol_v1_spec.rb
+- spec/lumberjack/acking_protocol_v2_spec.rb
 - spec/lumberjack/client_spec.rb
+- spec/lumberjack/connection_spec.rb
 - spec/lumberjack/server_spec.rb