jls-lumberjack 0.0.23 → 0.0.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5ecf8e491d03fcc59fb32625cd5ad69018071f89
-  data.tar.gz: 3fc5d05089e0b192d736fb046854635757bf10e6
+  metadata.gz: d58912ce6afda9045d016510618a04363349628d
+  data.tar.gz: faeaba187853930185d78673f4b69c16c5b107bb
 SHA512:
-  metadata.gz: 4d6cc57fc88054f98a72bf6ccb51bdd2d91b01d6e1ba4232ac7bda25d3cf21acdb07e3940b6c0938ac6f2807f8701404819b30cbe3cfaf9edcd6d7cf919fffe6
-  data.tar.gz: 79946f5b1c4237e14e13a44a9d257635722469f57832a8ced321e45606236279c2b380d57c18ed8d9a18c13fa5306fb044c150a26ff46c74da5dc10d11d1b33b
+  metadata.gz: e40270de1f12efc47375be7a5a9cd9c83fae6ce2b615fe50885037746ecf074e1a26bf2dd4b2bf8a2ebdd1816c95f6d36b56e84de32e3fc5f73a225be7bc6aa7
+  data.tar.gz: 6224d5a7b5475195b0afd5168eea3c37a8c704cef432cf90b01cb8fcd520ea3293f1503e89b6bd4e0eaf2e84f24afeab78b7681ab9be4999e2720b2987291dd8

data/lib/lumberjack.rb

@@ -0,0 +1,3 @@
+module Lumberjack
+  SEQUENCE_MAX = (2**32-1).freeze
+end

data/lib/lumberjack/client.rb

@@ -1,20 +1,17 @@
 # encoding: utf-8
+require "lumberjack"
 require "socket"
 require "thread"
 require "openssl"
 require "zlib"
 
 module Lumberjack
-
-  SEQUENCE_MAX = (2**32-1).freeze
-
   class Client
     def initialize(opts={})
       @opts = {
         :port => 0,
         :addresses => [],
         :ssl_certificate => nil,
-        :window_size => 5000
       }.merge(opts)
 
       @opts[:addresses] = [@opts[:addresses]] if @opts[:addresses].class == String
@@ -40,8 +37,8 @@ module Lumberjack
     end
 
     public
-    def write(hash)
-      @socket.write_hash(hash)
+    def write(elements)
+      @socket.write_sync(elements)
     end
 
     public
@@ -51,7 +48,6 @@ module Lumberjack
   end
 
   class Socket
-
     # Create a new Lumberjack Socket.
     #
     # - options is a hash. Valid options are:
@@ -60,7 +56,6 @@ module Lumberjack
     # * :address - the host/address to bind to
     # * :ssl_certificate - the path to the ssl cert to use
     attr_reader :sequence
-    attr_reader :window_size
     attr_reader :host
     def initialize(opts={})
       @sequence = 0
@@ -69,10 +64,8 @@ module Lumberjack
         :port => 0,
         :address => "127.0.0.1",
         :ssl_certificate => nil,
-        :window_size => 5000
       }.merge(opts)
       @host = @opts[:address]
-      @window_size = @opts[:window_size]
 
       connection_start(opts)
     end
@@ -92,7 +85,6 @@ module Lumberjack
 
       @socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, ssl_context)
       @socket.connect
-      @socket.syswrite(["1", "W", @window_size].pack("AAN"))
     end
 
     private 
@@ -102,9 +94,12 @@ module Lumberjack
     end
 
     private
-    def write(msg)
-      compress = Zlib::Deflate.deflate(msg)
-      payload = ["1","C",compress.length,compress].pack("AANA#{compress.length}")
+    def send_window_size(size)
+      @socket.syswrite(["1", "W", size].pack("AAN"))
+    end
+
+    private
+    def send_payload(payload)
       # SSLSocket has a limit of 16k per message
       # execute multiple writes if needed
       bytes_written = 0
@@ -114,18 +109,28 @@ module Lumberjack
     end
 
     public
-    def write_hash(hash)
-      frame = Encoder.to_compressed_frame(hash, inc)
-      ack if unacked_sequence_size >= @window_size
-      write frame
+    def write_sync(elements)
+      elements = [elements] if elements.is_a?(Hash)
+      send_window_size(elements.size)
+
+      payload = elements.map { |element| Encoder.to_frame(element, inc) }.join
+      compress = compress_payload(payload)
+      send_payload(compress)
+
+      ack(elements.size)
+    end
+
+    private 
+    def compress_payload(payload)
+      compress = Zlib::Deflate.deflate(payload)
+      ["1", "C", compress.bytesize, compress].pack("AANA*")
     end
 
     private
-    def ack
+    def ack(size)
       _, type = read_version_and_type
       raise "Whoa we shouldn't get this frame: #{type}" if type != "A"
       @last_ack = read_last_ack
-      ack if unacked_sequence_size >= @window_size
     end
 
     private
@@ -139,6 +144,7 @@ module Lumberjack
       type    = @socket.read(1)
       [version, type]
     end
+
     private
     def read_last_ack
       @socket.read(4).unpack("N").first
@@ -146,11 +152,6 @@ module Lumberjack
   end
 
   module Encoder
-    def self.to_compressed_frame(hash, sequence)
-      compress = Zlib::Deflate.deflate(to_frame(hash, sequence))
-      ["1", "C", compress.bytesize, compress].pack("AANA#{compress.length}")
-    end
-
     def self.to_frame(hash, sequence)
       frame = ["1", "D", sequence]
       pack = "AAN"

data/lib/lumberjack/server.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require "lumberjack"
 require "socket"
 require "thread"
 require "openssl"
@@ -207,11 +208,12 @@ module Lumberjack
   end # class Parser
 
   class Connection
+    READ_SIZE = 16384
+
     def initialize(fd)
       super()
       @parser = Parser.new
       @fd = fd
-      @last_ack = 0
 
       # a safe default until we are told by the client what window size to use
       @window_size = 1 
@@ -219,19 +221,8 @@ module Lumberjack
 
     def run(&block)
       while true
-        # TODO(sissel): Ack on idle.
-        # X: - if any unacked, IO.select
-        # X:   - on timeout, ack all.
-        # X: Doing so will prevent slow streams from retransmitting
-        # X: too many events after errors.
-        @parser.feed(@fd.sysread(16384)) do |event, *args|
-          case event
-          when :window_size; window_size(*args, &block)
-          when :data; data(*args, &block)
-          end
-          #send(event, *args)
-        end # feed
-      end # while true
+        read_socket(&block)
+      end
     rescue EOFError, OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET
       # EOF or other read errors, only action is to shutdown which we'll do in
       # 'ensure'
@@ -239,6 +230,24 @@ module Lumberjack
       close rescue 'Already closed stream'
     end # def run
 
+    def read_socket(&block)
+      # TODO(sissel): Ack on idle.
+      # X: - if any unacked, IO.select
+      # X:   - on timeout, ack all.
+      # X: Doing so will prevent slow streams from retransmitting
+      # X: too many events after errors.
+      @parser.feed(@fd.sysread(READ_SIZE)) do |event, *args|
+        case event
+        when :window_size
+          # We receive a new payload
+          window_size(*args)
+          reset_next_ack
+        when :data
+          data(*args, &block)
+        end
+      end
+    end
+
     def close
       @fd.close
     end
@@ -247,13 +256,24 @@ module Lumberjack
       @window_size = size
     end
 
+    def reset_next_ack
+      @next_ack = nil
+    end
+
     def data(sequence, map, &block)
       block.call(map) if block_given?
-      if (sequence - @last_ack) >= @window_size
-        @fd.syswrite(["1A", sequence].pack("A*N"))
-        @last_ack = sequence
-      end
+      ack_if_needed(sequence)
+    end
+    
+    def compute_next_ack(sequence)
+      (sequence + @window_size - 1) % SEQUENCE_MAX
     end
-  end # class Connection
 
+    def ack_if_needed(sequence)
+      # The first encoded event will contain the sequence number 
+      # this is needed to know when we should ack.
+      @next_ack = compute_next_ack(sequence) if @next_ack.nil?
+      @fd.syswrite(["1A", sequence].pack("A*N")) if sequence == @next_ack
+    end
+  end # class Connection
 end # module Lumberjack

data/spec/integration_spec.rb

@@ -7,12 +7,15 @@ require "fileutils"
 require "thread"
 require "spec_helper"
 
+Thread.abort_on_exception = true
+
 describe "A client" do
   let(:certificate) { Flores::PKI.generate }
   let(:certificate_file_crt) { "certificate.crt" }
   let(:certificate_file_key) { "certificate.key" }
   let(:port) { Flores::Random.integer(1024..65335) }
   let(:host) { "127.0.0.1" }
+  let(:queue) { [] }
 
   before do
     expect(File).to receive(:read).at_least(1).with(certificate_file_crt) { certificate.first.to_s }
@@ -23,8 +26,8 @@ describe "A client" do
                                     :ssl_certificate => certificate_file_crt,
                                     :ssl_key => certificate_file_key)
 
-    Thread.new do
-      server.run { |data| }
+    @server = Thread.new do
+      server.run { |data| queue << data }
     end
   end
 
@@ -58,4 +61,73 @@ describe "A client" do
       }.to raise_error(OpenSSL::SSL::SSLError, /certificate verify failed/)
     end
   end
+  
+  shared_examples "send payload" do
+    it "supports single element" do
+      (1..random_number_of_events).each do |n|
+        expect(client.write(payload)).to eq(sequence_start + n)
+      end
+      sleep(0.5) # give time to the server to read the events
+      expect(queue.size).to eq(random_number_of_events)
+    end
+
+    it "support sending multiple elements in one payload" do
+      expect(client.write(batch_payload)).to eq(sequence_start + batch_size)
+      sleep(0.5)
+
+      expect(queue.size).to eq(batch_size)
+      expect(queue).to match_array(batch_payload)
+    end
+  end
+
+  context "When transmitting a payload" do
+    let(:random_number_of_events) { Flores::Random.integer(2..10) }
+    let(:payload) { { "line" => "foobar" } }
+    let(:client) do
+      Lumberjack::Client.new(:port => port, 
+                             :host => host,
+                             :addresses => host,
+                             :ssl_certificate => certificate_file_crt)
+    end
+    let(:batch_size) { Flores::Random.integer(1..1024) }
+    let(:batch_payload) do
+      batch = []
+      batch_size.times do |n|
+        batch <<  { "line" => "foobar #{n}" }
+      end
+      batch
+    end
+
+    context "when sequence start at 0" do
+      let(:sequence_start) { 0 }
+
+      include_examples "send payload"
+    end
+
+    context "when sequence doesn't start at zero" do
+      let(:sequence_start) { Flores::Random.integer(1..2000) }
+
+      before do
+        client.instance_variable_get(:@socket).instance_variable_set(:@sequence, sequence_start)
+      end
+
+      include_examples "send payload"
+    end
+
+    context "when the sequence rollover" do
+      let(:batch_size) { 100 }
+      let(:sequence_start) { Lumberjack::SEQUENCE_MAX - batch_size / 2 }
+
+      before do
+        client.instance_variable_get(:@socket).instance_variable_set(:@sequence, sequence_start)
+      end
+
+      it "adjusts the ack" do
+        expect(client.write(batch_payload)).to eq(batch_size / 2)
+        sleep(0.5)
+        expect(queue.size).to eq(batch_size)
+        expect(queue).to match_array(batch_payload)
+      end
+    end
+  end
 end

data/spec/lumberjack/client_spec.rb

@@ -17,49 +17,25 @@ describe "Lumberjack::Client" do
 
     before do
       allow_any_instance_of(Lumberjack::Socket).to receive(:connection_start).and_return(true)
+      # mock any network call
+      allow(socket).to receive(:send_window_size).with(kind_of(Integer)).and_return(true)
+      allow(socket).to receive(:send_payload).with(kind_of(String)).and_return(true)
     end
 
     context "sequence" do
-
      let(:hash)   { {:a => 1, :b => 2}}
      let(:max_unsigned_int) { (2**32)-1 }
 
       before(:each) do
         allow(socket).to receive(:ack).and_return(true)
-        allow(socket).to receive(:write).and_return(true)
       end
 
       it "force sequence to be an unsigned 32 bits int" do
         socket.instance_variable_set(:@sequence, max_unsigned_int)
-        socket.write_hash(hash)
+        socket.write_sync(hash)
         expect(socket.sequence).to eq(1)
       end
     end
-
-    context "ack" do
-
-      let(:hash)   { {:a => 1, :b => 2}}
-
-      before(:each) do
-        allow(socket).to receive(:write).and_return(true)
-      end
-
-      it "increments the sequence per windows size" do
-        allow(socket).to receive(:read_version_and_type).and_return([1, 'A'])
-        expect(socket).to receive(:ack).twice.and_call_original
-
-        [5000, 10000].each do |last_ack|
-          windows_size = 5001
-
-          allow(socket).to receive(:read_last_ack).and_return(last_ack)
-
-          windows_size.times do
-            socket.write_hash(hash)
-          end
-        end
-
-      end
-    end
   end
 
   describe Lumberjack::Encoder do
@@ -84,15 +60,5 @@ describe "Lumberjack::Client" do
         expect(data["message"].force_encoding('UTF-8')).to eq(content["message"])
       end
     end
-
-    it 'should creates compressed frames' do
-      content = {
-        "message" => "国際ホッケー連盟" # International Hockey Federation
-      }
-      parser = Lumberjack::Parser.new
-      parser.feed(Lumberjack::Encoder.to_compressed_frame(content, 0)) do |code, sequence, data|
-        expect(data["message"].force_encoding('UTF-8')).to eq(content["message"])
-      end
-    end
   end
 end

data/spec/lumberjack/server_spec.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+require "lumberjack/server"
+require "spec_helper"
+require "flores/random"
+
+describe "Server" do
+  let(:socket) { double("socket") }
+  let(:connection) { Lumberjack::Connection.new(socket) }
+  let(:payload) { {"line" => "foobar" } }
+  let(:start_sequence) { Flores::Random.integer(0..2000) }
+  let(:random_number_of_events) { Flores::Random.integer(2..200) }
+
+  before do
+    expect(socket).to receive(:sysread).at_least(:once).with(Lumberjack::Connection::READ_SIZE).and_return("")
+    allow(socket).to receive(:syswrite).with(anything).and_return(true)
+    allow(socket).to receive(:close)
+
+    expectation = receive(:feed)
+      .with("")
+      .and_yield(:window_size, random_number_of_events)
+
+    random_number_of_events.times { |n| expectation.and_yield(:data, start_sequence + n + 1, payload) }
+
+    expect_any_instance_of(Lumberjack::Parser).to expectation
+  end
+
+  describe "Connnection" do
+    it "should ack the end of a sequence" do
+      expect(socket).to receive(:syswrite).with(["1A", random_number_of_events + start_sequence].pack("A*N"))
+      connection.read_socket
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -2,3 +2,7 @@
 require 'rspec'
 require 'rspec/mocks'
 $: << File.realpath(File.join(File.dirname(__FILE__), "..", "lib"))
+
+RSpec.configure do |config|
+  config.order = :rand
+end

metadata

@@ -1,55 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: jls-lumberjack
 version: !ruby/object:Gem::Version
-  version: 0.0.23
+  version: 0.0.24
 platform: ruby
 authors:
 - Jordan Sissel
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-31 00:00:00.000000000 Z
+date: 2015-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: flores
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.6
-  type: :development
+  name: flores
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.6
 - !ruby/object:Gem::Dependency
-  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: rspec
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: stud
+  prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: pry
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: lumberjack log transport library
@@ -59,35 +73,38 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/lumberjack.rb
 - lib/lumberjack/client.rb
 - lib/lumberjack/server.rb
 - spec/integration_spec.rb
 - spec/lumberjack/client_spec.rb
+- spec/lumberjack/server_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/jordansissel/lumberjack
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
 specification_version: 4
 summary: lumberjack log transport library
 test_files:
 - spec/integration_spec.rb
-- spec/lumberjack/client_spec.rb
 - spec/spec_helper.rb
+- spec/lumberjack/client_spec.rb
+- spec/lumberjack/server_spec.rb