jls-lumberjack 0.0.25 → 0.0.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55126155d83d8600b7bdb19be71a4c1ea995ca9f
-  data.tar.gz: c4ec4e35094641991017a659963291587225c1e3
+  metadata.gz: d0a7b199bd230caef5c79839f655b115bdd0dece
+  data.tar.gz: b2ff3afdc5164f2cfc14516d999f75f33b44eec8
 SHA512:
-  metadata.gz: e9d2d85c73ca79c218b7d951095bf49a3962fa43e992f3ba4d667ef7a1ba7bbaecba4841706eba2de5db0c3ebbffa72e13181aecacb8509dc58bc67dfd2fa28b
-  data.tar.gz: ea44f28977d8eb96c78c68b913a1c61b37d1e936ad190a7b07d947b6f2bc9f131f15491a0705f0745e718fb77733157d760537ff362634d89e196093bfa56d6e
+  metadata.gz: 7ce044107a0d0f981186aa8b95d5938e6caf9d1ef4f108f08c5af6c65cb505763eff1eefff4987be950c5fadaa067ef38eb5e70adce22f1b524ac922b42bc09f
+  data.tar.gz: 9562ba9687ef1aafa71514aaf028d5d66206eff57a9020e6f28441954c36837431b27a010f5883ea25ae0e8276f30e070d889e39f05145e273aba118ea4fff45

data/lib/lumberjack.rb

@@ -1,3 +1,22 @@
+require "json"
+
 module Lumberjack
   SEQUENCE_MAX = (2**32-1).freeze
+
+  @@json = Class.new do
+    def self.load(blob)
+      JSON.parse(blob)
+    end
+    def self.dump(v)
+      v.to_json
+    end
+  end
+
+  def self.json
+    @@json
+  end
+
+  def self.json=(j)
+    @@json = j
+  end
 end

data/lib/lumberjack/client.rb

@@ -13,6 +13,7 @@ module Lumberjack
         :addresses => [],
         :ssl_certificate => nil,
         :ssl => true,
+        :json => false,
       }.merge(opts)
 
       @opts[:addresses] = [@opts[:addresses]] if @opts[:addresses].class == String
@@ -21,7 +22,6 @@ module Lumberjack
       raise "Must set a ssl certificate or path" if @opts[:ssl_certificate].nil? && @opts[:ssl]
 
       @socket = connect
-
     end
 
     private
@@ -38,8 +38,8 @@ module Lumberjack
     end
 
     public
-    def write(elements)
-      @socket.write_sync(elements)
+    def write(elements, opts={})
+      @socket.write_sync(elements, opts)
     end
 
     public
@@ -69,6 +69,7 @@ module Lumberjack
         :address => "127.0.0.1",
         :ssl_certificate => nil,
         :ssl => true,
+        :json => false,
       }.merge(opts)
       @host = @opts[:address]
 
@@ -117,11 +118,16 @@ module Lumberjack
     end
 
     public
-    def write_sync(elements)
+    def write_sync(elements, opts={})
+      options = {
+        :json => @opts[:json],
+      }.merge(opts)
+
       elements = [elements] if elements.is_a?(Hash)
       send_window_size(elements.size)
 
-      payload = elements.map { |element| Encoder.to_frame(element, inc) }.join
+      encoder = options[:json] ? JsonEncoder : FrameEncoder
+      payload = elements.map { |element| encoder.to_frame(element, inc) }.join
       compress = compress_payload(payload)
       send_payload(compress)
 
@@ -159,7 +165,17 @@ module Lumberjack
     end
   end
 
-  module Encoder
+  module JsonEncoder
+    def self.to_frame(hash, sequence)
+      json = Lumberjack::json.dump(hash)
+      json_length = json.bytesize
+      pack = "AANNA#{json_length}"
+      frame = ["1", "J", sequence, json_length, json]
+      frame.pack(pack)
+    end
+  end # JsonEncoder
+
+  module FrameEncoder
     def self.to_frame(hash, sequence)
       frame = ["1", "D", sequence]
       pack = "AAN"

data/lib/lumberjack/server.rb

@@ -4,6 +4,7 @@ require "socket"
 require "thread"
 require "openssl"
 require "zlib"
+require "json"
 require "concurrent"
 
 module Lumberjack
@@ -65,7 +66,6 @@ module Lumberjack
         # unless the server is closing
         next unless connection
 
-
         Thread.new(connection) do |connection|
           connection.run(&block)
         end
@@ -94,7 +94,7 @@ module Lumberjack
         begin
           IO.select([@server], nil, nil, SOCKET_TIMEOUT)
           retry unless closed?
-        rescue IOError => e # we currently closing
+        rescue IOError, Errno::EBADF => e # we currently closing
           raise e unless closed?
         end
       end
@@ -128,6 +128,11 @@ module Lumberjack
   end # class Server
 
   class Parser
+    PROTOCOL_VERSION_1 = "1".ord
+    PROTOCOL_VERSION_2 = "2".ord
+
+    SUPPORTED_PROTOCOLS = [PROTOCOL_VERSION_1, PROTOCOL_VERSION_2]
+
     def initialize
       @buffer_offset = 0
       @buffer = ""
@@ -196,24 +201,52 @@ module Lumberjack
 
     FRAME_WINDOW = "W".ord
     FRAME_DATA = "D".ord
+    FRAME_JSON_DATA = "J".ord
     FRAME_COMPRESSED = "C".ord
     def header(&block)
       version, frame_type = get.bytes.to_a[0..1]
+      version ||= PROTOCOL_VERSION_1
+
+      handle_version(version, &block)
 
       case frame_type
       when FRAME_WINDOW; transition(:window_size, 4)
       when FRAME_DATA; transition(:data_lead, 8)
+      when FRAME_JSON_DATA; transition(:json_data_lead, 8)
       when FRAME_COMPRESSED; transition(:compressed_lead, 4)
-      else; raise "Unknown frame type: #{frame_type}"
+      else; raise "Unknown frame type: `#{frame_type}`"
       end
     end
 
+    def handle_version(version, &block)
+      if supported_protocol?(version)
+        yield :version, version
+      else
+        raise "unsupported protocol #{version}"
+      end
+    end
+
+    def supported_protocol?(version)
+      SUPPORTED_PROTOCOLS.include?(version)
+    end
+
     def window_size(&block)
       @window_size = get.unpack("N").first
       transition(:header, 2)
       yield :window_size, @window_size
     end # def window_size
 
+    def json_data_lead(&block)
+      @sequence, payload_size = get.unpack("NN")
+      transition(:json_data_payload, payload_size)
+    end
+
+    def json_data_payload(&block)
+      payload = get
+      yield :json, @sequence, Lumberjack::json.load(payload)
+      transition(:header, 2)
+    end
+
     def data_lead(&block)
       @sequence, @data_count = get.unpack("NN")
       @data = {}
@@ -274,8 +307,7 @@ module Lumberjack
       @fd = fd
 
       @server = server
-      # a safe default until we are told by the client what window size to use
-      @window_size = 1 
+      @ack_handler = nil
     end
 
     def run(&block)
@@ -285,6 +317,8 @@ module Lumberjack
     rescue EOFError, OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET
       # EOF or other read errors, only action is to shutdown which we'll do in
       # 'ensure'
+    rescue Errno::EAGAIN
+      retry unless server.closed?
     ensure
       close rescue 'Already closed stream'
     end # def run
@@ -297,42 +331,105 @@ module Lumberjack
       # X: too many events after errors.
       @parser.feed(@fd.sysread(READ_SIZE)) do |event, *args|
         case event
+        when :version
+          version(*args)
         when :window_size
-          # We receive a new payload
-          window_size(*args)
-          reset_next_ack
+          reset_next_ack(*args)
         when :data
-          data(*args, &block)
+          sequence, map = args
+          ack_if_needed(sequence) { data(map, &block) }
+        when :json
+          # If the payload is an array of items we will emit multiple events
+          # this behavior was moved from the plugin to the library.
+          # see this commit: https://github.com/logstash-plugins/logstash-input-lumberjack/pull/57/files#diff-1b9590423b15f04f215635164e7376ecR158
+          sequence, map = args
+
+          ack_if_needed(sequence) do
+            if map.is_a?(Array)
+              map.each { |e| data(e, &block) }
+            else
+              data(map, &block)
+            end
+          end
         end
       end
     end
 
+    def version(version)
+      @version = version
+    end
+
+    def ack_if_needed(sequence, &block)
+      block.call
+      send_ack(sequence) if @ack_handler.ack?(sequence)
+    end
+
     def close
       @fd.close unless @fd.closed?
     end
 
-    def window_size(size)
-      @window_size = size
+    def data(map, &block)
+      block.call(map) if block_given?
+    end
+
+    def reset_next_ack(window_size)
+      klass = (@version == Parser::PROTOCOL_VERSION_1) ? AckingProtocolV1 : AckingProtocolV2
+      @ack_handler = klass.new(window_size)
+    end
+
+    def send_ack(sequence)
+      @fd.syswrite(@ack_handler.ack_frame(sequence))
     end
+  end # class Connection
 
-    def reset_next_ack
+  class AckingProtocolV1
+    def initialize(window_size)
       @next_ack = nil
+      @window_size = window_size
     end
 
-    def data(sequence, map, &block)
-      block.call(map) if block_given?
-      ack_if_needed(sequence)
+    def ack?(sequence)
+      # The first encoded event will contain the sequence number 
+      # this is needed to know when we should ack.
+      @next_ack = compute_next_ack(sequence) if @next_ack.nil?
+      sequence == @next_ack
+    end
+
+    def ack_frame(sequence)
+      ["1A", sequence].pack("A*N")
     end
-    
+
+    private
     def compute_next_ack(sequence)
       (sequence + @window_size - 1) % SEQUENCE_MAX
     end
+  end
+
+  # Allow lumberjack to send partial ack back to the producer
+  # only V2 client support partial Acks
+  #
+  # Send Ack on every 20% of the data, so with default settings every 200 events
+  # This should reduce the congestion on retransmit.
+  class AckingProtocolV2
+    ACK_RATIO = 5
+
+    def initialize(window_size)
+      @window_size = window_size
+      @every = (window_size / ACK_RATIO).round
+    end
 
-    def ack_if_needed(sequence)
-      # The first encoded event will contain the sequence number 
-      # this is needed to know when we should ack.
-      @next_ack = compute_next_ack(sequence) if @next_ack.nil?
-      @fd.syswrite(["1A", sequence].pack("A*N")) if sequence == @next_ack
+    def ack?(sequence)
+      if @window_size == sequence
+        true
+      elsif sequence % @every == 0
+        true
+      else
+        false
+      end
     end
-  end # class Connection
+
+    def ack_frame(sequence)
+      ["2A", sequence].pack("A*N")
+    end
+  end
 end # module Lumberjack

data/spec/integration_spec.rb

@@ -7,6 +7,7 @@ require "fileutils"
 require "thread"
 require "spec_helper"
 
+Thread.abort_on_exception = true
 describe "A client" do
   let(:certificate) { Flores::PKI.generate }
   let(:certificate_file_crt) { "certificate.crt" }
@@ -134,10 +135,17 @@ describe "A client" do
     end
 
     context "When transmitting a payload" do
-      let(:client) do
-        Lumberjack::Client.new(:port => tcp_port, :host => host, :addresses => host, :ssl => false)
+      let(:options) { {:port => tcp_port, :host => host, :addresses => host, :ssl => false } }
+      let(:client) { Lumberjack::Client.new(options) }
+
+      context "json" do
+        let(:options) { super.merge({ :json => true }) }
+        include_examples "transmit payloads"
+      end
+
+      context "v1 frame" do
+        include_examples "transmit payloads"
       end
-      include_examples "transmit payloads"
     end
   end
 
@@ -188,7 +196,15 @@ describe "A client" do
                                :addresses => host,
                                :ssl_certificate => certificate_file_crt)
       end
-      include_examples "transmit payloads"
+
+      context "json" do
+        let(:options) { super.merge({ :json => true }) }
+        include_examples "transmit payloads"
+      end
+
+      context "v1 frame" do
+        include_examples "transmit payloads"
+      end
     end
   end
 end

data/spec/lumberjack/acking_protocol_v1_spec.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require "lumberjack/server"
+require "flores/random"
+
+describe Lumberjack::AckingProtocolV1 do
+  let(:number_of_events) { Flores::Random.integer(100..1024) }
+
+  subject { Lumberjack::AckingProtocolV1.new(number_of_events) }
+
+  it "should return true only once" do
+    results = []
+    number_of_events.times { |n| results << subject.ack?(n) }
+    expect(results.size).to eq(number_of_events)
+    expect(results.count(true)).to eq(1)
+  end
+end

data/spec/lumberjack/acking_protocol_v2_spec.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+require "lumberjack/server"
+require "flores/random"
+
+describe Lumberjack::AckingProtocolV2 do
+  let(:results) { [] }
+  subject { Lumberjack::AckingProtocolV2.new(number_of_events) }
+  before { 1.upto(number_of_events) { |n| results << subject.ack?(n) } }
+
+  context "with multiples events" do
+    let(:number_of_events) { Flores::Random.integer(100..1024) }
+
+    it "should return multiples partial acks" do
+      expect(results.size).to eq(number_of_events)
+      expect(results.count(true)).to be_within(1).of((number_of_events / number_of_events * Lumberjack::AckingProtocolV2::ACK_RATIO).ceil)
+    end
+
+    it "last ack should be true" do
+      expect(results.last).to be_truthy
+    end
+  end
+
+  context "with only one event" do
+    let(:number_of_events) { 1 }
+
+    it "should return true only once" do
+      expect(results.size).to eq(number_of_events)
+      expect(results.count(true)).to eq(1)
+    end
+  end
+end

data/spec/lumberjack/client_spec.rb

@@ -8,9 +8,7 @@ require "openssl"
 require "zlib"
 
 describe "Lumberjack::Client" do
-
   describe "Lumberjack::Socket" do
-
     let(:port)   { 5000 }
 
     subject(:socket) { Lumberjack::Socket.new(:port => port, :ssl_certificate => "" ) }
@@ -38,16 +36,60 @@ describe "Lumberjack::Client" do
     end
   end
 
-  describe Lumberjack::Encoder do
+  describe Lumberjack::FrameEncoder do
+    it 'should creates frames without truncating accentued characters' do
+      content = {
+        "message" => "Le Canadien de Montréal est la meilleure équipe au monde!",
+        "other" => "éléphant"
+      }
+      parser = Lumberjack::Parser.new
+      parser.feed(Lumberjack::FrameEncoder.to_frame(content, 0)) do |code, sequence, data|
+        if code == :data
+          expect(data["message"].force_encoding('UTF-8')).to eq(content["message"])
+          expect(data["other"].force_encoding('UTF-8')).to eq(content["other"])
+        end
+      end
+    end
+
+    it 'should creates frames without dropping multibytes characters' do
+      content = {
+        "message" => "国際ホッケー連盟" # International Hockey Federation
+      }
+      parser = Lumberjack::Parser.new
+      parser.feed(Lumberjack::FrameEncoder.to_frame(content, 0)) do |code, sequence, data|
+        expect(data["message"].force_encoding('UTF-8')).to eq(content["message"]) if code == :data
+      end
+    end
+  end
+
+  describe Lumberjack::JsonEncoder do
+    it 'should create frames from nested hash' do
+      content = {
+        "number" => 1,
+        "string" => "hello world",
+        "array" => [1,2,3],
+        "sub" => {
+          "a" => 1
+        }
+      }
+      parser = Lumberjack::Parser.new
+      frame = Lumberjack::JsonEncoder.to_frame(content, 0)
+      parser.feed(frame) do |code, sequence, data|
+        expect(data).to eq(content) if code == :json
+      end
+    end
+
     it 'should creates frames without truncating accentued characters' do
       content = {
         "message" => "Le Canadien de Montréal est la meilleure équipe au monde!",
         "other" => "éléphant"
       }
       parser = Lumberjack::Parser.new
-      parser.feed(Lumberjack::Encoder.to_frame(content, 0)) do |code, sequence, data|
-        expect(data["message"].force_encoding('UTF-8')).to eq(content["message"])
-        expect(data["other"].force_encoding('UTF-8')).to eq(content["other"])
+      parser.feed(Lumberjack::JsonEncoder.to_frame(content, 0)) do |code, sequence, data|
+        if code == :json
+          expect(data["message"]).to eq(content["message"])
+          expect(data["other"]).to eq(content["other"])
+        end
       end
     end
 
@@ -56,8 +98,8 @@ describe "Lumberjack::Client" do
         "message" => "国際ホッケー連盟" # International Hockey Federation
       }
       parser = Lumberjack::Parser.new
-      parser.feed(Lumberjack::Encoder.to_frame(content, 0)) do |code, sequence, data|
-        expect(data["message"].force_encoding('UTF-8')).to eq(content["message"])
+      parser.feed(Lumberjack::JsonEncoder.to_frame(content, 0)) do |code, sequence, data|
+        expect(data["message"]).to eq(content["message"]) if code == :json
       end
     end
   end

data/spec/lumberjack/connection_spec.rb

@@ -17,8 +17,10 @@ describe "Connnection" do
       allow(socket).to receive(:syswrite).with(anything).and_return(true)
       allow(socket).to receive(:close)
 
+
       expectation = receive(:feed)
         .with("")
+        .and_yield(:version, Lumberjack::Parser::PROTOCOL_VERSION_1)
         .and_yield(:window_size, random_number_of_events)
 
       random_number_of_events.times { |n| expectation.and_yield(:data, start_sequence + n + 1, payload) }

metadata

@@ -1,97 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: jls-lumberjack
 version: !ruby/object:Gem::Version
-  version: 0.0.25
+  version: 0.0.26
 platform: ruby
 authors:
 - Jordan Sissel
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-12 00:00:00.000000000 Z
+date: 2015-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: concurrent-ruby
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: flores
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.6
-  type: :development
+  name: flores
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.6
 - !ruby/object:Gem::Dependency
-  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: rspec
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: stud
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: pry
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec-wait
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: rspec-wait
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: lumberjack log transport library
@@ -105,6 +105,8 @@ files:
 - lib/lumberjack/client.rb
 - lib/lumberjack/server.rb
 - spec/integration_spec.rb
+- spec/lumberjack/acking_protocol_v1_spec.rb
+- spec/lumberjack/acking_protocol_v2_spec.rb
 - spec/lumberjack/client_spec.rb
 - spec/lumberjack/connection_spec.rb
 - spec/lumberjack/server_spec.rb
@@ -112,29 +114,31 @@ files:
 homepage: https://github.com/jordansissel/lumberjack
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.6
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
 specification_version: 4
 summary: lumberjack log transport library
 test_files:
 - spec/integration_spec.rb
+- spec/spec_helper.rb
+- spec/lumberjack/acking_protocol_v1_spec.rb
+- spec/lumberjack/acking_protocol_v2_spec.rb
 - spec/lumberjack/client_spec.rb
 - spec/lumberjack/connection_spec.rb
 - spec/lumberjack/server_spec.rb
-- spec/spec_helper.rb