jls-lumberjack 0.0.25.beta1 → 0.0.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e2b2959f02e7f417cf30eb4d840508950edf0808
-  data.tar.gz: a1f37209be258dbcc97617b8eed7e5c61ca66db5
+  metadata.gz: 55126155d83d8600b7bdb19be71a4c1ea995ca9f
+  data.tar.gz: c4ec4e35094641991017a659963291587225c1e3
 SHA512:
-  metadata.gz: a87220c6b45a3c53f4bf685c2309c37cdd953c06d489f56cb442eb9ebc482357067b3fcad526b7e44c28455117be4c189226815184d65049bb6b429326ef5d42
-  data.tar.gz: 8d326476bf7fe3d76873fc998dae45fbaf94cbe776de1a9e276fe8d4fadcfe6243a3ac2db7c90568056998c8296dabe44424fc3b03dfb1a7f91f7d87a5df41ae
+  metadata.gz: e9d2d85c73ca79c218b7d951095bf49a3962fa43e992f3ba4d667ef7a1ba7bbaecba4841706eba2de5db0c3ebbffa72e13181aecacb8509dc58bc67dfd2fa28b
+  data.tar.gz: ea44f28977d8eb96c78c68b913a1c61b37d1e936ad190a7b07d947b6f2bc9f131f15491a0705f0745e718fb77733157d760537ff362634d89e196093bfa56d6e

data/lib/lumberjack.rb

@@ -1,22 +1,3 @@
-require "json"
-
 module Lumberjack
   SEQUENCE_MAX = (2**32-1).freeze
-
-  @@json = Class.new do
-    def self.load(blob)
-      JSON.parse(blob)
-    end
-    def self.dump(v)
-      v.to_json
-    end
-  end
-
-  def self.json
-    @@json
-  end
-
-  def self.json=(j)
-    @@json = j
-  end
 end

data/lib/lumberjack/client.rb

@@ -13,7 +13,6 @@ module Lumberjack
         :addresses => [],
         :ssl_certificate => nil,
         :ssl => true,
-        :json => false,
       }.merge(opts)
 
       @opts[:addresses] = [@opts[:addresses]] if @opts[:addresses].class == String
@@ -22,6 +21,7 @@ module Lumberjack
       raise "Must set a ssl certificate or path" if @opts[:ssl_certificate].nil? && @opts[:ssl]
 
       @socket = connect
+
     end
 
     private
@@ -38,8 +38,8 @@ module Lumberjack
     end
 
     public
-    def write(elements, opts={})
-      @socket.write_sync(elements, opts)
+    def write(elements)
+      @socket.write_sync(elements)
     end
 
     public
@@ -69,7 +69,6 @@ module Lumberjack
         :address => "127.0.0.1",
         :ssl_certificate => nil,
         :ssl => true,
-        :json => false,
       }.merge(opts)
       @host = @opts[:address]
 
@@ -118,16 +117,11 @@ module Lumberjack
     end
 
     public
-    def write_sync(elements, opts={})
-      options = {
-        :json => @opts[:json],
-      }.merge(opts)
-
+    def write_sync(elements)
       elements = [elements] if elements.is_a?(Hash)
       send_window_size(elements.size)
 
-      encoder = options[:json] ? JsonEncoder : FrameEncoder
-      payload = elements.map { |element| encoder.to_frame(element, inc) }.join
+      payload = elements.map { |element| Encoder.to_frame(element, inc) }.join
       compress = compress_payload(payload)
       send_payload(compress)
 
@@ -165,17 +159,7 @@ module Lumberjack
     end
   end
 
-  module JsonEncoder
-    def self.to_frame(hash, sequence)
-      json = Lumberjack::json.dump(hash)
-      json_length = json.bytesize
-      pack = "AANNA#{json_length}"
-      frame = ["1", "J", sequence, json_length, json]
-      frame.pack(pack)
-    end
-  end # JsonEncoder
-
-  module FrameEncoder
+  module Encoder
     def self.to_frame(hash, sequence)
       frame = ["1", "D", sequence]
       pack = "AAN"

data/lib/lumberjack/server.rb

@@ -4,7 +4,6 @@ require "socket"
 require "thread"
 require "openssl"
 require "zlib"
-require "json"
 require "concurrent"
 
 module Lumberjack
@@ -66,6 +65,7 @@ module Lumberjack
         # unless the server is closing
         next unless connection
 
+
         Thread.new(connection) do |connection|
           connection.run(&block)
         end
@@ -94,7 +94,7 @@ module Lumberjack
         begin
           IO.select([@server], nil, nil, SOCKET_TIMEOUT)
           retry unless closed?
-        rescue IOError, Errno::EBADF => e # we currently closing
+        rescue IOError => e # we currently closing
           raise e unless closed?
         end
       end
@@ -128,11 +128,6 @@ module Lumberjack
   end # class Server
 
   class Parser
-    PROTOCOL_VERSION_1 = "1".ord
-    PROTOCOL_VERSION_2 = "2".ord
-
-    SUPPORTED_PROTOCOLS = [PROTOCOL_VERSION_1, PROTOCOL_VERSION_2]
-
     def initialize
       @buffer_offset = 0
       @buffer = ""
@@ -201,52 +196,24 @@ module Lumberjack
 
     FRAME_WINDOW = "W".ord
     FRAME_DATA = "D".ord
-    FRAME_JSON_DATA = "J".ord
     FRAME_COMPRESSED = "C".ord
     def header(&block)
       version, frame_type = get.bytes.to_a[0..1]
-      version ||= PROTOCOL_VERSION_1
-
-      handle_version(version, &block)
 
       case frame_type
       when FRAME_WINDOW; transition(:window_size, 4)
       when FRAME_DATA; transition(:data_lead, 8)
-      when FRAME_JSON_DATA; transition(:json_data_lead, 8)
       when FRAME_COMPRESSED; transition(:compressed_lead, 4)
-      else; raise "Unknown frame type: `#{frame_type}`"
-      end
-    end
-
-    def handle_version(version, &block)
-      if supported_protocol?(version)
-        yield :version, version
-      else
-        raise "unsupported protocol #{version}"
+      else; raise "Unknown frame type: #{frame_type}"
       end
     end
 
-    def supported_protocol?(version)
-      SUPPORTED_PROTOCOLS.include?(version)
-    end
-
     def window_size(&block)
       @window_size = get.unpack("N").first
       transition(:header, 2)
       yield :window_size, @window_size
     end # def window_size
 
-    def json_data_lead(&block)
-      @sequence, payload_size = get.unpack("NN")
-      transition(:json_data_payload, payload_size)
-    end
-
-    def json_data_payload(&block)
-      payload = get
-      yield :json, @sequence, Lumberjack::json.load(payload)
-      transition(:header, 2)
-    end
-
     def data_lead(&block)
       @sequence, @data_count = get.unpack("NN")
       @data = {}
@@ -307,7 +274,8 @@ module Lumberjack
       @fd = fd
 
       @server = server
-      @ack_handler = nil
+      # a safe default until we are told by the client what window size to use
+      @window_size = 1 
     end
 
     def run(&block)
@@ -329,97 +297,42 @@ module Lumberjack
       # X: too many events after errors.
       @parser.feed(@fd.sysread(READ_SIZE)) do |event, *args|
         case event
-        when :version
-          version(*args)
         when :window_size
-          reset_next_ack(*args)
+          # We receive a new payload
+          window_size(*args)
+          reset_next_ack
         when :data
-          sequence, map = args
-          ack_if_needed(sequence) { data(map, &block) }
-        when :json
-          # If the payload is an array of items we will emit multiple events
-          # this behavior was moved from the plugin to the library.
-          # see this commit: https://github.com/logstash-plugins/logstash-input-lumberjack/pull/57/files#diff-1b9590423b15f04f215635164e7376ecR158
-          sequence, map = args
-
-          ack_if_needed(sequence) do
-            if map.is_a?(Array)
-              map.each { |e| data(e, &block) }
-            else
-              data(map, &block)
-            end
-          end
+          data(*args, &block)
         end
       end
     end
 
-    def version(version)
-      @version = version
-    end
-
-    def ack_if_needed(sequence, &block)
-      block.call
-      send_ack(sequence) if @ack_handler.ack?(sequence)
-    end
-
     def close
       @fd.close unless @fd.closed?
     end
 
-    def data(map, &block)
-      block.call(map) if block_given?
-    end
-
-    def reset_next_ack(window_size)
-      klass = (@version == Parser::PROTOCOL_VERSION_1) ? AckingProtocolV1 : AckingProtocolV2
-      @ack_handler = klass.new(window_size)
-    end
-
-    def send_ack(sequence)
-      @fd.syswrite(["1A", sequence].pack("A*N"))
+    def window_size(size)
+      @window_size = size
     end
-  end # class Connection
 
-  class AckingProtocolV1
-    def initialize(window_size)
+    def reset_next_ack
       @next_ack = nil
-      @window_size = window_size
     end
 
-    def ack?(sequence)
-      # The first encoded event will contain the sequence number 
-      # this is needed to know when we should ack.
-      @next_ack = compute_next_ack(sequence) if @next_ack.nil?
-      sequence == @next_ack
+    def data(sequence, map, &block)
+      block.call(map) if block_given?
+      ack_if_needed(sequence)
     end
-
-    private
+    
     def compute_next_ack(sequence)
       (sequence + @window_size - 1) % SEQUENCE_MAX
     end
-  end
-
-  # Allow lumberjack to send partial ack back to the producer
-  # only V2 client support partial Acks
-  #
-  # Send Ack on every 20% of the data, so with default settings every 200 events
-  # This should reduce the congestion on retransmit.
-  class AckingProtocolV2
-    ACK_RATIO = 5
-
-    def initialize(window_size)
-      @window_size = window_size
-      @every = (window_size / ACK_RATIO).round
-    end
 
-    def ack?(sequence)
-      if @window_size == sequence
-        true
-      elsif sequence % @every == 0
-        true
-      else
-        false
-      end
+    def ack_if_needed(sequence)
+      # The first encoded event will contain the sequence number 
+      # this is needed to know when we should ack.
+      @next_ack = compute_next_ack(sequence) if @next_ack.nil?
+      @fd.syswrite(["1A", sequence].pack("A*N")) if sequence == @next_ack
     end
-  end
+  end # class Connection
 end # module Lumberjack

data/spec/integration_spec.rb

@@ -7,7 +7,6 @@ require "fileutils"
 require "thread"
 require "spec_helper"
 
-Thread.abort_on_exception = true
 describe "A client" do
   let(:certificate) { Flores::PKI.generate }
   let(:certificate_file_crt) { "certificate.crt" }
@@ -135,17 +134,10 @@ describe "A client" do
     end
 
     context "When transmitting a payload" do
-      let(:options) { {:port => tcp_port, :host => host, :addresses => host, :ssl => false } }
-      let(:client) { Lumberjack::Client.new(options) }
-
-      context "json" do
-        let(:options) { super.merge({ :json => true }) }
-        include_examples "transmit payloads"
-      end
-
-      context "v1 frame" do
-        include_examples "transmit payloads"
+      let(:client) do
+        Lumberjack::Client.new(:port => tcp_port, :host => host, :addresses => host, :ssl => false)
       end
+      include_examples "transmit payloads"
     end
   end
 
@@ -196,15 +188,7 @@ describe "A client" do
                                :addresses => host,
                                :ssl_certificate => certificate_file_crt)
       end
-
-      context "json" do
-        let(:options) { super.merge({ :json => true }) }
-        include_examples "transmit payloads"
-      end
-
-      context "v1 frame" do
-        include_examples "transmit payloads"
-      end
+      include_examples "transmit payloads"
     end
   end
 end

data/spec/lumberjack/client_spec.rb

@@ -8,7 +8,9 @@ require "openssl"
 require "zlib"
 
 describe "Lumberjack::Client" do
+
   describe "Lumberjack::Socket" do
+
     let(:port)   { 5000 }
 
     subject(:socket) { Lumberjack::Socket.new(:port => port, :ssl_certificate => "" ) }
@@ -36,60 +38,16 @@ describe "Lumberjack::Client" do
     end
   end
 
-  describe Lumberjack::FrameEncoder do
-    it 'should creates frames without truncating accentued characters' do
-      content = {
-        "message" => "Le Canadien de Montréal est la meilleure équipe au monde!",
-        "other" => "éléphant"
-      }
-      parser = Lumberjack::Parser.new
-      parser.feed(Lumberjack::FrameEncoder.to_frame(content, 0)) do |code, sequence, data|
-        if code == :data
-          expect(data["message"].force_encoding('UTF-8')).to eq(content["message"])
-          expect(data["other"].force_encoding('UTF-8')).to eq(content["other"])
-        end
-      end
-    end
-
-    it 'should creates frames without dropping multibytes characters' do
-      content = {
-        "message" => "国際ホッケー連盟" # International Hockey Federation
-      }
-      parser = Lumberjack::Parser.new
-      parser.feed(Lumberjack::FrameEncoder.to_frame(content, 0)) do |code, sequence, data|
-        expect(data["message"].force_encoding('UTF-8')).to eq(content["message"]) if code == :data
-      end
-    end
-  end
-
-  describe Lumberjack::JsonEncoder do
-    it 'should create frames from nested hash' do
-      content = {
-        "number" => 1,
-        "string" => "hello world",
-        "array" => [1,2,3],
-        "sub" => {
-          "a" => 1
-        }
-      }
-      parser = Lumberjack::Parser.new
-      frame = Lumberjack::JsonEncoder.to_frame(content, 0)
-      parser.feed(frame) do |code, sequence, data|
-        expect(data).to eq(content) if code == :json
-      end
-    end
-
+  describe Lumberjack::Encoder do
     it 'should creates frames without truncating accentued characters' do
       content = {
         "message" => "Le Canadien de Montréal est la meilleure équipe au monde!",
         "other" => "éléphant"
       }
       parser = Lumberjack::Parser.new
-      parser.feed(Lumberjack::JsonEncoder.to_frame(content, 0)) do |code, sequence, data|
-        if code == :json
-          expect(data["message"]).to eq(content["message"])
-          expect(data["other"]).to eq(content["other"])
-        end
+      parser.feed(Lumberjack::Encoder.to_frame(content, 0)) do |code, sequence, data|
+        expect(data["message"].force_encoding('UTF-8')).to eq(content["message"])
+        expect(data["other"].force_encoding('UTF-8')).to eq(content["other"])
       end
     end
 
@@ -98,8 +56,8 @@ describe "Lumberjack::Client" do
         "message" => "国際ホッケー連盟" # International Hockey Federation
       }
       parser = Lumberjack::Parser.new
-      parser.feed(Lumberjack::JsonEncoder.to_frame(content, 0)) do |code, sequence, data|
-        expect(data["message"]).to eq(content["message"]) if code == :json
+      parser.feed(Lumberjack::Encoder.to_frame(content, 0)) do |code, sequence, data|
+        expect(data["message"].force_encoding('UTF-8')).to eq(content["message"])
       end
     end
   end

data/spec/lumberjack/connection_spec.rb

@@ -17,10 +17,8 @@ describe "Connnection" do
       allow(socket).to receive(:syswrite).with(anything).and_return(true)
       allow(socket).to receive(:close)
 
-
       expectation = receive(:feed)
         .with("")
-        .and_yield(:version, Lumberjack::Parser::PROTOCOL_VERSION_1)
         .and_yield(:window_size, random_number_of_events)
 
       random_number_of_events.times { |n| expectation.and_yield(:data, start_sequence + n + 1, payload) }

metadata

@@ -1,97 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: jls-lumberjack
 version: !ruby/object:Gem::Version
-  version: 0.0.25.beta1
+  version: 0.0.25
 platform: ruby
 authors:
 - Jordan Sissel
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-17 00:00:00.000000000 Z
+date: 2015-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: concurrent-ruby
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: flores
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.6
-  name: flores
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.6
 - !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: stud
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: pry
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rspec-wait
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec-wait
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: lumberjack log transport library
@@ -105,8 +105,6 @@ files:
 - lib/lumberjack/client.rb
 - lib/lumberjack/server.rb
 - spec/integration_spec.rb
-- spec/lumberjack/acking_protocol_v1_spec.rb
-- spec/lumberjack/acking_protocol_v2_spec.rb
 - spec/lumberjack/client_spec.rb
 - spec/lumberjack/connection_spec.rb
 - spec/lumberjack/server_spec.rb
@@ -114,31 +112,29 @@ files:
 homepage: https://github.com/jordansissel/lumberjack
 licenses: []
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>'
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.4.8
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
 specification_version: 4
 summary: lumberjack log transport library
 test_files:
 - spec/integration_spec.rb
-- spec/spec_helper.rb
-- spec/lumberjack/acking_protocol_v1_spec.rb
-- spec/lumberjack/acking_protocol_v2_spec.rb
 - spec/lumberjack/client_spec.rb
 - spec/lumberjack/connection_spec.rb
 - spec/lumberjack/server_spec.rb
+- spec/spec_helper.rb

data/spec/lumberjack/acking_protocol_v1_spec.rb

@@ -1,16 +0,0 @@
-# encoding: utf-8
-require "lumberjack/server"
-require "flores/random"
-
-describe Lumberjack::AckingProtocolV1 do
-  let(:number_of_events) { Flores::Random.integer(100..1024) }
-
-  subject { Lumberjack::AckingProtocolV1.new(number_of_events) }
-
-  it "should return true only once" do
-    results = []
-    number_of_events.times { |n| results << subject.ack?(n) }
-    expect(results.size).to eq(number_of_events)
-    expect(results.count(true)).to eq(1)
-  end
-end

data/spec/lumberjack/acking_protocol_v2_spec.rb

@@ -1,31 +0,0 @@
-# encoding: utf-8
-require "lumberjack/server"
-require "flores/random"
-
-describe Lumberjack::AckingProtocolV2 do
-  let(:results) { [] }
-  subject { Lumberjack::AckingProtocolV2.new(number_of_events) }
-  before { 1.upto(number_of_events) { |n| results << subject.ack?(n) } }
-
-  context "with multiples events" do
-    let(:number_of_events) { Flores::Random.integer(100..1024) }
-
-    it "should return multiples partial acks" do
-      expect(results.size).to eq(number_of_events)
-      expect(results.count(true)).to eq((number_of_events / number_of_events * Lumberjack::AckingProtocolV2::ACK_RATIO).ceil + 1)
-    end
-
-    it "last ack should be true" do
-      expect(results.last).to be_truthy
-    end
-  end
-
-  context "with only one event" do
-    let(:number_of_events) { 1 }
-
-    it "should return true only once" do
-      expect(results.size).to eq(number_of_events)
-      expect(results.count(true)).to eq(1)
-    end
-  end
-end