logstash-input-beats 0.9 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 321ea86b3fdaf2c093f7604a05bde775ef6476ea
-  data.tar.gz: e0c553294e292ca1226f79dde49aa6a8d38ef965
+  metadata.gz: a07778f736810709bc97224c7470aa71093d9d38
+  data.tar.gz: 7ae28a43b020ede92e3d4da12b4796166c8c9e90
 SHA512:
-  metadata.gz: a7484e019d51a415086ac269d7c563be1eea08a3dbabb43f632565f2b5885b0dca60f86913a56526f9e8e3b4413882993cb4384fad703ac1e064df9c45272be7
-  data.tar.gz: 4e39b085af9c6794de64d68a5d3b52c49655d2789952d1f0e469dc4d0cf66362f34037cb6301173c4c31cb5f72fb7aba007d6d218a8e6e529cd6b37032ababa2
+  metadata.gz: c4f711f60d736f3ccf00b8be4bc54a6771cc3592ad6b6ccd081eba6d27fce538d0e7a775429158ecd2ab36639d4fd3bcaba5327097a152a2e0eb59a850d88bf9
+  data.tar.gz: f95b08c8cb8a261dfdc24f51b9d3561c23b9b028f7fd4a2276cc2ea1023392a30c393fe233e484229e73621621224323f4e6259de8f17ef78985b882924e904c

data/CHANGELOG.md

@@ -1,3 +1,5 @@
+# 0.9.1
+  - Move the ruby-lumberjack library into the plugin
 # 0.9
   - Created from `logstash-input-lumberjack` version 2.0.2 https://github.com/logstash-plugins/logstash-input-lumberjack
   - Use SSL off by default

data/Gemfile

@@ -1,2 +1,4 @@
 source 'https://rubygems.org'
-gemspec
+gemspec
+gem "logstash-core", "2.0.0.beta3"
+# gem "logstash-core", "1.5.4"

data/LICENSE

@@ -1,13 +1,14 @@
-Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+Copyright 2012–2015 Jordan Sissel, Elasticsearch and contributors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+

data/PROTOCOL.md

@@ -0,0 +1,127 @@
+# The Lumberjack Protocol
+
+The needs that lead to this protocol are:
+
+* Encryption and Authentication to protect 
+* Compression should be used to reduce bandwidth
+* Round-trip latency should not damage throughput
+* Application-level message acknowledgement
+
+## Implementation Considerations
+
+# Lumberjack Protocol v1
+
+## Behavior
+
+Sequence and ack behavior (including sliding window, etc) is similar to TCP,
+but instead of bytes, messages are the base unit.
+
+A writer with a window size of 50 events can send up to 50 unacked events
+before blocking. A reader can acknowledge the 'last event' received to
+support bulk acknowledgements.
+
+Reliable, ordered byte transport is ensured by using TCP (or TLS on top), and
+this protocol aims to provide reliable, application-level, message transport.
+
+## Encryption and Authentication
+
+Currently this is to be handled by TLS.
+
+## Wire Format
+
+### Layering
+
+This entire protocol is built to be layered on top of TCP or TLS.
+
+### Framing
+
+      0                   1                   2                   3
+      0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+     +---------------+---------------+-------------------------------+
+     |   version(1)  |   frame type  |     payload ...               |
+     +---------------------------------------------------------------+
+     |   payload continued...                                        |
+     +---------------------------------------------------------------+
+
+### 'data' frame type
+
+* SENT FROM WRITER ONLY
+* frame type value: ASCII 'D' aka byte value 0x44
+
+data is a map of string:string pairs. This is analogous to a Hash in Ruby, a
+JSON map, etc, but only strings are supported at this time.
+
+Payload:
+
+* 32bit unsigned sequence number
+* 32bit 'pair' count (how many key/value sequences follow)
+* 32bit unsigned key length followed by that many bytes for the key
+* 32bit unsigned value length followed by that many bytes for the value
+* repeat key/value 'count' times.
+
+Sequence number roll-over: If you receive a sequence number less than the
+previous value, this signals that the sequence number has rolled over.
+
+### 'json' frame type
+
+* SENT FROM WRITER ONLY
+* frame type value: ASCII 'J' aka byte value 0x4a
+
+data is json encoded.
+
+Payload:
+* 32bit unsigned sequence number
+* 32bit payload length (length in bytes of embedded json document)
+* 'length' bytes of json payload
+
+Sequence number roll-over: If you receive a sequence number less than the
+previous value, this signals that the sequence number has rolled over.
+
+### 'ack' frame type
+
+* SENT FROM READER ONLY
+* frame type value: ASCII 'A' aka byte value 0x41
+
+Payload:
+
+* 32bit unsigned sequence number.
+
+Bulk acks are supported. If you receive data frames in sequence order
+1,2,3,4,5,6, you can send an ack for '6' and the writer will take this to
+mean you are acknowledging all data frames before and including '6'.
+
+### 'window size' frame type
+
+* SENT FROM WRITER ONLY
+* frame type value: ASCII 'W' aka byte value 0x57
+
+Payload:
+
+* 32bit unsigned window size value in units of whole data frames.
+
+This frame is used to tell the reader the maximum number of unacknowledged
+data frames the writer will send before blocking for acks.
+
+### 'compressed' frame type
+
+* SENT FROM WRITER ONLY
+* frame type value: ASCII 'C' aka byte value 0x43
+
+Payload:
+
+* 32bit unsigned payload length 
+* 'length' bytes of compressed payload
+
+This frame type allows you to compress many frames into a single compressed
+envelope and is useful for efficiently compressing many small data frames.
+
+The compressed payload MUST contain full frames only, not partial frames.
+The uncompressed payload MUST be a valid frame stream by itself. As an example,
+you could have 3 data frames compressed into a single 'compressed' frame type:
+1D{k,v}{k,v}1D{k,v}{k,v}1D{k,v}{k,v} - when uncompressed, you should process
+the uncompressed payload as you would reading uncompressed frames from the
+network.
+
+TODO(sissel): It's likely this model is suboptimal, instead choose to
+use whole-stream compression z_stream in zlib (Zlib::ZStream in ruby) might be
+preferable.

data/lib/logstash/inputs/beats.rb

@@ -2,10 +2,11 @@
 require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/compatibility_layer_api_v1"
-require "lumberjack"
+require "lumberjack/beats"
+require "lumberjack/beats/server"
 
 # use Logstash provided json decoder
-Lumberjack::json = LogStash::Json
+Lumberjack::Beats::json = LogStash::Json
 
 # Allow Logstash to receive events from Beats
 #
@@ -50,7 +51,6 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   RECONNECT_BACKOFF_SLEEP = 0.5
   
   def register
-    require "lumberjack/server"
     require "concurrent"
     require "logstash/circuit_breaker"
     require "logstash/sized_queue_timeout"
@@ -63,7 +63,7 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
     end
 
     @logger.info("Starting Beats input listener", :address => "#{@host}:#{@port}")
-    @lumberjack = Lumberjack::Server.new(:address => @host, :port => @port,
+    @lumberjack = Lumberjack::Beats::Server.new(:address => @host, :port => @port,
       :ssl => @ssl, :ssl_certificate => @ssl_certificate, :ssl_key => @ssl_key,
       :ssl_key_passphrase => @ssl_key_passphrase)
 

data/lib/lumberjack/beats.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require "json"
+module Lumberjack module Beats
+  SEQUENCE_MAX = (2**32-1).freeze
+
+  @@json = Class.new do
+    def self.load(blob)
+      JSON.parse(blob)
+    end
+    def self.dump(v)
+      v.to_json
+    end
+  end
+
+  def self.json
+    @@json
+  end
+
+  def self.json=(j)
+    @@json = j
+  end
+end; end

data/lib/lumberjack/beats/client.rb

@@ -0,0 +1,219 @@
+# encoding: utf-8
+require "lumberjack/beats"
+require "socket"
+require "thread"
+require "openssl"
+require "zlib"
+
+module Lumberjack module Beats
+  class Client
+    def initialize(opts={})
+      @opts = {
+        :port => 0,
+        :addresses => [],
+        :ssl_certificate => nil,
+        :ssl => true,
+        :json => false,
+      }.merge(opts)
+
+      @opts[:addresses] = [@opts[:addresses]] if @opts[:addresses].class == String
+      raise "Must set a port." if @opts[:port] == 0
+      raise "Must set atleast one address" if @opts[:addresses].empty? == 0
+      raise "Must set a ssl certificate or path" if @opts[:ssl_certificate].nil? && @opts[:ssl]
+
+      @socket = connect
+    end
+
+    private
+    def connect
+      addrs = @opts[:addresses].shuffle
+      begin
+        raise "Could not connect to any hosts" if addrs.empty?
+        opts = @opts
+        opts[:address] = addrs.pop
+        Lumberjack::Beats::Socket.new(opts)
+      rescue *[Errno::ECONNREFUSED,SocketError]
+        retry
+      end
+    end
+
+    public
+    def write(elements, opts={})
+      @socket.write_sync(elements, opts)
+    end
+
+    public
+    def host
+      @socket.host
+    end
+  end
+
+  class Socket
+    # Create a new Lumberjack Socket.
+    #
+    # - options is a hash. Valid options are:
+    #
+    # * :port - the port to listen on
+    # * :address - the host/address to bind to
+    # * :ssl - enable/disable ssl support
+    # * :ssl_certificate - the path to the ssl cert to use.
+    #                      If ssl_certificate is not set, a plain tcp connection
+    #                      will be used.
+    attr_reader :sequence
+    attr_reader :host
+    def initialize(opts={})
+      @sequence = 0
+      @last_ack = 0
+      @opts = {
+        :port => 0,
+        :address => "127.0.0.1",
+        :ssl_certificate => nil,
+        :ssl => true,
+        :json => false,
+      }.merge(opts)
+      @host = @opts[:address]
+
+      connection_start(opts)
+    end
+
+    private
+    def connection_start(opts)
+      tcp_socket = TCPSocket.new(opts[:address], opts[:port])
+      if !opts[:ssl]
+        @socket = tcp_socket
+      else
+        certificate = OpenSSL::X509::Certificate.new(File.read(opts[:ssl_certificate]))
+
+        certificate_store = OpenSSL::X509::Store.new
+        certificate_store.add_cert(certificate)
+
+        ssl_context = OpenSSL::SSL::SSLContext.new
+        ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        ssl_context.cert_store = certificate_store
+
+        @socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, ssl_context)
+        @socket.connect
+      end
+    end
+
+    private 
+    def inc
+      @sequence = 0 if @sequence + 1 > Lumberjack::Beats::SEQUENCE_MAX
+      @sequence = @sequence + 1
+    end
+
+    private
+    def send_window_size(size)
+      @socket.syswrite(["1", "W", size].pack("AAN"))
+    end
+
+    private
+    def send_payload(payload)
+      # SSLSocket has a limit of 16k per message
+      # execute multiple writes if needed
+      bytes_written = 0
+      while bytes_written < payload.bytesize
+        bytes_written += @socket.syswrite(payload.byteslice(bytes_written..-1))
+      end
+    end
+
+    public
+    def write_sync(elements, opts={})
+      options = {
+        :json => @opts[:json],
+      }.merge(opts)
+
+      elements = [elements] if elements.is_a?(Hash)
+      send_window_size(elements.size)
+
+      encoder = options[:json] ? JsonEncoder : FrameEncoder
+      payload = elements.map { |element| encoder.to_frame(element, inc) }.join
+      compress = compress_payload(payload)
+      send_payload(compress)
+
+      ack(elements.size)
+    end
+
+    private 
+    def compress_payload(payload)
+      compress = Zlib::Deflate.deflate(payload)
+      ["1", "C", compress.bytesize, compress].pack("AANA*")
+    end
+
+    private
+    def ack(size)
+      _, type = read_version_and_type
+      raise "Whoa we shouldn't get this frame: #{type}" if type != "A"
+      @last_ack = read_last_ack
+    end
+
+    private
+    def unacked_sequence_size
+      sequence - (@last_ack + 1)
+    end
+
+    private
+    def read_version_and_type
+      version = @socket.read(1)
+      type    = @socket.read(1)
+      [version, type]
+    end
+
+    private
+    def read_last_ack
+      @socket.read(4).unpack("N").first
+    end
+  end
+
+  module JsonEncoder
+    def self.to_frame(hash, sequence)
+      json = Lumberjack::Beats::json.dump(hash)
+      json_length = json.bytesize
+      pack = "AANNA#{json_length}"
+      frame = ["1", "J", sequence, json_length, json]
+      frame.pack(pack)
+    end
+  end # JsonEncoder
+
+  module FrameEncoder
+    def self.to_frame(hash, sequence)
+      frame = ["1", "D", sequence]
+      pack = "AAN"
+      keys = deep_keys(hash)
+      frame << keys.length
+      pack << "N"
+      keys.each do |k|
+        val = deep_get(hash,k)
+        key_length = k.bytesize
+        val_length = val.bytesize
+        frame << key_length
+        pack << "N"
+        frame << k
+        pack << "A#{key_length}"
+        frame << val_length
+        pack << "N"
+        frame << val
+        pack << "A#{val_length}"
+      end
+      frame.pack(pack)
+    end
+
+    private
+    def self.deep_get(hash, key="")
+      return hash if key.nil?
+      deep_get(
+        hash[key.split('.').first],
+        key[key.split('.').first.length+1..key.length]
+      )
+    end
+    private
+    def self.deep_keys(hash, prefix="")
+      keys = []
+      hash.each do |k,v|
+        keys << "#{prefix}#{k}" if v.class == String
+        keys << deep_keys(hash[k], "#{k}.") if v.class == Hash
+      end
+      keys.flatten
+    end
+  end # module Encoder
+end; end

data/lib/lumberjack/beats/server.rb

@@ -0,0 +1,433 @@
+# encoding: utf-8
+require "lumberjack/beats"
+require "socket"
+require "thread"
+require "openssl"
+require "zlib"
+require "json"
+require "concurrent"
+
+module Lumberjack module Beats
+  class Server
+    SOCKET_TIMEOUT = 1 # seconds
+
+    attr_reader :port
+
+    # Create a new Lumberjack server.
+    #
+    # - options is a hash. Valid options are:
+    #
+    # * :port - the port to listen on
+    # * :address - the host/address to bind to
+    # * :ssl_certificate - the path to the ssl cert to use
+    # * :ssl_key - the path to the ssl key to use
+    # * :ssl_key_passphrase - the key passphrase (optional)
+    def initialize(options={})
+      @options = {
+        :port => 0,
+        :address => "0.0.0.0",
+        :ssl => true,
+        :ssl_certificate => nil,
+        :ssl_key => nil,
+        :ssl_key_passphrase => nil
+      }.merge(options)
+
+      if @options[:ssl]
+        [:ssl_certificate, :ssl_key].each do |k|
+          if @options[k].nil?
+            raise "You must specify #{k} in Lumberjack::Server.new(...)"
+          end
+        end
+      end
+
+      @server = TCPServer.new(@options[:address], @options[:port])
+
+      @close = Concurrent::AtomicBoolean.new
+
+      # Query the port in case the port number is '0'
+      # TCPServer#addr == [ address_family, port, address, address ]
+      @port = @server.addr[1]
+
+      if @options[:ssl]
+        # load SSL certificate
+        @ssl = OpenSSL::SSL::SSLContext.new
+        @ssl.cert = OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate]))
+        @ssl.key = OpenSSL::PKey::RSA.new(File.read(@options[:ssl_key]),
+          @options[:ssl_key_passphrase])
+      end
+    end # def initialize
+
+    def run(&block)
+      while !closed?
+        connection = accept
+
+        # Some exception may occur in the accept loop
+        # we will try again in the next iteration
+        # unless the server is closing
+        next unless connection
+
+        Thread.new(connection) do |connection|
+          connection.run(&block)
+        end
+      end
+    end # def run
+
+    def ssl?
+      @ssl
+    end
+
+    def accept(&block)
+      begin
+        socket = @server.accept_nonblock
+        # update the socket with a SSL layer
+        socket = accept_ssl(socket) if ssl?
+
+        if block_given?
+          block.call(socket, self)
+        else
+          return Connection.new(socket, self)
+        end
+      rescue OpenSSL::SSL::SSLError, IOError, EOFError, Errno::EBADF
+        socket.close rescue nil
+        retry unless closed?
+      rescue IO::WaitReadable, Errno::EAGAIN # Resource not ready yet, so lets try again
+        begin
+          IO.select([@server], nil, nil, SOCKET_TIMEOUT)
+          retry unless closed?
+        rescue IOError, Errno::EBADF => e # we currently closing
+          raise e unless closed?
+        end
+      end
+    end
+
+    def accept_ssl(tcp_socket)
+      ssl_socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, @ssl)
+      ssl_socket.sync_close
+
+      begin
+        ssl_socket.accept_nonblock
+
+        return ssl_socket
+      rescue IO::WaitReadable # handshake
+        IO.select([ssl_socket], nil, nil, SOCKET_TIMEOUT)
+        retry unless closed?
+      rescue IO::WaitWritable # handshake
+        IO.select(nil, [ssl_socket], nil, SOCKET_TIMEOUT)
+        retry unless closed?
+      end
+    end
+
+    def closed?
+      @close.value
+    end
+
+    def close
+      @close.make_true
+      @server.close unless @server.closed?
+    end
+  end # class Server
+
+  class Parser
+    PROTOCOL_VERSION_1 = "1".ord
+    PROTOCOL_VERSION_2 = "2".ord
+
+    SUPPORTED_PROTOCOLS = [PROTOCOL_VERSION_1, PROTOCOL_VERSION_2]
+
+    def initialize
+      @buffer_offset = 0
+      @buffer = ""
+      @buffer.force_encoding("BINARY")
+      transition(:header, 2)
+    end # def initialize
+
+    def transition(state, next_length)
+      @state = state
+      #puts :transition => state
+      # TODO(sissel): Assert this self.respond_to?(state)
+      # TODO(sissel): Assert state is in STATES
+      # TODO(sissel): Assert next_length is a number
+      need(next_length)
+    end # def transition
+
+    # Feed data to this parser.
+    # 
+    # Currently, it will return the raw payload of websocket messages.
+    # Otherwise, it returns nil if no complete message has yet been consumed.
+    #
+    # @param [String] the string data to feed into the parser. 
+    # @return [String, nil] the websocket message payload, if any, nil otherwise.
+    def feed(data, &block)
+      @buffer << data
+      #p :need => @need
+      while have?(@need)
+        send(@state, &block) 
+        #case @state
+        #when :header; header(&block)
+        #when :window_size; window_size(&block)
+        #when :data_lead; data_lead(&block)
+        #when :data_field_key_len; data_field_key_len(&block)
+        #when :data_field_key; data_field_key(&block)
+        #when :data_field_value_len; data_field_value_len(&block)
+        #when :data_field_value; data_field_value(&block)
+        #when :data_field_value; data_field_value(&block)
+        #when :compressed_lead; compressed_lead(&block)
+        #when :compressed_payload; compressed_payload(&block)
+        #end # case @state
+      end
+      return nil
+    end # def <<
+
+    # Do we have at least 'length' bytes in the buffer?
+    def have?(length)
+      return length <= (@buffer.size - @buffer_offset)
+    end # def have?
+
+    # Get 'length' string from the buffer.
+    def get(length=nil)
+      length = @need if length.nil?
+      data = @buffer[@buffer_offset ... @buffer_offset + length]
+      @buffer_offset += length
+      if @buffer_offset > 16384
+        @buffer = @buffer[@buffer_offset  .. -1]
+        @buffer_offset = 0
+      end
+      return data
+    end # def get
+
+    # Set the minimum number of bytes we need in the buffer for the next read.
+    def need(length)
+      @need = length
+    end # def need
+
+    FRAME_WINDOW = "W".ord
+    FRAME_DATA = "D".ord
+    FRAME_JSON_DATA = "J".ord
+    FRAME_COMPRESSED = "C".ord
+    def header(&block)
+      version, frame_type = get.bytes.to_a[0..1]
+      version ||= PROTOCOL_VERSION_1
+
+      handle_version(version, &block)
+
+      case frame_type
+      when FRAME_WINDOW; transition(:window_size, 4)
+      when FRAME_DATA; transition(:data_lead, 8)
+      when FRAME_JSON_DATA; transition(:json_data_lead, 8)
+      when FRAME_COMPRESSED; transition(:compressed_lead, 4)
+      else; raise "Unknown frame type: `#{frame_type}`"
+      end
+    end
+
+    def handle_version(version, &block)
+      if supported_protocol?(version)
+        yield :version, version
+      else
+        raise "unsupported protocol #{version}"
+      end
+    end
+
+    def supported_protocol?(version)
+      SUPPORTED_PROTOCOLS.include?(version)
+    end
+
+    def window_size(&block)
+      @window_size = get.unpack("N").first
+      transition(:header, 2)
+      yield :window_size, @window_size
+    end # def window_size
+
+    def json_data_lead(&block)
+      @sequence, payload_size = get.unpack("NN")
+      transition(:json_data_payload, payload_size)
+    end
+
+    def json_data_payload(&block)
+      payload = get
+      yield :json, @sequence, Lumberjack::Beats::json.load(payload)
+      transition(:header, 2)
+    end
+
+    def data_lead(&block)
+      @sequence, @data_count = get.unpack("NN")
+      @data = {}
+      transition(:data_field_key_len, 4)
+    end
+
+    def data_field_key_len(&block)
+      key_len = get.unpack("N").first
+      transition(:data_field_key, key_len)
+    end
+
+    def data_field_key(&block)
+      @key = get
+      transition(:data_field_value_len, 4)
+    end
+
+    def data_field_value_len(&block)
+      transition(:data_field_value, get.unpack("N").first)
+    end
+
+    def data_field_value(&block)
+      @value = get
+
+      @data_count -= 1
+      @data[@key] = @value
+
+      if @data_count > 0
+        transition(:data_field_key_len, 4)
+      else
+        # emit the whole map now that we found the end of the data fields list.
+        yield :data, @sequence, @data
+        transition(:header, 2)
+      end
+
+    end # def data_field_value
+
+    def compressed_lead(&block)
+      length = get.unpack("N").first
+      transition(:compressed_payload, length)
+    end
+
+    def compressed_payload(&block)
+      original = Zlib::Inflate.inflate(get)
+      transition(:header, 2)
+
+      # Parse the uncompressed payload.
+      feed(original, &block)
+    end
+  end # class Parser
+
+  class Connection
+    READ_SIZE = 16384
+
+    attr_accessor :server
+
+    def initialize(fd, server)
+      @parser = Parser.new
+      @fd = fd
+
+      @server = server
+      @ack_handler = nil
+    end
+
+    def run(&block)
+      while !server.closed?
+        read_socket(&block)
+      end
+    rescue EOFError, OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET
+      # EOF or other read errors, only action is to shutdown which we'll do in
+      # 'ensure'
+    ensure
+      close rescue 'Already closed stream'
+    end # def run
+
+    def read_socket(&block)
+      # TODO(sissel): Ack on idle.
+      # X: - if any unacked, IO.select
+      # X:   - on timeout, ack all.
+      # X: Doing so will prevent slow streams from retransmitting
+      # X: too many events after errors.
+      @parser.feed(@fd.sysread(READ_SIZE)) do |event, *args|
+        case event
+        when :version
+          version(*args)
+        when :window_size
+          reset_next_ack(*args)
+        when :data
+          sequence, map = args
+          ack_if_needed(sequence) { data(map, &block) }
+        when :json
+          # If the payload is an array of items we will emit multiple events
+          # this behavior was moved from the plugin to the library.
+          # see this commit: https://github.com/logstash-plugins/logstash-input-lumberjack/pull/57/files#diff-1b9590423b15f04f215635164e7376ecR158
+          sequence, map = args
+
+          ack_if_needed(sequence) do
+            if map.is_a?(Array)
+              map.each { |e| data(e, &block) }
+            else
+              data(map, &block)
+            end
+          end
+        end
+      end
+    end
+
+    def version(version)
+      @version = version
+    end
+
+    def ack_if_needed(sequence, &block)
+      block.call
+      send_ack(sequence) if @ack_handler.ack?(sequence)
+    end
+
+    def close
+      @fd.close unless @fd.closed?
+    end
+
+    def data(map, &block)
+      block.call(map) if block_given?
+    end
+
+    def reset_next_ack(window_size)
+      klass = (@version == Parser::PROTOCOL_VERSION_1) ? AckingProtocolV1 : AckingProtocolV2
+      @ack_handler = klass.new(window_size)
+    end
+
+    def send_ack(sequence)
+      @fd.syswrite(@ack_handler.ack_frame(sequence))
+    end
+  end # class Connection
+
+  class AckingProtocolV1
+    def initialize(window_size)
+      @next_ack = nil
+      @window_size = window_size
+    end
+
+    def ack?(sequence)
+      # The first encoded event will contain the sequence number 
+      # this is needed to know when we should ack.
+      @next_ack = compute_next_ack(sequence) if @next_ack.nil?
+      sequence == @next_ack
+    end
+
+    def ack_frame(sequence)
+      ["1A", sequence].pack("A*N")
+    end
+
+    private
+    def compute_next_ack(sequence)
+      (sequence + @window_size - 1) % SEQUENCE_MAX
+    end
+  end
+
+  # Allow lumberjack to send partial ack back to the producer
+  # only V2 client support partial Acks
+  #
+  # Send Ack on every 20% of the data, so with default settings every 200 events
+  # This should reduce the congestion on retransmit.
+  class AckingProtocolV2
+    ACK_RATIO = 5
+
+    def initialize(window_size)
+      @window_size = window_size
+      @every = (window_size / ACK_RATIO).round
+    end
+
+    def ack?(sequence)
+      if @window_size == sequence
+        true
+      elsif sequence % @every == 0
+        true
+      else
+        false
+      end
+    end
+
+    def ack_frame(sequence)
+      ["2A", sequence].pack("A*N")
+    end
+  end
+end; end# module Lumberjack