logstash-output-lumberjack 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 75bed5f35c130740c722189aa544dbe88f12d798
-  data.tar.gz: 70f3fd8b618bd928a6ed17308678db824ae5d25d
+  metadata.gz: 7de0e64427c75281d60a669c7374e8ba2e8d2c24
+  data.tar.gz: 34eb43c4342dc6c4f8dc1cb1e6d0f6c32243013f
 SHA512:
-  metadata.gz: 92b876a22c31d93a35cd8f2fb1511c0d95eeb24a31d67093db5d266ae152f4c0f8431f86b956050df04cc999fb1d914fef9a41f1b52650a6b5b4047e250bef42
-  data.tar.gz: a02a9d5ad0d5bc15e3089b2a8f5d5a1ebe7a9897f1da29413ee90f979a243fcbc47a7fe01fceb8b2ad9b7f2938839887528dd1e550ef07662e5827c6dce97d49
+  metadata.gz: c415eec32245cce31416af9c575c8612a06f8125c40c26c36db42d0e4b383291ad3f78693f41de22cde49d63e39d702491715eb12e1d19e0592924d4f47609b5
+  data.tar.gz: 7a20703d9e80df5d3cc847d269c0c45b7793f424dcaf5ff113f1a3362e52ffda109ca8e1ffb21f826ef3b6fcf71977c94948f5f27d23d3f4d9ae7270064c2857

data/lib/logstash/outputs/lumberjack.rb

@@ -1,5 +1,10 @@
 # encoding: utf-8
+require "logstash/outputs/base"
+require "stud/buffer"
+require "thread"
+
 class LogStash::Outputs::Lumberjack < LogStash::Outputs::Base
+  include Stud::Buffer
 
   config_name "lumberjack"
 
@@ -13,22 +18,41 @@ class LogStash::Outputs::Lumberjack < LogStash::Outputs::Base
   config :ssl_certificate, :validate => :path, :required => true
 
   # window size
-  config :window_size, :validate => :number, :default => 5000
+  config :window_size, :validate => :number, :deprecated => "Use `flush_size`", :require => false
+  
+  # To make efficient calls to the lumberjack output we are buffering events locally. 
+  # if the number of events exceed the number the declared `flush_size` we will 
+  # send them to the logstash server.
+  config :flush_size, :validate => :number, :default => 1024
+
+  # The amount of time since last flush before a flush is forced.
+  #
+  # This setting helps ensure slow event rates don't get stuck in Logstash.
+  # For example, if your `flush_size` is 100, and you have received 10 events,
+  # and it has been more than `idle_flush_time` seconds since the last flush,
+  # Logstash will flush those 10 events automatically.
+  #
+  # This helps keep both fast and slow log streams moving along in
+  # near-real-time.
+  config :idle_flush_time, :validate => :number, :default => 1
+
+  RECONNECT_BACKOFF_SLEEP = 0.5
 
   public
   def register
     require 'lumberjack/client'
+
+    buffer_initialize(
+      :max_items => max_items,
+      :max_interval => @idle_flush_time,
+      :logger => @logger
+    )
+
     connect
 
     @codec.on_event do |event, payload|
-      begin
-        @client.write({ 'line' => payload })
-      rescue Exception => e
-        @logger.error("Client write error, trying connect", :e => e, :backtrace => e.backtrace)
-        connect
-        retry
-      end # begin
-    end # @codec
+      buffer_receive({ "line" => payload })
+    end
   end # def register
 
   public
@@ -41,7 +65,23 @@ class LogStash::Outputs::Lumberjack < LogStash::Outputs::Base
     @codec.encode(event)
   end # def receive
 
+  def flush(events, teardown = false)
+    begin
+      @logger.info("Sending events to lumberjack", :size => events.size)
+      @client.write(events)
+    rescue Exception => e
+      @logger.error("Client write error, trying connect", :e => e, :backtrace => e.backtrace)
+      sleep(RECONNECT_BACKOFF_SLEEP)
+      connect
+      retry
+    end # begin
+  end
+  
   private 
+  def max_items
+    @window_size || @flush_size
+  end
+
   def connect
     require 'resolv'
     @logger.info("Connecting to lumberjack server.", :addresses => @hosts, :port => @port, 

data/logstash-output-lumberjack.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-lumberjack'
-  s.version         = '1.0.1'
+  s.version         = '1.0.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Send events using the lumberjack protocol"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -23,8 +23,12 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
 
-  s.add_runtime_dependency 'jls-lumberjack', ['>=0.0.23']
+  s.add_runtime_dependency 'jls-lumberjack', ['>=0.0.24']
+  s.add_runtime_dependency "stud"
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency "logstash-output-lumberjack"
+  s.add_development_dependency "logstash-codec-plain"
+  s.add_development_dependency "flores"
 end
 

data/spec/outputs/lumberjack_spec.rb

@@ -1 +1,76 @@
+# encoding: utf-8
+require "logstash/outputs/lumberjack"
+require "logstash/event"
 require "logstash/devutils/rspec/spec_helper"
+require "lumberjack/server"
+require "flores/pki"
+require "stud/temporary"
+require "fileutils"
+
+describe "Sending events" do
+  let(:batch_size) { Flores::Random.integer(20..100) }
+  let(:batch_payload) do
+    batch_size.times.collect { |n| LogStash::Event.new({ "message" => "foobar #{n}" }) }
+  end
+
+  let(:number_of_crash) { Flores::Random.integer(1..10) }
+  let(:certificate) { Flores::PKI.generate }
+  let(:certificate_file_crt) { Stud::Temporary.pathname }
+  let(:certificate_file_key) { Stud::Temporary.pathname }
+  let(:port) { Flores::Random.integer(1024..65535) }
+  let(:host) { "127.0.0.1" }
+  let(:queue) { [] }
+
+  let(:client_options) {
+    {
+      "hosts" => [host],
+      "port" => port,
+      "ssl_certificate" => certificate_file_crt,
+      "flush_size" => batch_size
+    }
+  }
+  let(:output) { LogStash::Outputs::Lumberjack.new(client_options) }
+
+  context "when the server closes the connection" do
+    before do
+      File.open(certificate_file_crt, "a") { |f| f.write(certificate.first) }
+      File.open(certificate_file_key, "a") { |f| f.write(certificate.last) }
+
+      server = Lumberjack::Server.new(:port => port,
+                                      :address => host,
+                                      :ssl_certificate => certificate_file_crt,
+                                      :ssl_key => certificate_file_key)
+
+      crashed_count = 0
+      @server = Thread.new do
+        begin
+          server.run do |data|
+            if crashed_count < number_of_crash
+              crashed_count += 1
+              raise "crashed"
+            end
+
+            queue << data
+          end
+        rescue
+        end
+      end
+
+      output.register
+    end
+
+    after do
+      FileUtils.rm_rf(certificate_file_crt)
+      FileUtils.rm_rf(certificate_file_key)
+    end
+
+    it "reconnects and resend the payload" do
+      # We guarantee at least once, 
+      # duplicates can happen in this scenario.
+      batch_payload.each { |event| output.receive(event) }
+
+      try(10) { expect(queue.size).to be >= batch_size }
+      expect(queue.map { |e| e["line"] }).to include(*batch_payload.map(&:to_s))
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-lumberjack
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-31 00:00:00.000000000 Z
+date: 2015-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -36,12 +36,26 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.23
+        version: 0.0.24
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.23
+        version: 0.0.24
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: stud
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -58,6 +72,48 @@ dependencies:
         version: '0'
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-output-lumberjack
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: flores
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []