logstash-input-lumberjack 2.0.4 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0a025fb8e90f7ef8cff219175a2a73291c282cad
-  data.tar.gz: 5377ffc16b90af44b33130735f6607f268e05c8f
+  metadata.gz: a42b36942f3d863258583d875580d5386940e590
+  data.tar.gz: 1e0eb3341743ce7d86663ec74eebfe5c2f839318
 SHA512:
-  metadata.gz: 491440a3ae82f0a6f13f57e580cbc4d8cdd9b9898fd8bc45486f30569422a62ede36d003501fa124ae5c94d57db57f95c01f3b75bd97aa906a84d2284cc9e8ac
-  data.tar.gz: af441b88b8ad828f51fe57c262cd6191cefb1c5111da8ea0d2901d40329dcbdb7d4f2eac3a45e26e57ed3f46c1937aaadbe9bf71d80c14ac251e89c603bfca38
+  metadata.gz: ce1c18afd1ebbddf7dbc8d4fc698c6d6c87eed0fed2f1b9364756b942569bf6d96e2f8382e6b89a135f77e4c675e74056247eaa3f31152c39617ba0878dde4fb
+  data.tar.gz: f26e84688ae7c19ae9f5c20fa3e2427e9a11c63cc32589657c9d7b2f83c0ea54b6147603e620f825a726244562ac311f15a3bc7c9f2b4afdcd9b2263ccf38d47

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 2.0.5
+ - Add support for `stream_identity` to make the lumberjack input able to do disambiguation of file when using the multiline codec #53
+
 ## 2.0.4
  - Update `ruby-lumberjack` dependency to 0.0.26
   

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-lumberjack-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-lumberjack-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/inputs/lumberjack.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/inputs/base"
 require "logstash/namespace"
+require "logstash/codecs/identity_map_codec"
 
 # Receive events using the lumberjack protocol.
 #
@@ -63,31 +64,28 @@ class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
     @circuit_breaker = LogStash::CircuitBreaker.new("Lumberjack input",
                             :exceptions => [LogStash::SizedQueueTimeout::TimeoutError])
 
+    @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
   end # def register
 
   def run(output_queue)
-    start_buffer_broker(output_queue)
+    @output_queue = output_queue
+    start_buffer_broker
 
+    @codec.eviction_block(method(:flush_event))
+
+    # Accepting new events coming from LSF
     while !stop? do
-      # Wrappingu the accept call into a CircuitBreaker
+      # Wrapping the accept call into a CircuitBreaker
       if @circuit_breaker.closed?
         connection = @lumberjack.accept # call that creates a new connection
         next if connection.nil? # if the connection is nil the connection was close.
-        invoke(connection, @codec.clone) do |_codec, line, fields|
+        invoke(connection) do |event|
           if stop?
             connection.close
             break
           end
 
-          _codec.decode(line) do |event|
-            begin
-              decorate(event)
-              fields.each { |k,v| event[k] = v; v.force_encoding(Encoding::UTF_8) }
-              @circuit_breaker.execute { @buffered_queue.push(event, @congestion_threshold) }
-            rescue => e
-              raise e
-            end
-          end
+          @circuit_breaker.execute { @buffered_queue.push(event, @congestion_threshold) }
         end
       else
         @logger.warn("Lumberjack input: the pipeline is blocked, temporary refusing new connection.")
@@ -99,16 +97,52 @@ class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
   public
   def stop
     @lumberjack.close
+    @codec.flush { |event| flush_event(event) }
+  end
+
+  # I have created this method to make testing a lot easier,
+  # mocking multiples levels of block is unfriendly especially with 
+  # connection based block.
+  public
+  def create_event(fields, &block)
+    line = fields.delete("line")
+
+    @codec.decode(line, identity(fields)) do |event|
+      decorate(event)
+      fields.each { |k,v| event[k] = v; v.force_encoding(Encoding::UTF_8) }
+      block.call(event)
+    end
+  end
+
+  private
+  # It use the host and the file as the differentiator,
+  # if anything is provided it should fallback to an empty string.
+  def identity(fields)
+    [fields["host"], fields["file"]].compact.join("-")
+  end
+  # There is a problem with the way the codecs work for this specific input,
+  # when the data is decoded there is no way to attach metadata with the decoded line.
+  # If you look at the block used by `@codec.decode`  it reference the fields variable
+  # which is available when the proc is created, the problem is that variable with the data is 
+  # not available at eviction time or when we force a flush on the codec before 
+  # shutting down the input.
+  #
+  # Not defining the method will make logstash lose data, so Its still better to force a flush
+  #
+  # See this issue https://github.com/elastic/logstash/issues/4289 for more discussion
+  def flush_event(event)
+    decorate(event)
+    @output_queue << event
   end
 
   private
-  def invoke(connection, codec, &block)
+  def invoke(connection, &block)
     @threadpool.post do
       begin
         # If any errors occur in from the events the connection should be closed in the 
         # library ensure block and the exception will be handled here
         connection.run do |fields|
-          block.call(codec, fields.delete("line"), fields)
+          create_event(fields, &block)
         end
 
         # When too many errors happen inside the circuit breaker it will throw 
@@ -125,10 +159,10 @@ class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
     end
   end
 
-  def start_buffer_broker(output_queue)
+  def start_buffer_broker
     @threadpool.post do
       while !stop?
-        output_queue << @buffered_queue.pop_no_timeout
+        @output_queue << @buffered_queue.pop_no_timeout
       end
     end
   end

data/logstash-input-lumberjack.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-lumberjack'
-  s.version         = '2.0.4'
+  s.version         = '2.0.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Receive events using the lumberjack protocol."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -23,11 +23,11 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
 
   s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency 'jls-lumberjack', ['>=0.0.26']
+  s.add_runtime_dependency 'jls-lumberjack', '~> 0.0.26'
   s.add_runtime_dependency "concurrent-ruby"
+  s.add_runtime_dependency 'logstash-codec-multiline', "~> 2.0.4"
 
   s.add_development_dependency 'logstash-devutils'
-  s.add_development_dependency 'logstash-codec-multiline'
   s.add_development_dependency "flores"
   s.add_development_dependency "stud"
 end

data/spec/inputs/lumberjack_spec.rb

@@ -6,14 +6,18 @@ require "logstash/codecs/plain"
 require "logstash/codecs/multiline"
 require "logstash/event"
 require "lumberjack/client"
+require "lumberjack/server"
 
 Thread.abort_on_exception = true
 describe LogStash::Inputs::Lumberjack do
   let(:connection) { double("connection") }
   let(:certificate) { LogStashTest.certificate }
-  let(:port) { LogStashTest.random_port }
-  let(:queue)  { Queue.new }
-  let(:config)   { { "port" => 0, "ssl_certificate" => certificate.ssl_cert, "ssl_key" => certificate.ssl_key, "type" => "example", "tags" => "lumberjack" } }
+  let(:queue)  { [] }
+  let(:config)   { { "port" => 0,
+                     "ssl_certificate" => certificate.ssl_cert,
+                     "ssl_key" => certificate.ssl_key,
+                     "type" => "example",
+                     "tags" => "lumberjack" } }
 
   subject(:lumberjack) { LogStash::Inputs::Lumberjack.new(config) }
 
@@ -25,25 +29,49 @@ describe LogStash::Inputs::Lumberjack do
   end
 
   describe "#processing of events" do
-    let(:lines) { {"line" => "one\ntwo\n  two.2\nthree\n", "tags" => ["syslog"]} }
-
-    before do
-      allow(connection).to receive(:run).and_yield(lines)
-      lumberjack.register
-      expect_any_instance_of(Lumberjack::Server).to receive(:accept).and_return(connection)
-    end
+    context "multiline" do
+      let(:codec) { LogStash::Codecs::Multiline.new("pattern" => '^2015',
+                                                    "what" => "previous",
+                                                    "negate" => true) }
+      let(:config) { super.merge({ "codec" => codec }) }
+      let(:events_map) do
+        [
+          { "host" => "machineA", "file" => "/var/log/line", "line" => "2015-11-10 10:14:38,907 line 1" },
+          { "host" => "machineA", "file" => "/var/log/other", "line" => "2015-11-10 10:14:38,907 xline 1" },
+          { "host" => "machineA", "file" => "/var/log/line", "line" => "line 1.1" },
+          { "host" => "machineA", "file" => "/var/log/other", "line" => "xline 1.1" },
+          { "host" => "machineA", "file" => "/var/log/line", "line" => "2015-11-10 10:16:38,907 line 2" },
+          { "host" => "machineA", "file" => "/var/log/other", "line" => "2015-11-10 10:16:38,907 xline 2" },
+          { "host" => "machineA", "file" => "/var/log/line", "line" => "line 2.1" },
+          { "host" => "machineA", "file" => "/var/log/other", "line" => "xline 2.1" },
+          { "host" => "machineA", "file" => "/var/log/line", "line" => "line 2.2" },
+          { "host" => "machineA", "file" => "/var/log/other", "line" => "xline 2.2" },
+          { "host" => "machineA", "file" => "/var/log/line", "line" => "line 2.3" },
+          { "host" => "machineA", "file" => "/var/log/other", "line" => "xline 2.3" },
+          { "host" => "machineA", "file" => "/var/log/line", "line" => "2015-11-10 10:18:38,907 line 3" },
+          { "host" => "machineA", "file" => "/var/log/other", "line" => "2015-11-10 10:18:38,907 xline 3" }
+        ]
+      end
 
-    context "#codecs" do
-      let(:config) do
-        { "port" => port, "ssl_certificate" => certificate.ssl_cert, "ssl_key" => certificate.ssl_key,
-          "type" => "example", "codec" => codec }
+      before do
+        lumberjack.register
+        Thread.new { lumberjack.run(queue) }
       end
 
-      let(:codec) { LogStash::Codecs::Multiline.new("pattern" => '\n', "what" => "previous") }
-      it "clone the codec per connection" do
-        expect(lumberjack.codec).to receive(:clone).once
-        expect(lumberjack).to receive(:invoke) { break }
-        lumberjack.run(queue)
+      it "should correctly merge multiple events" do
+        # This test, cannot currently work without explicitely call a flush
+        # the flush is never timebased, if no new data is coming in we wont flush the buffer
+        # https://github.com/logstash-plugins/logstash-codec-multiline/issues/11
+        events_map.each { |e| lumberjack.create_event(e) { |e| queue << e } }
+        lumberjack.stop
+
+        expect(queue.size).to eq(6)
+        expect(queue.collect { |e| e["message"] }).to include("2015-11-10 10:14:38,907 line 1\nline 1.1",
+                                                              "2015-11-10 10:14:38,907 xline 1\nxline 1.1",
+                                                              "2015-11-10 10:16:38,907 line 2\nline 2.1\nline 2.2\nline 2.3",
+                                                              "2015-11-10 10:16:38,907 xline 2\nxline 2.1\nxline 2.2\nxline 2.3",
+                                                              "2015-11-10 10:18:38,907 line 3",
+                                                              "2015-11-10 10:18:38,907 xline 3")
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-lumberjack
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.0.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-19 00:00:00.000000000 Z
+date: 2015-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -48,12 +48,12 @@ dependencies:
   name: jls-lumberjack
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.26
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.26
   prerelease: false
@@ -73,21 +73,21 @@ dependencies:
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
+  name: logstash-codec-multiline
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.0.4
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.0.4
   prerelease: false
-  type: :development
+  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-multiline
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -134,21 +134,21 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
 - lib/logstash/circuit_breaker.rb
-- lib/logstash/sized_queue_timeout.rb
 - lib/logstash/inputs/lumberjack.rb
-- spec/spec_helper.rb
+- lib/logstash/sized_queue_timeout.rb
+- logstash-input-lumberjack.gemspec
 - spec/inputs/lumberjack_spec.rb
 - spec/logstash/circuit_breaker_spec.rb
 - spec/logstash/size_queue_timeout_spec.rb
+- spec/spec_helper.rb
 - spec/support/logstash_test.rb
-- logstash-input-lumberjack.gemspec
-- README.md
-- CHANGELOG.md
-- CONTRIBUTORS
-- Gemfile
-- LICENSE
-- NOTICE.TXT
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -171,13 +171,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Receive events using the lumberjack protocol.
 test_files:
-- spec/spec_helper.rb
 - spec/inputs/lumberjack_spec.rb
 - spec/logstash/circuit_breaker_spec.rb
 - spec/logstash/size_queue_timeout_spec.rb
+- spec/spec_helper.rb
 - spec/support/logstash_test.rb