logstash-input-beats 2.2.5 → 2.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fff429c1372d2d1a71ae92cadf753d421b009d46
-  data.tar.gz: 2f84e5dc8bc9ff6c79a0d7cb0485b2a2ee5fb07e
+  metadata.gz: 0c9f5e238355c46a4cf8de5a79ab788941bfd524
+  data.tar.gz: c40c248be28f9c7a28764761340dd0d926f6cdb6
 SHA512:
-  metadata.gz: 5018ac549a816be327dab8590b918db01c6a3453c3aead991c8cbe63af9cbb47cf7a8d5445dbe8d425647db9c753ffb14f55877b3c031566c300d2448a3370d7
-  data.tar.gz: ce32435856bc68e86dd6df0355c4fab9d10dd7237ee11f99782fb9b905b90a4e3a6aa6a7b9b1c0b940f2d8754ece577f303b523d98752fe0c08bf3e58c0d7e87
+  metadata.gz: 12996169fca9958102f280cb5d532cb17717f940ac62f4d7deaa19a851b875817e8da5432fd0fec5733971b7de9de4d2b2fec498bb9f8f281bbb7d54b6a056bb
+  data.tar.gz: 1a3a6d86aaf2f30a63524d98664b1abdce873e03e1dca0b180c2650314f19f6229730cbcfa4902d3eb8b0a2b14e1b43c8c04c3133399dee6921ab64a46fecbec

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 2.2.7
+  - More robust test when using a random port #60
+  - Fix LSF integration tests #52
+# 2.2.6
+  - Do not use the identity map if we don't explicitly use the `multiline` codec
 # 2.2.5
   - Fix failing tests introduce by the `ssl_key_passphrase` changes.
   - Added an integration test for the `ssl_key_passphrase`

data/lib/logstash/inputs/beats.rb

@@ -5,6 +5,7 @@ require "logstash/timestamp"
 require "lumberjack/beats"
 require "lumberjack/beats/server"
 require "logstash/codecs/identity_map_codec"
+require "logstash/codecs/multiline"
 require "logstash/inputs/beats_support/circuit_breaker"
 require "logstash/inputs/beats_support/codec_callback_listener"
 require "logstash/inputs/beats_support/connection_handler"
@@ -127,8 +128,13 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
                             :exceptions => [InsertingToQueueTakeTooLong])
 
     # wrap the configured codec to support identity stream
-    # from the producers
-    @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
+    # from the producers if running with the multiline codec.
+    #
+    # If they dont need an identity map, codec are stateless and can be reused
+    # accross multiples connections.
+    if need_identity_map?
+      @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
+    end
 
     # Keep a list of active connections so we can flush their codec on shutdown
 
@@ -256,4 +262,8 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
       end
     end
   end
+
+  def need_identity_map?
+    @codec.kind_of?(LogStash::Codecs::Multiline)
+  end
 end # class LogStash::Inputs::Beats

data/lib/logstash/inputs/beats_support/decoded_event_transform.rb

@@ -11,7 +11,7 @@ module LogStash::Inputs::BeatsSupport
       event["@timestamp"] = ts unless ts.nil?
       hash.each { |k, v| event[k] = v }
       super(event)
-      event.tag("beats_input_codec_#{@input.codec.base_codec.class.config_name}_applied")
+      event.tag("beats_input_codec_#{codec_name}_applied")
       event
     end
 

data/lib/logstash/inputs/beats_support/event_transform_common.rb

@@ -31,6 +31,14 @@ module LogStash::Inputs::BeatsSupport
       @input.send(:decorate, event)
     end
 
+    def codec_name
+      @codec_name ||= if @input.codec.respond_to?(:base_codec)
+                        @input.codec.base_codec.class.config
+                      else
+                        @input.codec.class.config_name
+                      end
+    end
+
     def transform(event)
       copy_beat_hostname(event)
       decorate(event)

data/logstash-input-beats.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = "logstash-input-beats"
-  s.version         = '2.2.5'
+  s.version         = '2.2.7'
   s.licenses        = ["Apache License (2.0)"]
   s.summary         = "Receive events using the lumberjack protocol."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/beats_spec.rb

@@ -16,6 +16,30 @@ describe LogStash::Inputs::Beats do
   let(:config)   { { "port" => 0, "ssl_certificate" => certificate.ssl_cert, "ssl_key" => certificate.ssl_key, "type" => "example", "tags" => "beats"} }
 
   context "#register" do
+    context "identity map" do
+      subject(:plugin) { LogStash::Inputs::Beats.new(config) }
+      before { plugin.register }
+
+      context "when using the multiline codec" do
+        let(:codec) { LogStash::Codecs::Multiline.new("pattern" => '^2015',
+                                                      "what" => "previous",
+                                                      "negate" => true) }
+        let(:config) { super.merge({ "codec" => codec }) }
+
+        it "wraps the codec with the identity_map" do
+          expect(plugin.codec).to be_kind_of(LogStash::Codecs::IdentityMapCodec)
+        end
+      end
+
+      context "when using non buffered codecs" do
+        let(:config) { super.merge({ "codec" => "json" }) }
+
+        it "doesnt wrap the codec with the identity map" do
+          expect(plugin.codec).to be_kind_of(LogStash::Codecs::JSON)
+        end
+      end
+    end
+
     it "raise no exception" do
       plugin = LogStash::Inputs::Beats.new(config)
       expect { plugin.register }.not_to raise_error

data/spec/integration/logstash_forwarder_spec.rb

@@ -10,7 +10,7 @@ require "spec_helper"
 
 LSF_BINARY = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "vendor", "logstash-forwarder", "logstash-forwarder"))
 
-xdescribe "Logstash-Forwarder", :integration => true do
+describe "Logstash-Forwarder", :integration => true do
   include ClientProcessHelpers
 
   before :all do
@@ -61,6 +61,7 @@ xdescribe "Logstash-Forwarder", :integration => true do
 
   context "Plain TCP" do
     include ClientProcessHelpers
+
     let(:certificate_authorities) { "" }
 
     it "should not send any events" do
@@ -85,13 +86,12 @@ xdescribe "Logstash-Forwarder", :integration => true do
       let(:certificate_authorities) { certificate_file }
 
       context "self signed certificate" do
-        include_examples "send events" do
-        end
+        include_examples "send events"
       end
 
       context "invalid CA on the client" do
         let(:invalid_data) { Flores::PKI.generate }
-        let(:certificate_authorities) { invalid_data.first }
+        let(:certificate_authorities) { f = Stud::Temporary.file; f.close; f.path }
 
         it "should not send any events" do
           expect(queue.size).to eq(0), @execution_output

data/spec/integration_spec.rb

@@ -30,8 +30,8 @@ describe "A client" do
     p.close
     p.path
   end
-  let(:port) { Flores::Random.integer(1024..65335) }
-  let(:tcp_port) { port + 1 }
+  let(:port) { Flores::Random.port }
+  let(:tcp_port) { Flores::Random.port }
   let(:host) { "127.0.0.1" }
   let(:queue) { [] }
   let(:config_ssl) do

data/spec/lumberjack/beats/client_spec.rb

@@ -11,7 +11,7 @@ require "zlib"
 
 describe Lumberjack::Beats::Client do
   describe Lumberjack::Beats::Socket do
-    let(:port)   { 5000 }
+    let(:port) { Flores::Random.port }
 
     subject(:socket) { Lumberjack::Beats::Socket.new(:port => port, :ssl_certificate => "" ) }
 

data/spec/lumberjack/beats/connection_spec.rb

@@ -5,7 +5,7 @@ require "flores/random"
 
 describe "Connnection" do
   let(:ip) { "192.168.1.2" }
-  let(:port) { 4444 }
+  let(:port) { Flores::Random.port }
   let(:server) { double("server", :closed? => false) }
   let(:socket) { double("socket", :closed? => false) }
   let(:connection) { Lumberjack::Beats::Connection.new(socket, server) }

data/spec/lumberjack/beats/server_spec.rb

@@ -4,6 +4,7 @@ require "lumberjack/beats/server"
 require "flores/random"
 require "flores/pki"
 require "spec_helper"
+require_relative "../../support/flores_extensions"
 
 Thread.abort_on_exception = true
 
@@ -11,8 +12,8 @@ describe "Server" do
   let(:certificate) { Flores::PKI.generate }
   let(:certificate_file_crt) { "certificate.crt" }
   let(:certificate_file_key) { "certificate.key" }
-  let(:port) { Flores::Random.integer(1024..65335) }
-  let(:tcp_port) { port + 1 }
+  let(:port) { Flores::Random.port }
+  let(:tcp_port) { Flores::Random.port }
   let(:host) { "127.0.0.1" }
   let(:queue) { [] }
 

data/spec/support/client_process_helpers.rb

@@ -10,6 +10,12 @@ module ClientProcessHelpers
     @process.io.stdout = @process.io.stderr = @client_out
     ChildProcess.posix_spawn = true
     @process.start
+
+    sleep(0.1)
+    @client_out.rewind
+
+    # can be used to helper debugging when a test fails
+    @execution_output = @client_out.read
   end
 
   def stop_client
@@ -18,10 +24,5 @@ module ClientProcessHelpers
     rescue ChildProcess::TimeoutError
       @process.stop
     end
-
-    @client_out.rewind
-
-    # can be used to helper debugging when a test fails
-    @execution_output = @client_out.read
   end
 end

data/spec/support/flores_extensions.rb

@@ -1,6 +1,46 @@
 # encoding: utf-8
 require "flores/pki"
+require "flores/random"
+
 module Flores
+  module Random
+    DEFAULT_PORT_RANGE = 1024..65535
+    DEFAULT_PORT_CHECK_TIMEOUT = 1
+    DEFAULT_MAXIMUM_PORT_FIND_TRY = 15
+
+    class << self
+      def port(range = DEFAULT_PORT_RANGE)
+        try = 0
+        while try < DEFAULT_MAXIMUM_PORT_FIND_TRY
+          candidate = integer(range)
+
+          if port_available?(candidate)
+            break
+          else
+            try += 1
+          end
+        end
+        
+        raise "Flores.random_port: Cannot find an available port, tried #{DEFAULT_MAXIMUM_PORT_FIND_TRY} times, range was: #{range}" if try == DEFAULT_MAXIMUM_PORT_FIND_TRY
+
+        candidate
+      end
+      
+      def port_available?(port)
+        begin
+          server = TCPServer.new(port)
+          available = true
+        rescue # Assume that any errors can do this
+          available = false
+        ensure
+          server.close if server
+        end
+
+        return available
+      end
+    end
+  end
+
   module PKI
     DEFAULT_CERTIFICATE_OPTIONS = {
       :duration => Flores::Random.number(100..2000),

data/spec/support/integration_shared_context.rb

@@ -3,7 +3,7 @@ require "flores/random"
 
 shared_examples "send events" do
   it "successfully send the events" do
-    expect(queue.size).to eq(number_of_events), "Expected: #{number_of_events} got: #{queue.size}, execution output:\n #{@execution_output}"
+    wait(15).for { queue.size }.to eq(number_of_events), "Expected: #{number_of_events} got: #{queue.size}, execution output:\n #{@execution_output}" 
     expect(queue.collect { |e| e["message"] }).to eq(events)
   end
 end
@@ -16,11 +16,11 @@ end
 
 shared_context "beats configuration" do
   # common
-  let(:port) { Flores::Random.integer(1024..65335) }
+  let(:port) { Flores::Random.port }
   let(:host) { "localhost" }
 
   let(:queue) { [] }
-  let(:log_file) { write_to_tmp_file(events.join("\n") + "\n") } # make sure we end of line
+  let_tmp_file(:log_file) { events.join("\n") + "\n" } # make sure we end of line
   let(:number_of_events) { 5 }
   let(:event) { "Hello world" }
   let(:events) do
@@ -44,7 +44,11 @@ shared_context "beats configuration" do
     beats.register
 
     @server = Thread.new do
-      beats.run(queue)
+      begin
+        beats.run(queue)
+      rescue
+        retry unless beats.stop?
+      end
     end
     @server.abort_on_exception = true
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-beats
 version: !ruby/object:Gem::Version
-  version: 2.2.5
+  version: 2.2.7
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-29 00:00:00.000000000 Z
+date: 2016-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement