logstash-input-beats 2.2.7 → 2.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0c9f5e238355c46a4cf8de5a79ab788941bfd524
-  data.tar.gz: c40c248be28f9c7a28764761340dd0d926f6cdb6
+  metadata.gz: c1727d19341688ab4551c91893080ea3c09fb0ae
+  data.tar.gz: 8065ae0fd7bbecb54a4fd07fc32dad7290cc314d
 SHA512:
-  metadata.gz: 12996169fca9958102f280cb5d532cb17717f940ac62f4d7deaa19a851b875817e8da5432fd0fec5733971b7de9de4d2b2fec498bb9f8f281bbb7d54b6a056bb
-  data.tar.gz: 1a3a6d86aaf2f30a63524d98664b1abdce873e03e1dca0b180c2650314f19f6229730cbcfa4902d3eb8b0a2b14e1b43c8c04c3133399dee6921ab64a46fecbec
+  metadata.gz: 010ce24ae384e93f786461e6cc0189b6fc34151fd07a8934763e121e341d5aff863d0feb719e1f855509560d070d9f3e24b9d267633cea2caa41fa29263ddaec
+  data.tar.gz: 8f29fe0aa689b94a6a4fb75e4622a38b031a6695443d1d0e647eef4d8fa2c845e3e3261b33f4eb7901bef88cba45c44665278e478f960b8007d8701ba21ceb58

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 2.2.8
+  - Fix #73 Bug in EventTransformCommon#codec_name, use config_name
+  - Add regression test for fix to #73
+  - Non deterministic error for the LSF integration test
+  - Make this plugin really a drop in replacement for the lumberjack input, so LSF can send their events to this plugin.
 # 2.2.7
   - More robust test when using a random port #60
   - Fix LSF integration tests #52

data/CONTRIBUTORS

@@ -9,6 +9,7 @@ Contributors:
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * Suyog Rao (suyograo)
+* Guy Boertje (guyboertje)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/README.md

@@ -1,7 +1,6 @@
 # Logstash Plugin
 
-[![Build
-Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-beats-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-beats-unit/)
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-beats.svg)](https://travis-ci.org/logstash-plugins/logstash-input-beats)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/lib/logstash/inputs/beats.rb

@@ -95,7 +95,7 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   config :congestion_threshold, :validate => :number, :default => 5
 
   # This is the default field that the specified codec will be applied
-  config :target_field_for_codec, :validate => :string, :default => "message"
+  config :target_field_for_codec, :validate => :string, :default => "message", :deprecated => "This option is now deprecated, the plugin is now compatible with Filebeat and Logstash-Forwarder"
 
   # TODO(sissel): Add CA to authenticate clients with.
   RECONNECT_BACKOFF_SLEEP = 0.5

data/lib/logstash/inputs/beats_support/connection_handler.rb

@@ -2,6 +2,7 @@
 require "logstash/inputs/beats"
 require "logstash/inputs/beats_support/decoded_event_transform"
 require "logstash/inputs/beats_support/raw_event_transform"
+require "lumberjack/beats"
 
 module LogStash::Inputs::BeatsSupport
   # Handle the data coming from a connection
@@ -40,7 +41,11 @@ module LogStash::Inputs::BeatsSupport
                                       :peer => @connection.peer)
 
       # Filebeats uses the `message` key and LSF `line`
-      target_field = @input.target_field_for_codec ? hash.delete(@input.target_field_for_codec) : nil
+      target_field = if from_filebeat?(hash)
+                       hash.delete(Lumberjack::Beats::FILEBEAT_LOG_LINE_FIELD)
+                     elsif from_logstash_forwarder?(hash)
+                       hash.delete(Lumberjack::Beats::LSF_LOG_LINE_FIELD)
+                     end
 
       if target_field.nil?
         @logger.debug? && @logger.debug("Beats input: not using the codec for this event, can't find the codec target field",
@@ -75,5 +80,14 @@ module LogStash::Inputs::BeatsSupport
 
       @codec.flush(&block)
     end
+
+    private
+    def from_filebeat?(hash)
+      !hash[Lumberjack::Beats::FILEBEAT_LOG_LINE_FIELD].nil?
+    end
+
+    def from_logstash_forwarder?(hash)
+      !hash[Lumberjack::Beats::LSF_LOG_LINE_FIELD].nil?
+    end
   end
 end

data/lib/logstash/inputs/beats_support/event_transform_common.rb

@@ -33,7 +33,7 @@ module LogStash::Inputs::BeatsSupport
 
     def codec_name
       @codec_name ||= if @input.codec.respond_to?(:base_codec)
-                        @input.codec.base_codec.class.config
+                        @input.codec.base_codec.class.config_name
                       else
                         @input.codec.class.config_name
                       end

data/lib/lumberjack/beats.rb

@@ -1,6 +1,8 @@
 # encoding: utf-8
 require "json"
 module Lumberjack module Beats
+  FILEBEAT_LOG_LINE_FIELD = "message".freeze
+  LSF_LOG_LINE_FIELD = "line".freeze
   SEQUENCE_MAX = (2**32-1).freeze
 
   @@json = Class.new do

data/lib/lumberjack/beats/server.rb

@@ -467,7 +467,7 @@ module Lumberjack module Beats
           reset_next_ack(*args)
         when :data
           sequence, map = args
-          ack_if_needed(sequence) { data(map, &block) }
+          ack_if_needed(sequence) { data(normalize_v1_metadata_encoding(map), &block) }
         when :json
           # If the payload is an array of items we will emit multiple events
           # this behavior was moved from the plugin to the library.
@@ -485,6 +485,15 @@ module Lumberjack module Beats
       end
     end
 
+    def normalize_v1_metadata_encoding(map)
+      # lets normalize the metadata of the v1 frame to make
+      # sure everything is in utf-8 format, because LSF don't enforce the encoding when he send
+      # the data to the server. Path, offset can be in another encoding, when the data is assigned to the event.
+      # the event will validate it and crash when the encoding is in the wrong format.
+      map.each { |k, v| map[k].force_encoding(Encoding::UTF_8) unless k == Lumberjack::Beats::LSF_LOG_LINE_FIELD }
+      map
+    end
+
     def version(version)
       @version = version
     end

data/logstash-input-beats.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = "logstash-input-beats"
-  s.version         = '2.2.7'
+  s.version         = '2.2.8'
   s.licenses        = ["Apache License (2.0)"]
   s.summary         = "Receive events using the lumberjack protocol."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/beats_spec.rb

@@ -10,8 +10,8 @@ require "lumberjack/beats/client"
 
 describe LogStash::Inputs::Beats do
   let(:connection) { double("connection") }
-  let(:certificate) { LogStashTest.certificate }
-  let(:port) { LogStashTest.random_port }
+  let(:certificate) { BeatsInputTest.certificate }
+  let(:port) { BeatsInputTest.random_port }
   let(:queue)  { Queue.new }
   let(:config)   { { "port" => 0, "ssl_certificate" => certificate.ssl_cert, "ssl_key" => certificate.ssl_key, "type" => "example", "tags" => "beats"} }
 
@@ -97,8 +97,8 @@ describe LogStash::Inputs::Beats do
   context "#handle_new_connection" do
     let(:config) {{ "ssl" => false, "port" => 0, "type" => "example", "tags" => "beats" }}
     let(:plugin) { LogStash::Inputs::Beats.new(config) }
-    let(:connection) { DummyConnection.new(events) }
-    let(:buffer_queue) { DummyNeverBlockedQueue.new }
+    let(:connection) { BeatsInputTest::DummyConnection.new(events) }
+    let(:buffer_queue) { BeatsInputTest::DummyNeverBlockedQueue.new }
     let(:pipeline_queue)  { [] }
     let(:events) {
       [

data/spec/inputs/beats_support/connection_handler_spec.rb

@@ -11,7 +11,7 @@ describe LogStash::Inputs::BeatsSupport::ConnectionHandler do
       "tags" => "beats"
     }
   end
-
+  # logger is not used and DummyLogger needs implementing
   let(:logger) { DummyLogger.new  }
   let(:input) do
     LogStash::Inputs::Beats.new(config).tap do |i|
@@ -19,12 +19,12 @@ describe LogStash::Inputs::BeatsSupport::ConnectionHandler do
     end
   end
   let(:connection) { double("connection") }
-  let(:queue) { DummyNeverBlockedQueue.new }
+  let(:queue) { BeatsInputTest::DummyNeverBlockedQueue.new }
 
   subject { described_class.new(connection, input, queue) }
-  
+
   context "#accept" do
-    let(:connection) { DummyConnection.new(events) }
+    let(:connection) { BeatsInputTest::DummyConnection.new(events) }
     let(:events) {
       [
         { :map => { "id" => 1 }, :identity_stream => "/var/log/message" },
@@ -59,7 +59,7 @@ describe LogStash::Inputs::BeatsSupport::ConnectionHandler do
       context "queue is blocked" do
         let(:queue_timeout) { 1 }
         let(:queue) { LogStash::Inputs::BeatsSupport::SynchronousQueueWithOffer.new(queue_timeout) }
-      
+
         it "raise an exception" do
           expect { subject.process(map, identity_stream) }.to raise_error(LogStash::Inputs::Beats::InsertingToQueueTakeTooLong)
         end

data/spec/inputs/beats_support/decoded_event_transform_spec.rb

@@ -64,4 +64,11 @@ describe LogStash::Inputs::BeatsSupport::DecodedEventTransform do
       expect(subject["@timestamp"]).to be_kind_of(LogStash::Timestamp)
     end
   end
+
+  context "when the codec is a base_codec wrapper" do
+    before { config.update("codec" => BeatsInputTest::DummyCodec.new) }
+    it "gets the codec config name from the base codec" do
+      expect(subject["tags"]).to include("beats_input_codec_dummy_applied")
+    end
+  end
 end

data/spec/integration/logstash_forwarder_spec.rb

@@ -19,10 +19,11 @@ describe "Logstash-Forwarder", :integration => true do
     end
   end
 
-  let(:client_wait_time) { 5 }
   include FileHelpers
   include_context "beats configuration"
 
+  let(:client_wait_time) { 5 }
+
   # Filebeat related variables
   let(:cmd) { [lsf_exec, "-config", lsf_config_path] }
 
@@ -53,6 +54,7 @@ describe "Logstash-Forwarder", :integration => true do
     File.open(log_file, "a") do |f|
       f.write(events.join("\n") + "\n")
     end
+    sleep(1) # give some time to the clients to pick up the changes
   end
 
   after :each do
@@ -76,7 +78,6 @@ describe "Logstash-Forwarder", :integration => true do
           "ssl" => true,
           "ssl_certificate" => certificate_file,
           "ssl_key" => certificate_key_file,
-          "target_field_for_codec" => "line"
         })
       end
 

data/spec/support/integration_shared_context.rb

@@ -3,7 +3,7 @@ require "flores/random"
 
 shared_examples "send events" do
   it "successfully send the events" do
-    wait(15).for { queue.size }.to eq(number_of_events), "Expected: #{number_of_events} got: #{queue.size}, execution output:\n #{@execution_output}" 
+    wait(20).for { queue.size }.to eq(number_of_events), "Expected: #{number_of_events} got: #{queue.size}, execution output:\n #{@execution_output}" 
     expect(queue.collect { |e| e["message"] }).to eq(events)
   end
 end
@@ -43,10 +43,21 @@ shared_context "beats configuration" do
   before :each do
     beats.register
 
+    @abort_on_exception = Thread.abort_on_exception
+    Thread.abort_on_exception = true
+
     @server = Thread.new do
       begin
+        # use to know what lumberjack is actually doing
+        if ENV["DEBUG"]
+          logger = Logger.new(STDOUT)
+          beats.logger = Cabin::Channel.new
+          beats.logger.subscribe(logger)
+          beats.logger.level = :debug
+        end
+
         beats.run(queue)
-      rescue
+      rescue => e
         retry unless beats.stop?
       end
     end
@@ -55,5 +66,8 @@ shared_context "beats configuration" do
     sleep(1) while @server.status != "run"
   end
 
-  after(:each) { beats.stop }
+  after(:each) do
+    beats.stop
+    Thread.abort_on_exception = @abort_on_exception
+  end
 end

data/spec/support/logstash_test.rb

@@ -1,5 +1,9 @@
 require "stud/temporary"
-module LogStashTest
+
+
+# namespace the Dummy* classes, they are reused names
+# use a more specific module name to prevent clashes
+module BeatsInputTest
   class Certicate
     attr_reader :ssl_key, :ssl_cert
 
@@ -20,29 +24,43 @@ module LogStashTest
       rand(2000..10000)
     end
   end
-end
 
-class DummyNeverBlockedQueue < Array
-  def offer(element, timeout = nil)
-    push(element)
+  class DummyNeverBlockedQueue < Array
+    def offer(element, timeout = nil)
+      push(element)
+    end
+
+    alias_method :take, :shift
   end
 
-  alias_method :take, :shift
-end
+  class DummyConnection
+    def initialize(events)
+      @events = events
+    end
 
-class DummyConnection
-  def initialize(events)
-    @events = events
-  end
+    def run
+      @events.each do |element|
+        yield element[:map], element[:identity_stream]
+      end
+    end
 
-  def run
-    @events.each do |element|
-      yield element[:map], element[:identity_stream] 
+    def peer
+      "localhost:5555"
     end
   end
 
-  def peer
-    "localhost:5555"
+  class DummyCodec
+    def register() end
+    def decode(*) end
+    def clone() self; end
+    def base_codec
+      self
+    end
+    def self.config_name
+      "dummy"
+    end
   end
 end
 
+
+

metadata

@@ -1,203 +1,203 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-beats
 version: !ruby/object:Gem::Version
-  version: 2.2.7
+  version: 2.2.8
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-31 00:00:00.000000000 Z
+date: 2016-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '1.0'
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 0.9.2
+    - - <=
+      - !ruby/object:Gem::Version
+        version: 1.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
-    - - "<="
+    - - <=
       - !ruby/object:Gem::Version
         version: 1.0.0
-  name: concurrent-ruby
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: thread_safe
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.9.2
-    - - "<="
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-- !ruby/object:Gem::Dependency
+        version: 0.3.5
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.3.5
-  name: thread_safe
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-multiline
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.3.5
-- !ruby/object:Gem::Dependency
+        version: 2.0.5
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.0.5
-  name: logstash-codec-multiline
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: flores
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.5
-- !ruby/object:Gem::Dependency
+        version: 0.0.6
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.6
-  name: flores
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: rspec
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.6
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: stud
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: stud
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: pry
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: pry
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: rspec-wait
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec-wait
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 0.0.18
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.18
-  name: logstash-devutils
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.18
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-json
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: childprocess
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: childprocess
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -257,12 +257,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []