logstash-input-beats 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: df4aadb42d40ce79081da88eb7519460e82cd37f
-  data.tar.gz: 5239753151ef027acb05146920ba553c0101ce21
+  metadata.gz: 41c0bf7e7993d390543fd95c3c16ee3847d03862
+  data.tar.gz: d91cf9c2a4931d1290825e3a13bfc73d2ac8b2a3
 SHA512:
-  metadata.gz: 350918e94dc2ba03d0b9c57abab07902fb8b7ca45354213932db23f0c7477b91f89813ba83d5dcbce1c8f01406976608b50b0a286a1c573ec0e6a28b55a8c45f
-  data.tar.gz: 6c3f2ad8a8fec46786012b8df87f9c89e4500b5d675fb21bd9cf917f87bb53e44ef6bf9b95694dfb236622beb57adfd4b354317a280e049e7c65d72f4bbab5cb
+  metadata.gz: 82d7afdf4d5067032c6ea26eff33c32e7ddac9cad88d432bbb275b6a9813aaf274a124a78d23fad5f91f40858591ffa211c921a2b71b8b848924c90a35614a94
+  data.tar.gz: 050af93c89cef0649ce6761ee21118c1f69335e63a506009c44232460986b75d8bfb96e7441d1baa5a3e2bf2517a9f53a4f59f98fef91fd27863604933650baa

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 2.0.1
+  - Copy the `beat.hostname` field into the `host` field for better compatibility with the other Logstash plugins #28
+  - Correctly merge multiple line with the multiline codec ref: #24
 # 2.0.0
   - Add support for stream identity, the ID will be generated from beat.id+resource_id or beat.name + beat.source if not present #22 #13
     The identity allow the multiline codec to correctly merge string from multiples files.

data/lib/logstash/inputs/beats.rb

@@ -92,7 +92,8 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   end
 
   def run(output_queue)
-    start_buffer_broker(output_queue)
+    @output_queue = output_queue
+    start_buffer_broker
 
     while !stop? do
       # Wrapping the accept call into a CircuitBreaker
@@ -123,29 +124,42 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
 
   public
   def stop
-    @lumberjack.close
+    # we may have some stuff in the buffer
+    @codec.flush { |event| @output_queue << event }
+    @lumberjack.close rescue nil
   end
 
   public
-  def create_event(map, identity_stream)
+  def create_event(map, identity_stream, &block)
     # Filebeats uses the `message` key and LSF `line`
     target_field = target_field_for_codec ? map.delete(target_field_for_codec) : nil
 
     if target_field.nil?
       event = LogStash::Event.new(map)
+      copy_beat_hostname(event)
       decorate(event)
-      return event
+      block.call(event)
     else
       # All codecs expects to work on string
       @codec.decode(target_field.to_s, identity_stream) do |decoded|
         ts = coerce_ts(map.delete("@timestamp"))
         decoded["@timestamp"] = ts unless ts.nil?
         map.each { |k, v| decoded[k] = v }
+        copy_beat_hostname(decoded)
         decorate(decoded)
-        return decoded
+        block.call(decoded)
       end
     end
-    return nil
+  end
+
+  # Copies the beat.hostname field into the host field unless
+  # the host field is already defined
+  private
+  def copy_beat_hostname(event)
+    host = event["beat"] ? event["beat"]["hostname"] : nil
+    if host && event["host"].nil?
+      event["host"] = host
+    end
   end
 
   private
@@ -168,8 +182,7 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
         # If any errors occur in from the events the connection should be closed in the
         # library ensure block and the exception will be handled here
         connection.run do |map, identity_stream|
-          event = create_event(map, identity_stream)
-          block.call(event) unless event.nil?
+          create_event(map, identity_stream, &block)
         end
 
         # When too many errors happen inside the circuit breaker it will throw
@@ -192,10 +205,10 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   #
   # We are using a proxy queue supporting blocking with a timeout and
   # this thread take the element from one queue into another one.
-  def start_buffer_broker(output_queue)
+  def start_buffer_broker
     @threadpool.post do
       while !stop?
-        output_queue << @buffered_queue.pop_no_timeout
+        @output_queue << @buffered_queue.pop_no_timeout
       end
     end
   end

data/logstash-input-beats.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = "logstash-input-beats"
-  s.version         = "2.0.0"
+  s.version         = "2.0.1"
   s.licenses        = ["Apache License (2.0)"]
   s.summary         = "Receive events using the lumberjack protocol."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -22,7 +22,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
 
   s.add_runtime_dependency "logstash-codec-plain"
-  s.add_runtime_dependency "concurrent-ruby", "0.9.1"
+  s.add_runtime_dependency "concurrent-ruby", "~> 0.9.2"
   s.add_runtime_dependency "logstash-codec-multiline", "~> 2.0.3"
 
   s.add_development_dependency "flores", "~>0.0.6"
@@ -31,5 +31,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency "pry"
   s.add_development_dependency "rspec-wait"
   s.add_development_dependency "logstash-devutils", "~> 0.0.18"
+  s.add_development_dependency "logstash-codec-json"
 end
 

data/spec/inputs/beats_spec.rb

@@ -3,6 +3,7 @@ require_relative "../spec_helper"
 require "stud/temporary"
 require "logstash/inputs/beats"
 require "logstash/codecs/plain"
+require "logstash/codecs/json"
 require "logstash/codecs/multiline"
 require "logstash/event"
 require "lumberjack/beats/client"
@@ -90,10 +91,11 @@ describe LogStash::Inputs::Beats do
 
       context "without a `target_field` defined" do
         it "decorates the event" do
-          event = beats.create_event(event_map, identity_stream)
-          expect(event["foo"]).to eq("bar")
-          expect(event["[@metadata][hidden]"]).to eq("secret")
-          expect(event["tags"]).to include("bonjour")
+          beats.create_event(event_map, identity_stream) do |event|
+            expect(event["foo"]).to eq("bar")
+            expect(event["[@metadata][hidden]"]).to eq("secret")
+            expect(event["tags"]).to include("bonjour")
+          end
         end
       end
 
@@ -101,10 +103,11 @@ describe LogStash::Inputs::Beats do
         let(:event_map) { super.merge({"message" => "with a field"}) }
 
         it "decorates the event" do
-          event = beats.create_event(event_map, identity_stream)
-          expect(event["foo"]).to eq("bar")
-          expect(event["[@metadata][hidden]"]).to eq("secret")
-          expect(event["tags"]).to include("bonjour")
+          beats.create_event(event_map, identity_stream) do |event|
+            expect(event["foo"]).to eq("bar")
+            expect(event["[@metadata][hidden]"]).to eq("secret")
+            expect(event["tags"]).to include("bonjour")
+          end
         end
       end
 
@@ -112,9 +115,89 @@ describe LogStash::Inputs::Beats do
         let(:codec) { LogStash::Codecs::Multiline.new("pattern" => '^\s', "what" => "previous") }
         let(:event_map) { {"message" => "hello?", "tags" => ["syslog"]} }
 
-        it "retuns nil" do
+        it "returns nil" do
+          expect { |b| beats.create_event(event_map, identity_stream, &b) }.not_to yield_control
+        end
+      end
+
+      context "multiline" do
+        let(:codec) { LogStash::Codecs::Multiline.new("pattern" => '^2015', "what" => "previous", "negate" => true) }
+        let(:events_map) do
+          [
+            { "beat" => { "id" => "main", "resource_id" => "md5"},  "message" => "2015-11-10 10:14:38,907 line 1" },
+            { "beat" => { "id" => "main", "resource_id" => "md5"}, "message" => "line 1.1" },
+            { "beat" => { "id" => "main", "resource_id" => "md5"}, "message" => "2015-11-10 10:16:38,907 line 2" },
+            { "beat" => { "id" => "main", "resource_id" => "md5"}, "message" => "line 2.1" },
+            { "beat" => { "id" => "main", "resource_id" => "md5"}, "message" => "line 2.2" },
+            { "beat" => { "id" => "main", "resource_id" => "md5"}, "message" => "line 2.3" },
+            { "beat" => { "id" => "main", "resource_id" => "md5"}, "message" => "2015-11-10 10:18:38,907 line 3" }
+          ]
+        end
+
+        let(:queue) { [] }
+        before do
+          Thread.new { beats.run(queue) }
+          sleep(0.1)
+        end
+
+        it "should correctly merge multiple events" do
+          events_map.each { |map| beats.create_event(map, identity_stream) { |e| queue << e } }
+          # This cannot currently work without explicitely call a flush
+          # the flush is never timebased, if no new data is coming in we wont flush the buffer
+          # https://github.com/logstash-plugins/logstash-codec-multiline/issues/11
+          beats.stop
+          expect(queue.size).to eq(3)
+          
+          expect(queue.collect { |e| e["message"] }).to include("2015-11-10 10:14:38,907 line 1\nline 1.1",
+                                                           "2015-11-10 10:16:38,907 line 2\nline 2.1\nline 2.2\nline 2.3",
+                                                           "2015-11-10 10:18:38,907 line 3")
+        end
+      end
+
+      context "with a beat.hostname field" do
+        let(:event_map) { {"message" => "hello", "beat" => {"hostname" => "linux01"} } }
+
+        it "copies it to the host field" do
+          event = beats.create_event(event_map, identity_stream)
+          expect(event["host"]).to eq("linux01")
+        end
+      end
+
+      context "with a beat.hostname field but without the message" do
+        let(:event_map) { {"beat" => {"hostname" => "linux01"} } }
+
+        it "copies it to the host field" do
+          event = beats.create_event(event_map, identity_stream)
+          expect(event["host"]).to eq("linux01")
+        end
+      end
+
+      context "without a beat.hostname field" do
+        let(:event_map) { {"message" => "hello", "beat" => {"name" => "linux01"} } }
+
+        it "should not add a host field" do
+          event = beats.create_event(event_map, identity_stream)
+          expect(event["beat"]["name"]).to eq("linux01")
+          expect(event["host"]).to be_nil
+        end
+      end
+
+      context "with a beat.hostname and host fields" do
+        let(:event_map) { {"message" => "hello", "host" => "linux02", "beat" => {"hostname" => "linux01"} } }
+
+        it "should not overwrite host" do
+          event = beats.create_event(event_map, identity_stream)
+          expect(event["host"]).to eq("linux02")
+        end
+      end
+
+      context "with a host field in the message" do
+        let(:codec) { LogStash::Codecs::JSON.new }
+        let(:event_map) { {"message" => '{"host": "linux02"}', "beat" => {"hostname" => "linux01"} } }
+
+        it "should take the host from the JSON message" do
           event = beats.create_event(event_map, identity_stream)
-          expect(event).to be_nil
+          expect(event["host"]).to eq("linux02")
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-beats
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-17 00:00:00.000000000 Z
+date: 2015-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -48,14 +48,14 @@ dependencies:
   name: concurrent-ruby
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.9.1
+        version: 0.9.2
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.9.1
+        version: 0.9.2
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -156,37 +156,51 @@ dependencies:
         version: 0.0.18
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- PROTOCOL.md
+- README.md
 - lib/logstash/circuit_breaker.rb
-- lib/logstash/sized_queue_timeout.rb
 - lib/logstash/inputs/beats.rb
+- lib/logstash/sized_queue_timeout.rb
 - lib/lumberjack/beats.rb
 - lib/lumberjack/beats/client.rb
 - lib/lumberjack/beats/server.rb
-- spec/spec_helper.rb
-- spec/integration_spec.rb
+- logstash-input-beats.gemspec
 - spec/inputs/beats_spec.rb
+- spec/integration_spec.rb
 - spec/logstash/circuit_breaker_spec.rb
 - spec/logstash/size_queue_timeout_spec.rb
 - spec/lumberjack/beats/acking_protocol_v1_spec.rb
 - spec/lumberjack/beats/acking_protocol_v2_spec.rb
 - spec/lumberjack/beats/client_spec.rb
-- spec/lumberjack/beats/server_spec.rb
 - spec/lumberjack/beats/connection_spec.rb
+- spec/lumberjack/beats/server_spec.rb
+- spec/spec_helper.rb
 - spec/support/logstash_test.rb
-- logstash-input-beats.gemspec
-- PROTOCOL.md
-- README.md
-- CHANGELOG.md
-- CONTRIBUTORS
-- Gemfile
-- LICENSE
-- NOTICE.TXT
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -209,19 +223,19 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Receive events using the lumberjack protocol.
 test_files:
-- spec/spec_helper.rb
-- spec/integration_spec.rb
 - spec/inputs/beats_spec.rb
+- spec/integration_spec.rb
 - spec/logstash/circuit_breaker_spec.rb
 - spec/logstash/size_queue_timeout_spec.rb
 - spec/lumberjack/beats/acking_protocol_v1_spec.rb
 - spec/lumberjack/beats/acking_protocol_v2_spec.rb
 - spec/lumberjack/beats/client_spec.rb
-- spec/lumberjack/beats/server_spec.rb
 - spec/lumberjack/beats/connection_spec.rb
+- spec/lumberjack/beats/server_spec.rb
+- spec/spec_helper.rb
 - spec/support/logstash_test.rb