logstash-input-beats 0.9.6 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1aed7cc18f6010a5e6e40d4dfa62c073dc3bd512
-  data.tar.gz: ed5565a612454c4905fa48edc7c6482af02a5008
+  metadata.gz: df4aadb42d40ce79081da88eb7519460e82cd37f
+  data.tar.gz: 5239753151ef027acb05146920ba553c0101ce21
 SHA512:
-  metadata.gz: fa7854d4255d44ca06916883f3cc24e78b0c778d6503d9d3b91c37d3b5e99e769474e12224ce5d226eaa4b56dca83083f1cd1d57a499557b3960b550a0d714e1
-  data.tar.gz: c423e548a02defb63de98ea6f65a3e339a0f55423d19b08b92af71fb033ebe8da7e1c0fd3add8b31f84b6c1f73699a57893fda9b6205ab3177d47656ba599edb
+  metadata.gz: 350918e94dc2ba03d0b9c57abab07902fb8b7ca45354213932db23f0c7477b91f89813ba83d5dcbce1c8f01406976608b50b0a286a1c573ec0e6a28b55a8c45f
+  data.tar.gz: 6c3f2ad8a8fec46786012b8df87f9c89e4500b5d675fb21bd9cf917f87bb53e44ef6bf9b95694dfb236622beb57adfd4b354317a280e049e7c65d72f4bbab5cb

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 2.0.0
+  - Add support for stream identity, the ID will be generated from beat.id+resource_id or beat.name + beat.source if not present #22 #13
+    The identity allow the multiline codec to correctly merge string from multiples files.
 # 0.9.6
   - Fix an issue with rogue events created by buffered codecs #19
 # 0.9.5

data/lib/logstash/inputs/beats.rb

@@ -2,9 +2,9 @@
 require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/timestamp"
-require "logstash/compatibility_layer_api_v1"
 require "lumberjack/beats"
 require "lumberjack/beats/server"
+require "logstash/codecs/identity_map_codec"
 
 # use Logstash provided json decoder
 Lumberjack::Beats::json = LogStash::Json
@@ -14,8 +14,6 @@ Lumberjack::Beats::json = LogStash::Json
 # https://github.com/elastic/filebeat[filebeat]
 #
 class LogStash::Inputs::Beats < LogStash::Inputs::Base
-  include LogStash::CompatibilityLayerApiV1
-
   config_name "beats"
 
   default :codec, "plain"
@@ -50,7 +48,7 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   # TODO(sissel): Add CA to authenticate clients with.
   BUFFERED_QUEUE_SIZE = 1
   RECONNECT_BACKOFF_SLEEP = 0.5
-  
+
   def register
     require "concurrent"
     require "logstash/circuit_breaker"
@@ -69,7 +67,7 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
       :ssl_key_passphrase => @ssl_key_passphrase)
 
     # Create a reusable threadpool, we do not limit the number of connections
-    # to the input, the circuit breaker with the timeout should take care 
+    # to the input, the circuit breaker with the timeout should take care
     # of `blocked` threads and prevent logstash to go oom.
     @threadpool = Concurrent::CachedThreadPool.new(:idletime => 15)
 
@@ -80,6 +78,9 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
     @circuit_breaker = LogStash::CircuitBreaker.new("Beats input",
                             :exceptions => [LogStash::SizedQueueTimeout::TimeoutError])
 
+    # wrap the configured codec to support identity stream
+    # from the producers
+    @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
   end # def register
 
   def ssl_configured?
@@ -99,7 +100,7 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
         connection = @lumberjack.accept # call that creates a new connection
         next if connection.nil? # if the connection is nil the connection was close.
 
-        invoke(connection, @codec.clone) do |event|
+        invoke(connection) do |event|
           if stop?
             connection.close
             break
@@ -126,17 +127,17 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   end
 
   public
-  def create_event(codec, map)
+  def create_event(map, identity_stream)
     # Filebeats uses the `message` key and LSF `line`
     target_field = target_field_for_codec ? map.delete(target_field_for_codec) : nil
 
     if target_field.nil?
-      event = LogStash::Event.new(map) 
+      event = LogStash::Event.new(map)
       decorate(event)
       return event
     else
       # All codecs expects to work on string
-      @codec.decode(target_field.to_s) do |decoded|
+      @codec.decode(target_field.to_s, identity_stream) do |decoded|
         ts = coerce_ts(map.delete("@timestamp"))
         decoded["@timestamp"] = ts unless ts.nil?
         map.each { |k, v| decoded[k] = v }
@@ -161,13 +162,13 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   end
 
   private
-  def invoke(connection, codec, &block)
+  def invoke(connection, &block)
     @threadpool.post do
       begin
         # If any errors occur in from the events the connection should be closed in the
         # library ensure block and the exception will be handled here
-        connection.run do |map|
-          event = create_event(codec, map)
+        connection.run do |map, identity_stream|
+          event = create_event(map, identity_stream)
           block.call(event) unless event.nil?
         end
 

data/lib/lumberjack/beats/server.rb

@@ -150,17 +150,17 @@ module Lumberjack module Beats
     end # def transition
 
     # Feed data to this parser.
-    # 
+    #
     # Currently, it will return the raw payload of websocket messages.
     # Otherwise, it returns nil if no complete message has yet been consumed.
     #
-    # @param [String] the string data to feed into the parser. 
+    # @param [String] the string data to feed into the parser.
     # @return [String, nil] the websocket message payload, if any, nil otherwise.
     def feed(data, &block)
       @buffer << data
       #p :need => @need
       while have?(@need)
-        send(@state, &block) 
+        send(@state, &block)
         #case @state
         #when :header; header(&block)
         #when :window_size; window_size(&block)
@@ -310,6 +310,10 @@ module Lumberjack module Beats
       @ack_handler = nil
     end
 
+    def peer
+      "#{@fd.peeraddr[3]}:#{@fd.peeraddr[1]}"
+    end
+
     def run(&block)
       while !server.closed?
         read_socket(&block)
@@ -375,17 +379,34 @@ module Lumberjack module Beats
     end
 
     def data(map, &block)
-      block.call(map) if block_given?
+      block.call(map, identity_stream(map)) if block_given?
     end
 
     def reset_next_ack(window_size)
-      klass = (@version == Parser::PROTOCOL_VERSION_1) ? AckingProtocolV1 : AckingProtocolV2
+      klass = version_1? ? AckingProtocolV1 : AckingProtocolV2
       @ack_handler = klass.new(window_size)
     end
 
     def send_ack(sequence)
       @fd.syswrite(@ack_handler.ack_frame(sequence))
     end
+
+    def version_1?
+      @version == Parser::PROTOCOL_VERSION_1
+    end
+
+    def identity_stream(map)
+      id = map.fetch("beat", {})["id"]
+
+      if id && map["resource_id"]
+        identity_values = [id, map["resource_id"]]
+      else
+        identity_values = [map.fetch("beat", {})["name"],
+                          map["source"]]
+      end
+
+      identity_values.compact.join("-")
+    end
   end # class Connection
 
   class AckingProtocolV1
@@ -395,7 +416,7 @@ module Lumberjack module Beats
     end
 
     def ack?(sequence)
-      # The first encoded event will contain the sequence number 
+      # The first encoded event will contain the sequence number
       # this is needed to know when we should ack.
       @next_ack = compute_next_ack(sequence) if @next_ack.nil?
       sequence == @next_ack

data/logstash-input-beats.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = "logstash-input-beats"
-  s.version         = "0.9.6"
+  s.version         = "2.0.0"
   s.licenses        = ["Apache License (2.0)"]
   s.summary         = "Receive events using the lumberjack protocol."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,10 +19,11 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 1.5.4", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
 
   s.add_runtime_dependency "logstash-codec-plain"
   s.add_runtime_dependency "concurrent-ruby", "0.9.1"
+  s.add_runtime_dependency "logstash-codec-multiline", "~> 2.0.3"
 
   s.add_development_dependency "flores", "~>0.0.6"
   s.add_development_dependency "rspec"
@@ -30,6 +31,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency "pry"
   s.add_development_dependency "rspec-wait"
   s.add_development_dependency "logstash-devutils", "~> 0.0.18"
-  s.add_development_dependency "logstash-codec-multiline"
 end
 

data/spec/inputs/beats_spec.rb

@@ -2,7 +2,6 @@
 require_relative "../spec_helper"
 require "stud/temporary"
 require "logstash/inputs/beats"
-require "logstash/compatibility_layer_api_v1"
 require "logstash/codecs/plain"
 require "logstash/codecs/multiline"
 require "logstash/event"
@@ -79,31 +78,19 @@ describe LogStash::Inputs::Beats do
         "type" => "example", "codec" => codec }
     end
 
-
-    context "#codecs" do
-      let(:lines) { {"line" => "one\ntwo\n  two.2\nthree\n", "tags" => ["syslog"]} }
-
-      before do
-        allow(connection).to receive(:run).and_yield(lines)
-        beats.register
-        expect_any_instance_of(Lumberjack::Beats::Server).to receive(:accept).and_return(connection)
-      end
-      
-      it "clone the codec per connection" do
-        expect(beats.codec).to receive(:clone).once
-        expect(beats).to receive(:invoke) { break }
-        beats.run(queue)
-      end
+    before do
+      beats.register
     end
 
     context "#create_event" do
       let(:config) { super.merge({ "add_field" => { "foo" => "bar", "[@metadata][hidden]" => "secret"}, "tags" => ["bonjour"]}) }
       let(:event_map) { { "hello" => "world" } }
       let(:codec) { LogStash::Codecs::Plain.new }
+      let(:identity_stream) { "custom-type-input_type-source" }
 
       context "without a `target_field` defined" do
         it "decorates the event" do
-          event = beats.create_event(codec, event_map)
+          event = beats.create_event(event_map, identity_stream)
           expect(event["foo"]).to eq("bar")
           expect(event["[@metadata][hidden]"]).to eq("secret")
           expect(event["tags"]).to include("bonjour")
@@ -114,7 +101,7 @@ describe LogStash::Inputs::Beats do
         let(:event_map) { super.merge({"message" => "with a field"}) }
 
         it "decorates the event" do
-          event = beats.create_event(beats.codec, event_map)
+          event = beats.create_event(event_map, identity_stream)
           expect(event["foo"]).to eq("bar")
           expect(event["[@metadata][hidden]"]).to eq("secret")
           expect(event["tags"]).to include("bonjour")
@@ -126,16 +113,14 @@ describe LogStash::Inputs::Beats do
         let(:event_map) { {"message" => "hello?", "tags" => ["syslog"]} }
 
         it "retuns nil" do
-          event = beats.create_event(beats.codec, event_map)
+          event = beats.create_event(event_map, identity_stream)
           expect(event).to be_nil
         end
       end
     end
   end
 
-  unless LogStash::CompatibilityLayerApiV1.is_v1?
-    context "when interrupting the plugin" do
-      it_behaves_like "an interruptible input plugin"
-    end
+  context "when interrupting the plugin" do
+    it_behaves_like "an interruptible input plugin"
   end
 end

data/spec/integration_spec.rb

@@ -33,7 +33,7 @@ describe "A client" do
         tcp_server.accept do |socket|
           con = Lumberjack::Beats::Connection.new(socket, tcp_server)
           begin
-            con.run { |data| queue << data }
+            con.run { |data, identity_stream| queue << [data, identity_stream] }
           rescue
             # Close connection on failure. For example SSL client will make
             # parser for TCP based server trip.
@@ -44,7 +44,7 @@ describe "A client" do
     end
 
     @ssl_server = Thread.new do
-      ssl_server.run { |data| queue << data }
+      ssl_server.run { |data, identity_stream| queue << [data, identity_stream] }
     end
   end
 
@@ -55,6 +55,7 @@ describe "A client" do
       end
       sleep(0.5) # give time to the server to read the events
       expect(queue.size).to eq(random_number_of_events)
+      expect(queue.collect(&:last).uniq.size).to eq(1)
     end
 
     it "support sending multiple elements in one payload" do
@@ -62,7 +63,7 @@ describe "A client" do
       sleep(0.5)
 
       expect(queue.size).to eq(batch_size)
-      expect(queue).to match_array(batch_payload)
+      expect(queue.collect(&:last).uniq.size).to eq(1)
     end
   end
 
@@ -106,7 +107,7 @@ describe "A client" do
         expect(client.write(batch_payload)).to eq(batch_size / 2)
         sleep(0.5)
         expect(queue.size).to eq(batch_size)
-        expect(queue).to match_array(batch_payload)
+        expect(queue.collect(&:first)).to match_array(batch_payload)
       end
     end
   end
@@ -152,7 +153,7 @@ describe "A client" do
   context "using ssl encrypted connection" do
     context "with a valid certificate" do
       it "successfully connect to the server" do
-        expect { 
+        expect {
           Lumberjack::Beats::Client.new(:port => port,
                                  :host => host,
                                  :addresses => host,
@@ -161,7 +162,7 @@ describe "A client" do
       end
 
       it "should fail connecting to plain tcp server" do
-        expect { 
+        expect {
           Lumberjack::Beats::Client.new(:port => tcp_port,
                                  :host => host,
                                  :addresses => host,

data/spec/lumberjack/beats/connection_spec.rb

@@ -4,6 +4,8 @@ require "spec_helper"
 require "flores/random"
 
 describe "Connnection" do
+  let(:ip) { "192.168.1.2" }
+  let(:port) { 4444 }
   let(:server) { double("server", :closed? => false) }
   let(:socket) { double("socket", :closed? => false) }
   let(:connection) { Lumberjack::Beats::Connection.new(socket, server) }
@@ -11,6 +13,55 @@ describe "Connnection" do
   let(:start_sequence) { Flores::Random.integer(0..2000) }
   let(:random_number_of_events) { Flores::Random.integer(2..200) }
 
+  subject { Lumberjack::Beats::Connection.new(socket, server) }
+
+  before do
+    allow(socket).to receive(:peeraddr).and_return(["AF_INET", port, "test.elastic.co", ip])
+  end
+
+  context "#peer" do
+    let(:socket) { double("socket", :closed? => false) }
+
+    it "return the ip and the port" do
+      expect(subject.peer).to eq("#{ip}:#{port}")
+    end
+  end
+
+  context "#identity_stream" do
+    let(:map) { { "message" => "Hello world" } }
+
+    context "with a map containing the beats.id and the file_id" do
+      let(:map) { super.merge({
+        "beat" => { "name" => "testing-host", "id" => "abc1234" },
+        "type" => "log",
+        "resource_id" => "123",
+        "input_type" => "propector",
+        "source" => "/var/log/message" }) }
+
+      it "generate a identity stream from an event start" do
+        expect(subject.identity_stream(map)).to eq("#{map["beat"]["id"]}-#{map["resource_id"]}")
+      end
+    end
+
+    context "with a map containing all the information" do
+      let(:map) { super.merge({
+        "beat" => { "name" => "testing-host" },
+        "type" => "log",
+        "input_type" => "propector",
+        "source" => "/var/log/message" }) }
+
+      it "generate a identity stream from an event start" do
+        expect(subject.identity_stream(map)).to eq("#{map["beat"]["name"]}-#{map["source"]}")
+      end
+    end
+
+    context "with a map containing no information" do
+      it "use the ip and the port" do
+        expect(subject.identity_stream(map)).to eq("")
+      end
+    end
+  end
+
   context "when the server is running" do
     before do
       allow(socket).to receive(:sysread).with(Lumberjack::Beats::Connection::READ_SIZE).and_return("")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-beats
 version: !ruby/object:Gem::Version
-  version: 0.9.6
+  version: 2.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-06 00:00:00.000000000 Z
+date: 2015-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.5.4
+        version: 2.0.0
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -24,7 +24,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.5.4
+        version: 2.0.0
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -58,6 +58,20 @@ dependencies:
         version: 0.9.1
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-multiline
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.3
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: flores
   version_requirements: !ruby/object:Gem::Requirement
@@ -142,20 +156,6 @@ dependencies:
         version: 0.0.18
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-multiline
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  prerelease: false
-  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -163,14 +163,13 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/logstash/circuit_breaker.rb
-- lib/logstash/compatibility_layer_api_v1.rb
 - lib/logstash/sized_queue_timeout.rb
 - lib/logstash/inputs/beats.rb
 - lib/lumberjack/beats.rb
 - lib/lumberjack/beats/client.rb
 - lib/lumberjack/beats/server.rb
-- spec/integration_spec.rb
 - spec/spec_helper.rb
+- spec/integration_spec.rb
 - spec/inputs/beats_spec.rb
 - spec/logstash/circuit_breaker_spec.rb
 - spec/logstash/size_queue_timeout_spec.rb
@@ -215,8 +214,8 @@ signing_key:
 specification_version: 4
 summary: Receive events using the lumberjack protocol.
 test_files:
-- spec/integration_spec.rb
 - spec/spec_helper.rb
+- spec/integration_spec.rb
 - spec/inputs/beats_spec.rb
 - spec/logstash/circuit_breaker_spec.rb
 - spec/logstash/size_queue_timeout_spec.rb

data/lib/logstash/compatibility_layer_api_v1.rb

@@ -1,28 +0,0 @@
-# encoding: utf-8
-require "logstash/version"
-require "gems"
-
-# This module allow this plugin to work with the v1 API.
-module LogStash::CompatibilityLayerApiV1
-  LOGSTASH_CORE_VERSION = Gem::Version.new(LOGSTASH_VERSION)
-  V2_VERSION = Gem::Version.new("2.0.0.beta2")
-
-  def self.included(base)
-    base.send(:include, InstanceMethods) if self.is_v1?
-  end
-
-  def self.is_v1?
-    LOGSTASH_CORE_VERSION < V2_VERSION
-  end
-  
-  # This allow this plugin to work both in V1 and v2 of logstash-core
-  module InstanceMethods
-    def stop?
-      false
-    end
-
-    def teardown
-      stop
-    end
-  end
-end