logstash-input-beats 3.1.0.beta1-java

data/lib/logstash/inputs/beats/codec_callback_listener.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+require "logstash/inputs/beats"
+
+module LogStash module Inputs class Beats
+  # Use the new callback based approch instead of using blocks
+  # so we can retain some context of the execution, and make it easier to test
+  class CodecCallbackListener
+    attr_accessor :data
+    # The path acts as the `stream_identity`, 
+    # usefull when the clients is reading multiples files
+    attr_accessor :path
+
+    def initialize(data, hash, path, transformer, queue)
+      @data = data
+      @hash = hash 
+      @path = path
+      @queue = queue
+      @transformer = transformer
+    end
+
+    def process_event(event)
+      @transformer.transform(event, @hash)
+      @queue << event
+    end
+  end
+end; end; end

data/lib/logstash/inputs/beats/decoded_event_transform.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+require "logstash/inputs/beats/event_transform_common"
+module LogStash module Inputs class Beats
+  # Take the extracted content from the codec, merged with the other data coming
+  # from beats, apply the configured tags, normalize the host and try to coerce
+  # the timestamp if it was provided in the hash.
+  class DecodedEventTransform < EventTransformCommon
+    def transform(event, hash)
+      ts = coerce_ts(hash.delete("@timestamp"))
+
+      event.set("@timestamp", ts) unless ts.nil?
+      hash.each { |k, v| event.set(k, v) }
+      super(event)
+      event.tag("beats_input_codec_#{codec_name}_applied")
+      event
+    end
+
+    private
+    def coerce_ts(ts)
+      return nil if ts.nil?
+      timestamp = LogStash::Timestamp.coerce(ts)
+
+      return timestamp if timestamp
+
+      @logger.warn("Unrecognized @timestamp value, setting current time to @timestamp",
+                   :value => ts.inspect)
+      return nil
+    rescue LogStash::TimestampParserError => e
+      @logger.warn("Error parsing @timestamp string, setting current time to @timestamp",
+                   :value => ts.inspect, :exception => e.message)
+      return nil
+    end
+  end
+end; end; end

data/lib/logstash/inputs/beats/event_transform_common.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+module LogStash module Inputs class Beats
+  # Base Transform class, expose the plugin decorate method,
+  # apply the tags and make sure we copy the beat hostname into `host`
+  # for backward compatibility.
+  class EventTransformCommon
+    def initialize(input)
+      @input = input
+      @logger = input.logger
+    end
+
+    # Copies the beat.hostname field into the host field unless
+    # the host field is already defined
+    def copy_beat_hostname(event)
+      host = event.get("[beat][hostname]")
+
+      if host && event.get("host").nil?
+        event.set("host", host)
+      end
+    end
+
+    # This break the `#decorate` method visibility of the plugin base
+    # class, the method is protected and we cannot access it, but well ruby
+    # can let you do all the wrong thing.
+    #
+    # I think the correct behavior would be to allow the plugin to return a 
+    # `Decorator` object that we can pass to other objects, since only the 
+    # plugin know the data used to decorate. This would allow a more component
+    # based workflow.
+    def decorate(event)
+      @input.send(:decorate, event)
+    end
+
+    def codec_name
+      @codec_name ||= if @input.codec.respond_to?(:base_codec)
+                        @input.codec.base_codec.class.config_name
+                      else
+                        @input.codec.class.config_name
+                      end
+    end
+
+    def transform(event)
+      copy_beat_hostname(event)
+      decorate(event)
+      event
+    end
+  end
+end; end; end

data/lib/logstash/inputs/beats/message_listener.rb

@@ -0,0 +1,96 @@
+# encoding: utf-8
+require "thread_safe"
+require "logstash-input-beats_jars"
+import "org.logstash.beats.MessageListener"
+
+module LogStash module Inputs class Beats
+  class MessageListener
+    include org.logstash.beats.IMessageListener
+
+    FILEBEAT_LOG_LINE_FIELD = "message".freeze
+    LSF_LOG_LINE_FIELD = "line".freeze
+
+    ConnectionState = Struct.new(:ctx, :codec)
+
+    attr_reader :logger, :input, :connections_list
+
+    def initialize(queue, input)
+      @connections_list = ThreadSafe::Hash.new
+      @queue = queue
+      @logger = input.logger
+      @input = input
+
+      @nocodec_transformer = RawEventTransform.new(@input)
+      @codec_transformer = DecodedEventTransform.new(@input)
+    end
+
+    def onNewMessage(ctx, message)
+      hash = message.getData()
+
+      target_field = extract_target_field(hash)
+
+      if target_field.nil?
+        event = LogStash::Event.new(hash)
+        @nocodec_transformer.transform(event)
+        @queue << event
+      else
+        codec(ctx).accept(CodecCallbackListener.new(target_field,
+                                                    hash,
+                                                    message.getIdentityStream(),
+                                                    @codec_transformer,
+                                                    @queue))
+      end
+    end
+
+    def onNewConnection(ctx)
+      register_connection(ctx)
+    end
+
+    def onConnectionClose(ctx)
+      unregister_connection(ctx)
+    end
+
+    def onException(ctx)
+      unregister_connection(ctx)
+    end
+
+    private
+    def codec(ctx)
+      connections_list[ctx].codec
+    end
+
+    def register_connection(ctx)
+      connections_list[ctx] = ConnectionState.new(ctx, input.codec.dup)
+    end
+
+    def unregister_connection(ctx)
+      flush_buffer(ctx)
+      connections_list.delete(ctx)
+    end
+
+    def flush_buffer(ctx)
+      transformer = EventTransformCommon.new(@input)
+
+      codec(ctx).flush do |event|
+        transformer.transform(event)
+        @queue << event
+      end
+    end
+
+    def extract_target_field(hash)
+      if from_filebeat?(hash)
+        hash.delete(FILEBEAT_LOG_LINE_FIELD).to_s
+      elsif from_logstash_forwarder?(hash)
+        hash.delete(LSF_LOG_LINE_FIELD).to_s
+      end
+    end
+
+    def from_filebeat?(hash)
+      !hash[FILEBEAT_LOG_LINE_FIELD].nil?
+    end
+
+    def from_logstash_forwarder?(hash)
+      !hash[LSF_LOG_LINE_FIELD].nil?
+    end
+  end
+end; end; end

data/lib/logstash/inputs/beats/raw_event_transform.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require "logstash/inputs/beats/event_transform_common"
+module LogStash module Inputs class Beats
+  # Take the the raw output from the library, decorate it with
+  # the configured tags in the plugins and normalize the hostname
+  # for backward compatibility
+  #
+  #
+  # @see [Lumberjack::Beats::Parser]
+  #
+  class RawEventTransform < EventTransformCommon
+    def transform(event)
+      super(event)
+      event.tag("beats_input_raw_event")
+      event
+    end
+  end
+end; end;end

data/lib/logstash/inputs/beats/tls.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+module LogStash module Inputs class Beats
+  class TLS
+    class TLSOption
+      include Comparable
+
+      attr_reader :name, :version
+      def initialize(name, version)
+        @name = name
+        @version = version
+      end
+
+      def <=>(other)
+        version <=> other.version
+      end
+    end
+
+    TLS_PROTOCOL_OPTIONS = [
+      TLSOption.new("TLSv1", 1),
+      TLSOption.new("TLSv1.1", 1.1),
+      TLSOption.new("TLSv1.2", 1.2)
+    ]
+
+    def self.min
+      TLS_PROTOCOL_OPTIONS.min
+    end
+
+    def self.max
+      TLS_PROTOCOL_OPTIONS.max
+    end
+
+    def self.get_supported(versions)
+      if versions.is_a?(Range)
+        TLS_PROTOCOL_OPTIONS.select { |tls| versions.cover?(tls.version) }
+      else 
+        TLS_PROTOCOL_OPTIONS.select { |tls| versions == tls.version }
+      end
+    end
+  end
+end; end; end

data/lib/tasks/build.rake

@@ -0,0 +1,15 @@
+# encoding: utf-8
+require "jars/installer"
+require "fileutils"
+
+task :vendor do
+  system("./gradlew vendor")
+  version = File.read("VERSION").strip
+end
+
+desc "clean"
+task :clean do
+  ["build", "vendor/jar-dependencies", "Gemfile.lock"].each do |p|
+    FileUtils.rm_rf(p)
+  end
+end

data/lib/tasks/test.rake

@@ -0,0 +1,65 @@
+# encoding: utf-8
+OS_PLATFORM = RbConfig::CONFIG["host_os"]
+VENDOR_PATH = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "vendor"))
+
+if OS_PLATFORM == "linux"
+  FILEBEAT_URL = "https://beats-nightlies.s3.amazonaws.com/filebeat/filebeat-5.0.0-alpha4-SNAPSHOT-linux-x86_64.tar.gz"
+elsif OS_PLATFORM == "darwin"
+  FILEBEAT_URL = "https://beats-nightlies.s3.amazonaws.com/filebeat/filebeat-5.0.0-alpha4-SNAPSHOT-darwin-x86_64.tar.gz"
+end
+
+LSF_URL = "https://download.elastic.co/logstash-forwarder/binaries/logstash-forwarder_#{OS_PLATFORM}_amd64"
+
+require "fileutils"
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"
+
+namespace :test do
+  namespace :integration do
+    task :setup do
+      Rake::Task["test:integration:setup:filebeat"].invoke
+      Rake::Task["test:integration:setup:lsf"].invoke
+    end
+
+    namespace :setup do
+      desc "Download lastest stable version of Logstash-forwarder"
+      task :lsf do
+        destination = File.join(VENDOR_PATH, "logstash-forwarder")
+        FileUtils.rm_rf(destination)
+        FileUtils.mkdir_p(destination)
+        download_destination = File.join(destination, "logstash-forwarder")
+        puts "Logstash-forwarder: downloading from #{LSF_URL} to #{download_destination}"
+        download(LSF_URL, download_destination)
+        File.chmod(0755, download_destination)
+      end
+
+      desc "Download nigthly filebeat for integration testing"
+      task :filebeat do
+        FileUtils.mkdir_p(VENDOR_PATH)
+        download_destination = File.join(VENDOR_PATH, "filebeat.tar.gz")
+        destination = File.join(VENDOR_PATH, "filebeat")
+        FileUtils.rm_rf(download_destination)
+        FileUtils.rm_rf(destination)
+        FileUtils.rm_rf(File.join(VENDOR_PATH, "filebeat.tar"))
+        puts "Filebeat: downloading from #{FILEBEAT_URL} to #{download_destination}"
+        download(FILEBEAT_URL, download_destination)
+
+        untar_all(download_destination, File.join(VENDOR_PATH, "filebeat")) { |e| e }
+      end
+    end
+  end
+end
+
+# Uncompress all the file from the archive this only work with 
+# one level directory structure and its fine for LSF and filebeat packaging.
+def untar_all(file, destination)
+  untar(file) do |entry| 
+    out = entry.full_name.split("/").last
+    File.join(destination, out)
+  end
+end

data/logstash-input-beats.gemspec

@@ -0,0 +1,41 @@
+BEATS_VERSION = File.read("VERSION").strip unless defined?(BEATS_VERSION)
+
+Gem::Specification.new do |s|
+  s.name            = "logstash-input-beats"
+  s.version         = BEATS_VERSION
+  s.licenses        = ["Apache License (2.0)"]
+  s.summary         = "Receive events using the lumberjack protocol."
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elastic"]
+  s.email           = "info@elastic.co"
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib", "vendor/jar-dependencies"]
+
+  # Files
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION"]
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_runtime_dependency "concurrent-ruby", [ ">= 0.9.2", "<= 1.0.0" ]
+  s.add_runtime_dependency "thread_safe", "~> 0.3.5"
+  s.add_runtime_dependency "logstash-codec-multiline", "~> 3.0"
+  s.add_runtime_dependency 'jar-dependencies', '~> 0.3.4'
+
+  s.add_development_dependency "flores", "~>0.0.6"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "stud"
+  s.add_development_dependency "pry"
+  s.add_development_dependency "rspec-wait"
+  s.add_development_dependency "logstash-devutils"
+  s.add_development_dependency "logstash-codec-json"
+  s.add_development_dependency "childprocess" # To make filebeat/LSF integration test easier to write.
+
+  s.platform = 'java'
+end

data/spec/inputs/beats/codec_callback_listener_spec.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+require "logstash/inputs/beats"
+require "logstash/event"
+require "spec_helper"
+require "thread"
+
+describe LogStash::Inputs::Beats::CodecCallbackListener do
+  let(:data) { "Hello world" }
+  let(:map) do
+    {
+      "beat" => { "hostname" => "newhost" }
+    }
+  end
+  let(:path) { "/var/log/message" }
+  let(:transformer) { double("codec_transformer") }
+  let(:queue_timeout) { 1 }
+  let(:event) { LogStash::Event.new }
+  let(:queue) { Queue.new }
+
+  before do
+    allow(transformer).to receive(:transform).with(event, map).and_return(event)
+  end
+
+  subject { described_class.new(data, map, path, transformer, queue) }
+
+  it "expose the data" do
+    expect(subject.data).to eq(data)
+  end
+
+  it "expose the path" do
+    expect(subject.path).to eq(path)
+  end
+end

data/spec/inputs/beats/decoded_event_transform_spec.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+require "logstash/event"
+require "logstash/timestamp"
+require "logstash/inputs/beats"
+require_relative "../../support/shared_examples"
+require "spec_helper"
+
+describe LogStash::Inputs::Beats::DecodedEventTransform do
+  let(:config) do
+    {
+      "port" => 0,
+      "type" => "example",
+      "tags" => "beats"
+    }
+  end
+
+  let(:input) do
+    LogStash::Inputs::Beats.new(config).tap do |i|
+      i.register
+    end
+  end
+  let(:event) { LogStash::Event.new }
+  let(:map) do
+    {
+      "@metadata" =>  { "hello" => "world"},
+      "super" => "mario"
+    }
+  end
+
+  subject { described_class.new(input).transform(event, map) }
+
+  include_examples "Common Event Transformation"
+
+  it "tags the event" do
+    expect(subject.get("tags")).to include("beats_input_codec_plain_applied")
+  end
+
+  it "merges the other data from the map to the event" do
+    expect(subject.get("super")).to eq(map["super"])
+    expect(subject.get("@metadata")).to include(map["@metadata"])
+  end
+
+  context "map contains a timestamp" do
+    context "when its valid" do
+      let(:timestamp) { Time.now }
+      let(:map) { super.merge({"@timestamp" => timestamp }) }
+     
+      it "uses as the event timestamp" do
+        expect(subject.get("@timestamp")).to eq(LogStash::Timestamp.coerce(timestamp)) 
+      end
+    end
+
+    context "when its not valid" do
+      let(:map) { super.merge({"@timestamp" => "invalid" }) }
+
+      it "fallback the current time" do
+        expect(subject.get("@timestamp")).to be_kind_of(LogStash::Timestamp)
+      end
+    end
+  end
+
+  context "when the map doesn't provide a timestamp" do
+    it "fallback the current time" do
+      expect(subject.get("@timestamp")).to be_kind_of(LogStash::Timestamp)
+    end
+  end
+
+  context "when the codec is a base_codec wrapper" do
+    before { config.update("codec" => BeatsInputTest::DummyCodec.new) }
+    it "gets the codec config name from the base codec" do
+      expect(subject.get("tags")).to include("beats_input_codec_dummy_applied")
+    end
+  end
+end