logstash-output-splunk 0.0.1

data/lib/logstash/outputs/splunk.rb

@@ -0,0 +1,344 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/json"
+require "uri"
+require "logstash/plugin_mixins/http_client"
+require "zlib"
+
+class LogStash::Outputs::Splunk < LogStash::Outputs::Base
+  include LogStash::PluginMixins::HttpClient
+
+  concurrency :shared
+
+  attr_accessor :is_batch
+
+  RETRYABLE_MANTICORE_EXCEPTIONS = [
+    ::Manticore::Timeout,
+    ::Manticore::SocketException,
+    ::Manticore::ClientProtocolException,
+    ::Manticore::ResolutionFailure,
+    ::Manticore::SocketTimeout
+  ]
+
+  # This output lets you send events to a
+  # generic HTTP(S) endpoint
+  #
+  # This output will execute up to 'pool_max' requests in parallel for performance.
+  # Consider this when tuning this plugin for performance.
+  #
+  # Additionally, note that when parallel execution is used strict ordering of events is not
+  # guaranteed!
+  #
+  # Beware, this gem does not yet support codecs. Please use the 'format' option for now.
+
+  config_name "splunk"
+
+  # URL to use
+  config :url, :validate => :string, :required => :true
+
+  # Custom headers to use
+  # format is `headers => ["X-My-Header", "%{host}"]`
+  config :headers, :validate => :hash, :default => {}
+
+  # Splunk HTTP Event Collector tokens to use
+  config :token, :validate => :string, :required => :true
+
+  # Content type
+  #
+  # If not specified, this defaults to the following:
+  #
+  config :content_type, :validate => :string
+
+  # Set this to false if you don't want this output to retry failed requests
+  config :retry_failed, :validate => :boolean, :default => true
+
+  # If encountered as response codes this plugin will retry these requests
+  config :retryable_codes, :validate => :number, :list => true, :default => [429, 500, 502, 503, 504]
+
+  # If you would like to consider some non-2xx codes to be successes
+  # enumerate them here. Responses returning these codes will be considered successes
+  config :ignorable_codes, :validate => :number, :list => true
+
+  # Set this to false if you don't want batch
+  config :is_batch, :validate => :boolean, :default => true
+
+  # This lets you choose the structure and parts of the event that are sent.
+  #
+  #
+  # For example:
+  # [source,ruby]
+  #    mapping => {"foo" => "%{host}"
+  #               "bar" => "%{type}"}
+  config :mapping, :validate => :hash
+
+  # Set the format of the http body.
+  #
+  # If form, then the body will be the mapping (or whole event) converted
+  # into a query parameter string, e.g. `foo=bar&baz=fizz...`
+  #
+  # If message, then the body will be the result of formatting the event according to message
+  #
+  # Set this to true if you want to enable gzip compression for your http requests
+  config :http_compression, :validate => :boolean, :default => false
+
+  config :message, :validate => :string
+
+  def register
+    # We count outstanding requests with this queue
+    # This queue tracks the requests to create backpressure
+    # When this queue is empty no new requests may be sent,
+    # tokens must be added back by the client on success
+    @request_tokens = SizedQueue.new(@pool_max)
+    @pool_max.times {|t| @request_tokens << true }
+    @requests = Array.new
+    @content_type = "application/json"
+    @is_batch = @is_batch
+    @headers["Content-Type"] = @content_type
+
+    # Splunk HEC token
+    @headers["Authorization"] = "Splunk " + @token
+
+    # Run named Timer as daemon thread
+    @timer = java.util.Timer.new("Splunk Output #{self.params['id']}", true)
+  end # def register
+
+  def multi_receive(events)
+    return if events.empty?
+    send_events(events)
+  end
+
+  class RetryTimerTask < java.util.TimerTask
+    def initialize(pending, event, attempt)
+      @pending = pending
+      @event = event
+      @attempt = attempt
+      super()
+    end
+
+    def run
+      @pending << [@event, @attempt]
+    end
+  end
+
+  def log_retryable_response(response)
+    if (response.code == 429)
+      @logger.debug? && @logger.debug("Encountered a 429 response, will retry. This is not serious, just flow control via HTTP")
+    else
+      @logger.warn("Encountered a retryable HTTP request in HTTP output, will retry", :code => response.code, :body => response.body)
+    end
+  end
+
+  def log_error_response(response, url, event)
+    log_failure(
+              "Encountered non-2xx HTTP code #{response.code}",
+              :response_code => response.code,
+              :url => url,
+              :event => event
+            )
+  end
+
+  def send_events(events)
+    successes = java.util.concurrent.atomic.AtomicInteger.new(0)
+    failures  = java.util.concurrent.atomic.AtomicInteger.new(0)
+    retries = java.util.concurrent.atomic.AtomicInteger.new(0)
+    event_count = @is_batch ? 1 : events.size
+
+    pending = Queue.new
+    if @is_batch
+      pending << [events, 0]
+    else
+      events.each {|e| pending << [e, 0]}
+    end
+
+    while popped = pending.pop
+      break if popped == :done
+
+      event, attempt = popped
+
+      action, event, attempt = send_event(event, attempt)
+      begin
+        action = :failure if action == :retry && !@retry_failed
+
+        case action
+        when :success
+          successes.incrementAndGet
+        when :retry
+          retries.incrementAndGet
+
+          next_attempt = attempt+1
+          sleep_for = sleep_for_attempt(next_attempt)
+          @logger.info("Retrying http request, will sleep for #{sleep_for} seconds")
+          timer_task = RetryTimerTask.new(pending, event, next_attempt)
+          @timer.schedule(timer_task, sleep_for*1000)
+        when :failure
+          failures.incrementAndGet
+        else
+          raise "Unknown action #{action}"
+        end
+
+        if action == :success || action == :failure
+          if successes.get+failures.get == event_count
+            pending << :done
+          end
+        end
+      rescue => e
+        # This should never happen unless there's a flat out bug in the code
+        @logger.error("Error sending HTTP Request",
+          :class => e.class.name,
+          :message => e.message,
+          :backtrace => e.backtrace)
+        failures.incrementAndGet
+        raise e
+      end
+    end
+  rescue => e
+    @logger.error("Error in http output loop",
+            :class => e.class.name,
+            :message => e.message,
+            :backtrace => e.backtrace)
+    raise e
+  end
+
+  def sleep_for_attempt(attempt)
+    sleep_for = attempt**2
+    sleep_for = sleep_for <= 60 ? sleep_for : 60
+    (sleep_for/2) + (rand(0..sleep_for)/2)
+  end
+
+  def send_event(event, attempt)
+    body = event_body(event)
+
+    # Send the request
+    url = @is_batch ? @url : event.sprintf(@url)
+    headers = @is_batch ? @headers : event_headers(event)
+
+    # Compress the body and add appropriate header
+    if @http_compression == true
+      headers["Content-Encoding"] = "gzip"
+      body = gzip(body)
+    end
+
+    # Create an async request
+    response = client.send(:post, url, :body => body, :headers => headers).call
+
+    if !response_success?(response)
+      if retryable_response?(response)
+        log_retryable_response(response)
+        return :retry, event, attempt
+      else
+        log_error_response(response, url, event)
+        return :failure, event, attempt
+      end
+    else
+      return :success, event, attempt
+    end
+
+  rescue => exception
+    will_retry = retryable_exception?(exception)
+    log_failure("Could not fetch URL",
+                :url => url,
+                :body => body,
+                :headers => headers,
+                :message => exception.message,
+                :class => exception.class.name,
+                :backtrace => exception.backtrace,
+                :will_retry => will_retry
+    )
+
+    if will_retry
+      return :retry, event, attempt
+    else
+      return :failure, event, attempt
+    end
+  end
+
+  def close
+    @timer.cancel
+    client.close
+  end
+
+  private
+
+  def response_success?(response)
+    code = response.code
+    return true if @ignorable_codes && @ignorable_codes.include?(code)
+    return code >= 200 && code <= 299
+  end
+
+  def retryable_response?(response)
+    @retryable_codes && @retryable_codes.include?(response.code)
+  end
+
+  def retryable_exception?(exception)
+    RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me) }
+  end
+
+  # This is split into a separate method mostly to help testing
+  def log_failure(message, opts)
+    @logger.error("[HTTP Output Failure] #{message}", opts)
+  end
+
+  # Format the HTTP body
+  def event_body(event)
+    # TODO: Create an HTTP post data codec, use that here
+    if @is_batch
+      event.map {|e| LogStash::Json.dump(map_event(e)) }.join("\n")
+    else
+      LogStash::Json.dump(map_event(event))
+    end
+  end
+
+  # gzip data
+  def gzip(data)
+    gz = StringIO.new
+    gz.set_encoding("BINARY")
+    z = Zlib::GzipWriter.new(gz)
+    z.write(data)
+    z.close
+    gz.string
+  end
+
+  def convert_mapping(mapping, event)
+    if mapping.is_a?(Hash)
+      mapping.reduce({}) do |acc, kv|
+        k, v = kv
+        acc[k] = convert_mapping(v, event)
+        acc
+      end
+    elsif mapping.is_a?(Array)
+      mapping.map { |elem| convert_mapping(elem, event) }
+    else
+      event.sprintf(mapping)
+    end
+  end
+
+  def map_event(event)
+    if @mapping
+      convert_mapping(@mapping, event)
+    else
+      event.to_hash
+    end
+  end
+
+  def event_headers(event)
+    custom_headers(event) || {}
+  end
+
+  def custom_headers(event)
+    return nil unless @headers
+
+    @headers.reduce({}) do |acc,kv|
+      k,v = kv
+      acc[k] = event.sprintf(v)
+      acc
+    end
+  end
+
+  #TODO Extract this to a codec
+  def encode(hash)
+    return hash.collect do |key, value|
+      CGI.escape(key) + "=" + CGI.escape(value.to_s)
+    end.join("&")
+  end
+end

data/logstash-output-splunk.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+  s.name            = 'logstash-output-splunk'
+  s.version         = '0.0.1'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Sends events to a Splunk HTTP Event Collector REST API endpoints"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Jian Chen"]
+  s.email           = 'jianchen2580@gmail.com'
+  # TODO
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 6.0.0", "< 8.0.0"
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'sinatra'
+  s.add_development_dependency 'webrick'
+end

data/spec/outputs/splunk_spec.rb

@@ -0,0 +1,364 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/http"
+require "logstash/codecs/plain"
+require "thread"
+require "sinatra"
+require_relative "../supports/compressed_requests"
+
+PORT = rand(65535-1024) + 1025
+
+class LogStash::Outputs::Splunk
+  attr_writer :agent
+  attr_reader :request_tokens
+end
+
+# note that Sinatra startup and shutdown messages are directly logged to stderr so
+# it is not really possible to disable them without reopening stderr which is not advisable.
+#
+# == Sinatra (v1.4.6) has taken the stage on 51572 for development with backup from WEBrick
+# == Sinatra has ended his set (crowd applauds)
+#
+class TestApp < Sinatra::Base
+  # on the fly uncompress gzip content
+  use CompressedRequests
+
+  # disable WEBrick logging
+  def self.server_settings
+    { :AccessLog => [], :Logger => WEBrick::BasicLog::new(nil, WEBrick::BasicLog::FATAL) }
+  end
+
+  def self.multiroute(methods, path, &block)
+    methods.each do |method|
+      method.to_sym
+      self.send method, path, &block
+    end
+  end
+
+  def self.last_request=(request)
+    @last_request = request
+  end
+
+  def self.last_request
+    @last_request
+  end
+
+  def self.retry_fail_count=(count)
+    @retry_fail_count = count
+  end
+
+  def self.retry_fail_count()
+    @retry_fail_count || 2
+  end
+
+  multiroute(%w(get post put patch delete), "/good") do
+    self.class.last_request = request
+    [200, "YUP"]
+  end
+
+  multiroute(%w(get post put patch delete), "/bad") do
+    self.class.last_request = request
+    [400, "YUP"]
+  end
+
+  multiroute(%w(get post put patch delete), "/retry") do
+    self.class.last_request = request
+
+    if self.class.retry_fail_count > 0
+      self.class.retry_fail_count -= 1
+      [429, "Will succeed in #{self.class.retry_fail_count}"]
+    else
+      [200, "Done Retrying"]
+    end
+  end
+end
+
+RSpec.configure do |config|
+  #http://stackoverflow.com/questions/6557079/start-and-call-ruby-http-server-in-the-same-script
+  def sinatra_run_wait(app, opts)
+    queue = Queue.new
+
+    t = java.lang.Thread.new(
+      proc do
+        begin
+          app.run!(opts) do |server|
+            queue.push("started")
+          end
+        rescue => e
+          puts "Error in webserver thread #{e}"
+          # ignore
+        end
+      end
+    )
+    t.daemon = true
+    t.start
+    queue.pop # blocks until the run! callback runs
+  end
+
+  config.before(:suite) do
+    sinatra_run_wait(TestApp, :port => PORT, :server => 'webrick')
+    puts "Test webserver on port #{PORT}"
+  end
+end
+
+describe LogStash::Outputs::Splunk do
+  # Wait for the async request to finish in this spinlock
+  # Requires pool_max to be 1
+
+  let(:port) { PORT }
+  let(:event) {
+    LogStash::Event.new({"message" => "hi"})
+  }
+  let(:url) { "http://localhost:#{port}/good" }
+  let(:method) { "post" }
+
+  shared_examples("verb behavior") do |method|
+    let(:verb_behavior_config) { {"url" => url, "http_method" => method, "pool_max" => 1} }
+    subject { LogStash::Outputs::Splunk.new(verb_behavior_config) }
+
+    let(:expected_method) { method.clone.to_sym }
+    let(:client) { subject.client }
+
+    before do
+      subject.register
+      allow(client).to receive(:send).
+                         with(expected_method, url, anything).
+                         and_call_original
+      allow(subject).to receive(:log_failure).with(any_args)
+      allow(subject).to receive(:log_retryable_response).with(any_args)
+    end
+
+    context 'sending no events' do
+      it 'should not block the pipeline' do
+        subject.multi_receive([])
+      end
+    end
+
+    context "performing a get" do
+      describe "invoking the request" do
+        before do
+          subject.multi_receive([event])
+        end
+
+        it "should execute the request" do
+          expect(client).to have_received(:send).
+                              with(expected_method, url, anything)
+        end
+      end
+
+      context "with passing requests" do
+        before do
+          subject.multi_receive([event])
+        end
+
+        it "should not log a failure" do
+          expect(subject).not_to have_received(:log_failure).with(any_args)
+        end
+      end
+
+      context "with failing requests" do
+        let(:url) { "http://localhost:#{port}/bad"}
+
+        before do
+          subject.multi_receive([event])
+        end
+
+        it "should log a failure" do
+          expect(subject).to have_received(:log_failure).with(any_args)
+        end
+      end
+
+      context "with ignorable failing requests" do
+        let(:url) { "http://localhost:#{port}/bad"}
+        let(:verb_behavior_config) { super.merge("ignorable_codes" => [400]) }
+
+        before do
+          subject.multi_receive([event])
+        end
+
+        it "should log a failure" do
+          expect(subject).not_to have_received(:log_failure).with(any_args)
+        end
+      end
+
+      context "with retryable failing requests" do
+        let(:url) { "http://localhost:#{port}/retry"}
+
+        before do
+          TestApp.retry_fail_count=2
+          allow(subject).to receive(:send_event).and_call_original
+          subject.multi_receive([event])
+        end
+
+        it "should log a retryable response 2 times" do
+          expect(subject).to have_received(:log_retryable_response).with(any_args).twice
+        end
+
+        it "should make three total requests" do
+          expect(subject).to have_received(:send_event).exactly(3).times
+        end
+      end
+
+    end
+  end
+
+  LogStash::Outputs::Splunk::VALID_METHODS.each do |method|
+    context "when using '#{method}'" do
+      include_examples("verb behavior", method)
+    end
+  end
+
+  shared_examples("a received event") do
+    before do
+      TestApp.last_request = nil
+    end
+
+    let(:events) { [event] }
+
+    describe "with a good code" do
+      before do
+        subject.multi_receive(events)
+      end
+
+      let(:last_request) { TestApp.last_request }
+      let(:body) { last_request.body.read }
+      let(:content_type) { last_request.env["CONTENT_TYPE"] }
+
+      it "should receive the request" do
+        expect(last_request).to be_truthy
+      end
+
+      it "should receive the event as a hash" do
+        expect(body).to eql(expected_body)
+      end
+
+      it "should have the correct content type" do
+        expect(content_type).to eql(expected_content_type)
+      end
+    end
+
+    describe "a retryable code" do
+      let(:url) { "http://localhost:#{port}/retry" }
+
+      before do
+        TestApp.retry_fail_count=2
+        allow(subject).to receive(:send_event).and_call_original
+        allow(subject).to receive(:log_retryable_response)
+        subject.multi_receive(events)
+      end
+
+      it "should retry" do
+        expect(subject).to have_received(:log_retryable_response).with(any_args).twice
+      end
+    end
+  end
+
+  shared_examples "integration tests" do
+    let(:base_config) { {} }
+    let(:url) { "http://localhost:#{port}/good" }
+    let(:event) {
+      LogStash::Event.new("foo" => "bar", "baz" => "bot", "user" => "McBest")
+    }
+
+    subject { LogStash::Outputs::Http.new(config) }
+
+    before do
+      subject.register
+    end
+
+    describe "sending with the default (JSON) config" do
+      let(:config) {
+        base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1})
+      }
+      let(:expected_body) { LogStash::Json.dump(event) }
+      let(:expected_content_type) { "application/json" }
+
+      include_examples("a received event")
+    end
+
+    describe "sending the batch as JSON" do
+      let(:config) do
+        base_config.merge({"url" => url, "http_method" => "post", "format" => "json_batch"})
+      end
+
+      let(:expected_body) { ::LogStash::Json.dump events }
+      let(:events) { [::LogStash::Event.new("a" => 1), ::LogStash::Event.new("b" => 2)]}
+      let(:expected_content_type) { "application/json" }
+      
+      include_examples("a received event")
+
+    end
+
+    describe "sending the event as a form" do
+      let(:config) {
+        base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "form"})
+      }
+      let(:expected_body) { subject.send(:encode, event.to_hash) }
+      let(:expected_content_type) { "application/x-www-form-urlencoded" }
+
+      include_examples("a received event")
+    end
+
+    describe "sending the event as a message" do
+      let(:config) {
+        base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "message", "message" => "%{foo} AND %{baz}"})
+      }
+      let(:expected_body) { "#{event.get("foo")} AND #{event.get("baz")}" }
+      let(:expected_content_type) { "text/plain" }
+
+      include_examples("a received event")
+    end
+
+    describe "sending a mapped event" do
+      let(:config) {
+        base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "mapping" => {"blah" => "X %{foo}"} })
+      }
+      let(:expected_body) { LogStash::Json.dump("blah" => "X #{event.get("foo")}") }
+      let(:expected_content_type) { "application/json" }
+
+      include_examples("a received event")
+    end
+
+    describe "sending a mapped, nested event" do
+      let(:config) {
+        base_config.merge({
+          "url" => url,
+          "http_method" => "post",
+          "pool_max" => 1,
+          "mapping" => {
+            "host" => "X %{foo}",
+            "event" => {
+              "user" => "Y %{user}"
+            },
+            "arrayevent" => [{
+              "user" => "Z %{user}"
+            }]
+          }
+        })
+      }
+      let(:expected_body) {
+        LogStash::Json.dump({
+          "host" => "X #{event.get("foo")}",
+          "event" => {
+            "user" => "Y #{event.get("user")}"
+          },
+          "arrayevent" => [{
+            "user" => "Z #{event.get("user")}"
+          }]
+        })
+      }
+      let(:expected_content_type) { "application/json" }
+
+      include_examples("a received event")
+    end
+  end
+
+  describe "integration test without gzip compression" do
+    include_examples("integration tests")
+  end
+
+  describe "integration test with gzip compression" do
+    include_examples("integration tests") do
+      let(:base_config) { { "http_compression" => true } }
+    end
+  end
+end