logstash-output-http 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d3bfc1e5f8add9c6a24ebd8b415320ccac0e9b9
-  data.tar.gz: 209ea0334d006933361f184ea611d9080d24c20b
+  metadata.gz: c0f8641077417db2ff06a2c110b1bd0db08ade0e
+  data.tar.gz: 1c0e81c71ec5058b354b76e8d6e895b0752c7f46
 SHA512:
-  metadata.gz: f5015f9d8389bc52e7b0d20edd54d93b2ead727f84929c1b158cd67bf64e0e689022b68d421e97081f0971669c18008cef5f34e739238cf4ecf5000c2404bc44
-  data.tar.gz: 2f7c7cb38cb6ef5054609424ad239d466c0faf74eea13d5533a19f8c3706fe53a15d76b022cc07f860483ca142d416520e66793ddac6b2b3f3c281c097dc6486
+  metadata.gz: f6fa16c1088eebbcf31f6570c889d5a99bea037cd80c1ee6e74d1b7aa3caca10d1441f6bcbc83ed6f857b1802f8220bed77964f7b47fbe7f75fa75610a36c3e7
+  data.tar.gz: 9565bfee79b50fb814ffbdd0ea735533070994d026033896f76b62ee5c1ccbf4fcab278a4b38cbd10dfcd94b1b3bc90137001ffb1f29d8cf3cebeef805dd66b4

data/.gitignore

@@ -2,3 +2,5 @@
 Gemfile.lock
 .bundle
 vendor
+.idea
+*~

data/CHANGELOG.md

@@ -0,0 +1,4 @@
+* 1.1.0
+  - Concurrent execution
+  - Add many HTTP options via the http_client mixin
+  - Switch to manticore as HTTP Client

data/lib/logstash/outputs/http.rb

@@ -2,26 +2,35 @@
 require "logstash/outputs/base"
 require "logstash/namespace"
 require "logstash/json"
+require "uri"
+require "logstash/plugin_mixins/http_client"
 
 class LogStash::Outputs::Http < LogStash::Outputs::Base
-  # This output lets you `PUT` or `POST` events to a
+  include LogStash::PluginMixins::HttpClient
+
+  VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"]
+
+  # This output lets you send events to a
   # generic HTTP(S) endpoint
   #
-  # Additionally, you are given the option to customize
-  # the headers sent as well as basic customization of the
-  # event json itself.
+  # This output will execute up to 'pool_max' requests in parallel for performance.
+  # Consider this when tuning this plugin for performance.
+  #
+  # Additionally, note that when parallel execution is used strict ordering of events is not
+  # guaranteed!
+  #
+  # Beware, this gem does not yet support codecs. Please use the 'format' option for now.
 
   config_name "http"
 
   # URL to use
   config :url, :validate => :string, :required => :true
 
-  # validate SSL?
-  config :verify_ssl, :validate => :boolean, :default => true
+  # DEPRECATED. Set 'ssl_certificate_validation' instead
+  config :verify_ssl, :validate => :boolean, :default => true, :deprecated => "Please use 'ssl_certificate_validation' instead. This option will be removed in a future release!"
 
-  # What verb to use
-  # only put and post are supported for now
-  config :http_method, :validate => ["put", "post"], :required => :true
+  # The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head"
+  config :http_method, :validate => VALID_METHODS, :required => :true
 
   # Custom headers to use
   # format is `headers => ["X-My-Header", "%{host}"]`
@@ -40,7 +49,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
   #
   # For example:
   # [source,ruby]
-  #    mapping => ["foo", "%{host}", "bar", "%{type}"]
+  #    mapping => {"foo", "%{host}", "bar", "%{type}"}
   config :mapping, :validate => :hash
 
   # Set the format of the http body.
@@ -55,88 +64,154 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
 
   config :message, :validate => :string
 
-  public
   def register
-    require "ftw"
-    require "uri"
-    @agent = FTW::Agent.new
-    # TODO(sissel): SSL verify mode?
+    # Handle this deprecated option. TODO: remove the option
+    @ssl_certificate_validation = @verify_ssl if @verify_ssl
+    @http_method = @http_method.to_sym
+
+    # We count outstanding requests with this queue
+    # This queue tracks the requests to create backpressure
+    # When this queue is empty no new requests may be sent,
+    # tokens must be added back by the client on success
+    @request_tokens = SizedQueue.new(@pool_max)
+    @pool_max.times {|t| @request_tokens << true }
+
+    @requests = Array.new
 
     if @content_type.nil?
       case @format
         when "form" ; @content_type = "application/x-www-form-urlencoded"
         when "json" ; @content_type = "application/json"
+        when "message" ; @content_type = "text/plain"
       end
     end
-    if @format == "message"
-      if @message.nil?
-        raise "message must be set if message format is used"
-      end
-      if @content_type.nil?
-        raise "content_type must be set if message format is used"
-      end
-      unless @mapping.nil?
-        @logger.warn "mapping is not supported and will be ignored if message format is used"
-      end
-    end
+
+    validate_format!
   end # def register
 
-  public
   def receive(event)
     return unless output?(event)
 
-    if @mapping
-      evt = Hash.new
-      @mapping.each do |k,v|
-        evt[k] = event.sprintf(v)
+    body = event_body(event)
+
+    # Block waiting for a token
+    token = @request_tokens.pop
+
+    # Send the request
+    url = event.sprintf(@url)
+    headers = event_headers(event)
+
+    # Create an async request
+    request = client.send(@http_method, url, body: body, headers: headers, async: true)
+
+    # Invoke it using the Manticore Executor (CachedThreadPool) directly
+    request_async_background(request)
+
+    # Make sure we return the token to the pool
+    request.on_complete do
+      @request_tokens << token
+    end
+
+    request.on_success do |response|
+      if response.code < 200 || response.code > 299
+        log_failure(
+          "Encountered non-200 HTTP code #{200}",
+          :response_code => response.code,
+          :url => url,
+          :event => event)
       end
-    else
-      evt = event.to_hash
     end
 
-    case @http_method
-    when "put"
-      request = @agent.put(event.sprintf(@url))
-    when "post"
-      request = @agent.post(event.sprintf(@url))
+    request.on_failure do |exception|
+      log_failure("Could not fetch URL",
+                  :url => url,
+                  :method => @http_method,
+                  :body => body,
+                  :headers => headers,
+                  :message => exception.message,
+                  :class => exception.class.name,
+                  :backtrace => exception.backtrace
+      )
+    end
+  end
+
+  private
+
+  # This is split into a separate method mostly to help testing
+  def log_failure(message, opts)
+    @logger.error("[HTTP Output Failure] #{message}", opts)
+  end
+
+  # Manticore doesn't provide a way to attach handlers to background or async requests well
+  # It wants you to use futures. The #async method kinda works but expects single thread batches
+  # and background only returns futures.
+  # Proposed fix to manticore here: https://github.com/cheald/manticore/issues/32
+  def request_async_background(request)
+    @method ||= client.executor.java_method(:submit, [java.util.concurrent.Callable.java_class])
+    @method.call(request)
+  end
+
+  # Format the HTTP body
+  def event_body(event)
+    # TODO: Create an HTTP post data codec, use that here
+    if @format == "json"
+      LogStash::Json.dump(map_event(event))
+    elsif @format == "message"
+      event.sprintf(@message)
     else
-      @logger.error("Unknown verb:", :verb => @http_method)
+      encode(map_event(event))
     end
+  end
 
-    if @headers
-      @headers.each do |k,v|
-        request.headers[k] = event.sprintf(v)
+  def map_event(event)
+    if @mapping
+      @mapping.reduce({}) do |acc,kv|
+        k,v = kv
+        acc[k] = event.sprintf(v)
+        acc
       end
+    else
+      event.to_hash
     end
+  end
 
-    request["Content-Type"] = @content_type
+  def event_headers(event)
+    headers = custom_headers(event) || {}
+    headers["Content-Type"] = @content_type
+    headers
+  end
 
-    begin
-      if @format == "json"
-        request.body = LogStash::Json.dump(evt)
-      elsif @format == "message"
-        request.body = event.sprintf(@message)
-      else
-        request.body = encode(evt)
-      end
-      #puts "#{request.port} / #{request.protocol}"
-      #puts request
-      #puts
-      #puts request.body
-      response = @agent.execute(request)
-
-      # Consume body to let this connection be reused
-      rbody = ""
-      response.read_body { |c| rbody << c }
-      #puts rbody
-    rescue Exception => e
-      @logger.warn("Unhandled exception", :request => request, :response => response, :exception => e, :stacktrace => e.backtrace)
+  def custom_headers(event)
+    return nil unless @headers
+
+    @headers.reduce({}) do |acc,kv|
+      k,v = kv
+      acc[k] = event.sprintf(v)
+      acc
     end
-  end # def receive
+  end
 
+  #TODO Extract this to a codec
   def encode(hash)
     return hash.collect do |key, value|
-      CGI.escape(key) + "=" + CGI.escape(value)
+      CGI.escape(key) + "=" + CGI.escape(value.to_s)
     end.join("&")
-  end # def encode
+  end
+
+
+  def validate_format!
+    if @format == "message"
+      if @message.nil?
+        raise "message must be set if message format is used"
+      end
+
+      if @content_type.nil?
+        raise "content_type must be set if message format is used"
+      end
+
+      unless @mapping.nil?
+        @logger.warn "mapping is not supported and will be ignored if message format is used"
+      end
+    end
+  end
 end

data/logstash-output-http.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-http'
-  s.version         = '1.0.0'
+  s.version         = '1.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This output lets you `PUT` or `POST` events to a generic HTTP(S) endpoint"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -21,9 +21,11 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
-
-  s.add_runtime_dependency 'ftw', ['~> 0.0.40']
+  s.add_runtime_dependency 'logstash-mixin-http_client', '>= 1.0.1', '< 2.0.0'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-codec-plain'
+  s.add_development_dependency 'sinatra'
+  s.add_development_dependency 'webrick'
 end
 

data/spec/outputs/http_spec.rb

@@ -1 +1,232 @@
 require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/http"
+require "thread"
+require "sinatra"
+
+PORT = rand(65535-1024) + 1025
+
+class LogStash::Outputs::Http
+  attr_writer :agent
+  attr_reader :request_tokens
+end
+
+class TestApp < Sinatra::Base
+  def self.multiroute(methods, path, &block)
+    methods.each do |method|
+      method.to_sym
+      self.send method, path, &block
+    end
+  end
+
+  def self.last_request=(request)
+    @last_request = request
+  end
+
+  def self.last_request
+    @last_request
+  end
+
+  multiroute(%w(get post put patch delete), "/good") do
+    self.class.last_request = request
+    [200, "YUP"]
+  end
+
+  multiroute(%w(get post put patch delete), "/bad") do
+    self.class.last_request = request
+    [500, "YUP"]
+  end
+end
+
+RSpec.configure do |config|
+  #http://stackoverflow.com/questions/6557079/start-and-call-ruby-http-server-in-the-same-script
+  def sinatra_run_wait(app, opts)
+    queue = Queue.new
+    thread = Thread.new do
+      Thread.abort_on_exception = true
+      app.run!(opts) do |server|
+        queue.push("started")
+      end
+    end
+    queue.pop # blocks until the run! callback runs
+  end
+
+
+  config.before(:suite) do
+    sinatra_run_wait(TestApp, :port => PORT, :server => 'webrick')
+  end
+end
+
+describe LogStash::Outputs::Http do
+  # Wait for the async request to finish in this spinlock
+  # Requires pool_max to be 1
+  def wait_for_request
+
+    loop do
+      break if subject.request_tokens.size > 0
+    end
+  end
+
+  let(:port) { PORT }
+  let(:event) { LogStash::Event.new("message" => "hi") }
+  let(:url) { "http://localhost:#{port}/good" }
+  let(:method) { "post" }
+
+  describe "when num requests > token count" do
+    let(:pool_max) { 10 }
+    let(:num_reqs) { pool_max / 2 }
+    let(:client) { subject.client }
+    subject {
+      LogStash::Outputs::Http.new("url" => url,
+                                  "http_method" => method,
+                                  "pool_max" => pool_max)
+    }
+
+    before do
+      subject.register
+    end
+
+    it "should receive all the requests" do
+      expect(client).to receive(:send).
+                          with(method.to_sym, url, anything).
+                          exactly(num_reqs).times.
+                          and_call_original
+
+      num_reqs.times {|t| subject.receive(event)}
+    end
+  end
+
+  shared_examples("verb behavior") do |method|
+    subject { LogStash::Outputs::Http.new("url" => url, "http_method" => method, "pool_max" => 1) }
+
+    let(:expected_method) { method.clone.to_sym }
+    let(:client) { subject.client }
+
+    before do
+      subject.register
+      allow(client).to receive(:send).
+                         with(expected_method, url, anything).
+                         and_call_original
+      allow(subject).to receive(:log_failure).with(any_args)
+    end
+
+    context "performing a get" do
+      describe "invoking the request" do
+        before do
+          subject.receive(event)
+        end
+
+        it "should execute the request" do
+          expect(client).to have_received(:send).
+                              with(expected_method, url, anything)
+        end
+      end
+
+      context "with passing requests" do
+        before do
+          subject.receive(event)
+        end
+
+        it "should not log a failure" do
+          expect(subject).not_to have_received(:log_failure).with(any_args)
+        end
+      end
+
+      context "with failing requests" do
+        let(:url) { "http://localhost:#{port}/bad"}
+
+        before do
+          subject.receive(event)
+          wait_for_request
+        end
+
+        it "should log a failure" do
+          expect(subject).to have_received(:log_failure).with(any_args)
+        end
+      end
+    end
+  end
+
+  LogStash::Outputs::Http::VALID_METHODS.each do |method|
+    context "when using '#{method}'" do
+      include_examples("verb behavior", method)
+    end
+  end
+
+  shared_examples("a received event") do
+    before do
+      TestApp.last_request = nil
+    end
+
+    before do
+      subject.receive(event)
+      wait_for_request
+    end
+
+    let(:last_request) { TestApp.last_request }
+    let(:body) { last_request.body.read }
+    let(:content_type) { last_request.env["CONTENT_TYPE"] }
+
+    it "should receive the request" do
+      expect(last_request).to be_truthy
+    end
+
+    it "should receive the event as a hash" do
+      expect(body).to eql(expected_body)
+    end
+
+    it "should have the correct content type" do
+      expect(content_type).to eql(expected_content_type)
+    end
+  end
+
+  describe "integration tests" do
+    let(:url) { "http://localhost:#{port}/good" }
+    let(:event) { LogStash::Event.new("foo" => "bar", "baz" => "bot")}
+
+    subject { LogStash::Outputs::Http.new(config) }
+
+    before do
+      subject.register
+    end
+
+    describe "sending with the default (JSON) config" do
+      let(:config) {
+        {"url" => url, "http_method" => "post", "pool_max" => 1}
+      }
+      let(:expected_body) { LogStash::Json.dump(event) }
+      let(:expected_content_type) { "application/json" }
+
+      include_examples("a received event")
+    end
+
+    describe "sending the event as a form" do
+      let(:config) {
+        {"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "form"}
+      }
+      let(:expected_body) { subject.send(:encode, event.to_hash) }
+      let(:expected_content_type) { "application/x-www-form-urlencoded" }
+
+      include_examples("a received event")
+    end
+
+    describe "sending the event as a message" do
+      let(:config) {
+        {"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "message", "message" => "%{foo} AND %{baz}"}
+      }
+      let(:expected_body) { "#{event["foo"]} AND #{event["baz"]}" }
+      let(:expected_content_type) { "text/plain" }
+
+      include_examples("a received event")
+    end
+
+    describe "sending a mapped event" do
+      let(:config) {
+        {"url" => url, "http_method" => "post", "pool_max" => 1, "mapping" => {"blah" => "X %{foo}"}}
+      }
+      let(:expected_body) { LogStash::Json.dump("blah" => "X #{event["foo"]}") }
+      let(:expected_content_type) { "application/json" }
+
+      include_examples("a received event")
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-http
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-24 00:00:00.000000000 Z
+date: 2015-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -31,17 +31,23 @@ dependencies:
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
-  name: ftw
+  name: logstash-mixin-http_client
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+    - - <
       - !ruby/object:Gem::Version
-        version: 0.0.40
+        version: 2.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+    - - <
       - !ruby/object:Gem::Version
-        version: 0.0.40
+        version: 2.0.0
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -58,6 +64,48 @@ dependencies:
         version: '0'
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: webrick
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -97,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: This output lets you `PUT` or `POST` events to a generic HTTP(S) endpoint