logstash-output-http 3.1.1 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 166795862fd592911c1662e2b5413de11f4824ba
-  data.tar.gz: 028265896afc48b3e0e0c806007c523eec50ae71
+  metadata.gz: fb4ed46a005d9af73da085335c1f1f516cd54f1b
+  data.tar.gz: ea8ded7326eef70f098bbcab8e910d9e3cfe7b2a
 SHA512:
-  metadata.gz: 1ca69a7fd82c6738025fd25eba6d2e3e728035e2e9d52bbf061fba6a4675392dc0d7744536546ade2b6564259806b42f80d7d7230a1755697548aea897d384fb
-  data.tar.gz: 869ce7b82b48ed382b602c3188e425e45a771ce2a7274aba3dba2670bee009198d5879020cf5b3e46a58a9d1b9c2e61591136698732d6372b9a936ce05aa5c92
+  metadata.gz: 7c7e3a3e18f434a3d575f280cd5f8f6502833338a270519facf6d6f9115f78e033df68bcbfb6d050fcda56230a914cab22f61a1c06854203af51ce8d54e3dc03
+  data.tar.gz: 7e8b34fc4f8872a4754bc2868d48dbd57e7db5bee7006bfc2f6d6be116457e36a6d50edcb7a8fd8a3fb8747b7e2c0b022a387c8642f599e44bcc621dbf878a16

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 4.0.0
+ - Major overhaul of internals, adds new retry options
+ - Allow users to specify non-standard response codes as ignorable
+ - Set concurrency level to shared allowing for greater efficiency across threads
+ 
 ## 3.1.1
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 
@@ -19,6 +24,7 @@
 ## 2.1.1
  - Require http_client mixin with better keepalive handling
 
+
 ## 2.1.0
  - Properly close the client on #close
  - Optimized execution for Logstash 2.2 ng pipeline
@@ -39,4 +45,3 @@
  - Concurrent execution
  - Add many HTTP options via the http_client mixin
  - Switch to manticore as HTTP Client
- 

data/lib/logstash/outputs/http.rb

@@ -7,8 +7,18 @@ require "logstash/plugin_mixins/http_client"
 
 class LogStash::Outputs::Http < LogStash::Outputs::Base
   include LogStash::PluginMixins::HttpClient
+  
+  concurrency :shared
 
   VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"]
+  
+  RETRYABLE_MANTICORE_EXCEPTIONS = [
+    ::Manticore::Timeout,
+    ::Manticore::SocketException,
+    ::Manticore::ClientProtocolException, 
+    ::Manticore::ResolutionFailure, 
+    ::Manticore::SocketTimeout
+  ]
 
   # This output lets you send events to a
   # generic HTTP(S) endpoint
@@ -40,13 +50,24 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
   # * if format is "json", "application/json"
   # * if format is "form", "application/x-www-form-urlencoded"
   config :content_type, :validate => :string
+  
+  # Set this to false if you don't want this output to retry failed requests
+  config :retry_failed, :validate => :boolean, :default => true
+  
+  # If encountered as response codes this plugin will retry these requests
+  config :retryable_codes, :validate => :number, :list => true, :default => [429, 500, 502, 503, 504]
+  
+  # If you would like to consider some non-2xx codes to be successes 
+  # enumerate them here. Responses returning these codes will be considered successes
+  config :ignorable_codes, :validate => :number, :list => true
 
   # This lets you choose the structure and parts of the event that are sent.
   #
   #
   # For example:
   # [source,ruby]
-  #    mapping => {"foo", "%{host}", "bar", "%{type}"}
+  #    mapping => {"foo" => "%{host}" 
+  #               "bar" => "%{type}"}
   config :mapping, :validate => :hash
 
   # Set the format of the http body.
@@ -82,70 +103,181 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
     end
 
     validate_format!
+    
+    # Run named Timer as daemon thread
+    @timer = java.util.Timer.new("HTTP Output #{self.params['id']}", true)
   end # def register
 
   def multi_receive(events)
-    events.each {|event| receive(event, :parallel)}
-    client.execute!
+    send_events(events)
   end
-
-  # Once we no longer need to support Logstash < 2.2 (pre-ng-pipeline)
-  # We don't need to handle :background style requests
-  #
-  # We use :background style requests for Logstash < 2.2 because before the microbatching
-  # pipeline performance is greatly improved by having some degree of async behavior.
-  #
-  # In Logstash 2.2 and after things are much simpler, we just run each batch in parallel
-  # This will make performance much easier to reason about, and more importantly let us guarantee
-  # that if `multi_receive` returns all items have been sent.
-  def receive(event, async_type=:background)
+  
+  class RetryTimerTask < java.util.TimerTask
+    def initialize(pending, event, attempt)
+      @pending = pending
+      @event = event
+      @attempt = attempt
+      super()
+    end
+    
+    def run
+      @pending << [@event, @attempt]
+    end
+  end
+  
+  def send_events(events)
+    successes = java.util.concurrent.atomic.AtomicInteger.new(0)
+    failures  = java.util.concurrent.atomic.AtomicInteger.new(0)
+    retries = java.util.concurrent.atomic.AtomicInteger.new(0)
+    
+    pending = Queue.new
+    events.each {|e| pending << [e, 0]}
+    
+    while popped = pending.pop
+      break if popped == :done
+      
+      event, attempt = popped
+      
+      send_event(event, attempt) do |action,event,attempt|
+        begin 
+          action = :failure if action == :retry && !@retry_failed
+          
+          case action
+          when :success
+            successes.incrementAndGet
+          when :retry
+            retries.incrementAndGet
+            
+            next_attempt = attempt+1
+            sleep_for = sleep_for_attempt(next_attempt)
+            @logger.info("Retrying http request, will sleep for #{sleep_for} seconds")
+            timer_task = RetryTimerTask.new(pending, event, next_attempt)
+            @timer.schedule(timer_task, sleep_for*1000)
+          when :failure 
+            failures.incrementAndGet
+          else
+            raise "Unknown action #{action}"
+          end
+          
+          if action == :success || action == :failure 
+            if successes.get+failures.get == events.size
+              pending << :done
+            end
+          end
+        rescue => e 
+          # This should never happen unless there's a flat out bug in the code
+          @logger.error("Error sending HTTP Request",
+            :class => e.class.name,
+            :message => e.message,
+            :backtrace => e.backtrace)
+          failures.incrementAndGet
+          raise e
+        end
+      end
+    end
+  rescue => e
+    @logger.error("Error in http output loop",
+            :class => e.class.name,
+            :message => e.message,
+            :backtrace => e.backtrace)
+    raise e
+  end
+  
+  def sleep_for_attempt(attempt)
+    sleep_for = attempt**2
+    sleep_for = sleep_for <= 60 ? sleep_for : 60
+    (sleep_for/2) + (rand(0..sleep_for)/2)
+  end
+  
+  def send_event(event, attempt)
     body = event_body(event)
 
-    # Block waiting for a token
-    token = @request_tokens.pop if async_type == :background
-
     # Send the request
     url = event.sprintf(@url)
     headers = event_headers(event)
 
     # Create an async request
-    request = client.send(async_type).send(@http_method, url, :body => body, :headers => headers)
-
-    request.on_complete do
-      # Make sure we return the token to the pool
-      @request_tokens << token  if async_type == :background
-    end
+    request = client.background.send(@http_method, url, :body => body, :headers => headers)
+    request.call # Actually invoke the request in the background
 
     request.on_success do |response|
-      if response.code < 200 || response.code > 299
-        log_failure(
-          "Encountered non-200 HTTP code #{200}",
-          :response_code => response.code,
-          :url => url,
-          :event => event)
+      begin
+        if !response_success?(response)
+          will_retry = retryable_response?(response)
+          log_failure(
+            "Encountered non-2xx HTTP code #{response.code}",
+            :response_code => response.code,
+            :url => url,
+            :event => event,
+            :will_retry => will_retry
+          )
+          
+          if will_retry 
+            yield :retry, event, attempt
+          else
+            yield :failure, event, attempt
+          end
+        else
+          yield :success, event, attempt
+        end
+      rescue => e 
+        # Shouldn't ever happen
+        @logger.error("Unexpected error in request success!",
+          :class => e.class.name,
+          :message => e.message,
+          :backtrace => e.backtrace)
       end
     end
 
     request.on_failure do |exception|
-      log_failure("Could not fetch URL",
-                  :url => url,
-                  :method => @http_method,
-                  :body => body,
-                  :headers => headers,
-                  :message => exception.message,
-                  :class => exception.class.name,
-                  :backtrace => exception.backtrace
-      )
+      begin 
+        will_retry = retryable_exception?(exception)
+        log_failure("Could not fetch URL",
+                    :url => url,
+                    :method => @http_method,
+                    :body => body,
+                    :headers => headers,
+                    :message => exception.message,
+                    :class => exception.class.name,
+                    :backtrace => exception.backtrace,
+                    :will_retry => will_retry
+        )
+        
+        if will_retry
+          yield :retry, event, attempt
+        else
+          yield :failure, event, attempt
+        end
+      rescue => e 
+        # Shouldn't ever happen
+        @logger.error("Unexpected error in request failure!",
+          :class => e.class.name,
+          :message => e.message,
+          :backtrace => e.backtrace)
+        end
     end
-
-    request.call if async_type == :background
   end
 
   def close
+    @timer.cancel
     client.close
   end
 
   private
+  
+  def response_success?(response)
+    code = response.code
+    return true if @ignorable_codes && @ignorable_codes.include?(code)
+    return code >= 200 && code <= 299
+  end
+  
+  def retryable_response?(response)
+    @retryable_codes.include?(response.code)
+  end
+  
+  def retryable_exception?(exception)
+    RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me) }
+  end
 
   # This is split into a separate method mostly to help testing
   def log_failure(message, opts)

data/logstash-output-http.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-http'
-  s.version         = '3.1.1'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This output lets you `PUT` or `POST` events to a generic HTTP(S) endpoint"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -27,4 +27,3 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'sinatra'
   s.add_development_dependency 'webrick'
 end
-

data/spec/outputs/http_spec.rb

@@ -1,5 +1,6 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/http"
+require "logstash/codecs/plain"
 require "thread"
 require "sinatra"
 
@@ -37,6 +38,14 @@ class TestApp < Sinatra::Base
   def self.last_request
     @last_request
   end
+  
+  def self.retry_fail_count=(count)
+    @retry_fail_count = count
+  end
+  
+  def self.retry_fail_count()
+    @retry_fail_count
+  end
 
   multiroute(%w(get post put patch delete), "/good") do
     self.class.last_request = request
@@ -45,7 +54,18 @@ class TestApp < Sinatra::Base
 
   multiroute(%w(get post put patch delete), "/bad") do
     self.class.last_request = request
-    [500, "YUP"]
+    [400, "YUP"]
+  end
+  
+  multiroute(%w(get post put patch delete), "/retry") do
+    self.class.last_request = request
+    
+    if self.class.retry_fail_count > 0
+      self.class.retry_fail_count -= 1
+      [429, "Will succeed in #{self.class.retry_fail_count}"]
+    else
+      [200, "Done Retrying"]
+    end
   end
 end
 
@@ -54,80 +74,48 @@ RSpec.configure do |config|
   def sinatra_run_wait(app, opts)
     queue = Queue.new
 
-    Thread.new(queue) do |queue|
-      begin
-        app.run!(opts) do |server|
-          queue.push("started")
+    t = java.lang.Thread.new(
+      proc do
+        begin
+          app.run!(opts) do |server|
+            queue.push("started")
+          end
+        rescue => e 
+          puts "Error in webserver thread #{e}"
+          # ignore
         end
-      rescue
-        # ignore
       end
-    end
-
+    )
+    t.daemon = true
+    t.start
     queue.pop # blocks until the run! callback runs
   end
 
   config.before(:suite) do
     sinatra_run_wait(TestApp, :port => PORT, :server => 'webrick')
+    puts "Test webserver on port #{PORT}"
   end
 end
 
 describe LogStash::Outputs::Http do
   # Wait for the async request to finish in this spinlock
   # Requires pool_max to be 1
-  def wait_for_request
-
-    loop do
-      sleep(0.1)
-      break if subject.request_tokens.size > 0
-    end
-  end
 
   let(:port) { PORT }
   let(:event) { LogStash::Event.new("message" => "hi") }
   let(:url) { "http://localhost:#{port}/good" }
   let(:method) { "post" }
 
-  describe "when num requests > token count" do
-    let(:pool_max) { 10 }
-    let(:num_reqs) { pool_max / 2 }
-    let(:client) { subject.client }
-    let(:client_proxy) { subject.client.background }
-
-    subject {
-      LogStash::Outputs::Http.new("url" => url,
-                                  "http_method" => method,
-                                  "pool_max" => pool_max)
-    }
-
-    before do
-      allow(client).to receive(:background).and_return(client_proxy)
-      subject.register
-    end
-
-    after do
-      subject.close
-    end
-
-    it "should receive all the requests" do
-      expect(client_proxy).to receive(:send).
-                          with(method.to_sym, url, anything).
-                          exactly(num_reqs).times.
-                          and_call_original
-
-      num_reqs.times {|t| subject.receive(event)}
-    end
-  end
-
-  shared_examples("verb behavior") do |method, async_type|
-    subject { LogStash::Outputs::Http.new("url" => url, "http_method" => method, "pool_max" => 1) }
+  shared_examples("verb behavior") do |method|
+    let(:verb_behavior_config) { {"url" => url, "http_method" => method, "pool_max" => 1} }
+    subject { LogStash::Outputs::Http.new(verb_behavior_config) }
 
     let(:expected_method) { method.clone.to_sym }
     let(:client) { subject.client }
-    let(:client_proxy) { subject.client.send(async_type) }
+    let(:client_proxy) { subject.client.background }
 
     before do
-      allow(client).to receive(async_type).and_return(client_proxy)
+      allow(client).to receive(:background).and_return(client_proxy)
       subject.register
       allow(client_proxy).to receive(:send).
                          with(expected_method, url, anything).
@@ -138,7 +126,7 @@ describe LogStash::Outputs::Http do
     context "performing a get" do
       describe "invoking the request" do
         before do
-          subject.receive(event, async_type)
+          subject.multi_receive([event])
         end
 
         it "should execute the request" do
@@ -149,7 +137,7 @@ describe LogStash::Outputs::Http do
 
       context "with passing requests" do
         before do
-          subject.receive(event)
+          subject.multi_receive([event])
         end
 
         it "should not log a failure" do
@@ -161,29 +149,51 @@ describe LogStash::Outputs::Http do
         let(:url) { "http://localhost:#{port}/bad"}
 
         before do
-          subject.receive(event, async_type)
-
-          if async_type == :background
-            wait_for_request
-          else
-            subject.client.execute!
-          end
+          subject.multi_receive([event])
         end
 
         it "should log a failure" do
           expect(subject).to have_received(:log_failure).with(any_args)
         end
       end
+      
+      context "with ignorable failing requests" do
+        let(:url) { "http://localhost:#{port}/bad"}
+        let(:verb_behavior_config) { super.merge("ignorable_codes" => [400]) }
+
+        before do
+          subject.multi_receive([event])
+        end
+
+        it "should log a failure" do
+          expect(subject).not_to have_received(:log_failure).with(any_args)
+        end
+      end
+      
+      context "with retryable failing requests" do
+        let(:url) { "http://localhost:#{port}/retry"}
+
+        before do
+          TestApp.retry_fail_count=2
+          allow(subject).to receive(:send_event).and_call_original
+          subject.multi_receive([event])
+        end
+
+        it "should log a failure 2 times" do
+          expect(subject).to have_received(:log_failure).with(any_args).twice
+        end
+        
+        it "should make three total requests" do
+          expect(subject).to have_received(:send_event).exactly(3).times
+        end
+      end  
+      
     end
   end
 
   LogStash::Outputs::Http::VALID_METHODS.each do |method|
-    context "when using '#{method}' via :background" do
-      include_examples("verb behavior", method, :background)
-    end
-
-    context "when using '#{method}' via :parallel" do
-      include_examples("verb behavior", method, :parallel)
+    context "when using '#{method}'" do
+      include_examples("verb behavior", method)
     end
   end
 
@@ -193,8 +203,7 @@ describe LogStash::Outputs::Http do
     end
 
     before do
-      subject.receive(event)
-      wait_for_request
+      subject.multi_receive([event])
     end
 
     let(:last_request) { TestApp.last_request }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-http
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2017-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -129,7 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This output lets you `PUT` or `POST` events to a generic HTTP(S) endpoint