logstash-input-http 3.0.7 → 3.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0cfc2c0ffdec2163b2bd28db4cac5e12509396828a8ed59ea768372a13c2726
-  data.tar.gz: 82bc2c8ae080551d5f3046870c26f0b28761a4ad9fbd9c716ca2a877a8dcc5f4
+  metadata.gz: 1975dd1f7608f77b01121212cb01df2a95c23eef97b40613ea1b5161bebe8772
+  data.tar.gz: 2de2626b158b0ee92fded043dfb6f32c11c0fdab2c515972157ef48e6e10a5cb
 SHA512:
-  metadata.gz: '06696c71d5be92acffde997e3f5f3cd591acb06bbe14237c70fa68e30343cd4a84f6323f8c826eb17d63afb08397aee162bfc414bda63f02f80f1ab039dc1e8c'
-  data.tar.gz: 7990cf7ed10d3e0b723becc3d01efb3da51f10c2fc895e0b8d8614dc4f87f2d0ce4f45c7fca0c675dd204cad27f42eaaffcd78067eb9adff9e21153f465324f8
+  metadata.gz: b1088e09605455d87a480afdeaf1b41aa1bae213b5cdb3ee91def0180589251b45748a6c3980eb895d24511c1371f71ac0546a5ce0b72849dd5f9fbe89f285cc
+  data.tar.gz: db68457ace06371dcaa312a9a2fa2ad3d96a7cfd1fe047a7e0c8cfed3a0a43c36521a73bf3d15fc39cdf947faf1b1f2f7ad590ea6c546cbb44df5b5bea3b7f8f

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.0.8
+  - In the event that all webserver threads are busy this plugin will now return a 429, busy, error.
+
 ## 3.0.7
   - Update gemspec summary
 

data/docs/index.asciidoc

@@ -31,6 +31,19 @@ This input can also be used to receive webhook requests to integrate with other
 and applications. By taking advantage of the vast plugin ecosystem available in Logstash
 you can trigger actionable events right from your application.
 
+==== Blocking Behavior
+
+The HTTP protocol doesn't deal well with long running requests. This plugin will either return 
+a 429 (busy) error when Logstash is backlogged, or it will time out the request. 
+
+If a 429 error is encountered clients should sleep, backing off exponentially with some random 
+jitter, then retry their request.
+
+This plugin will block if the Logstash queue is blocked and there are available HTTP input threads.
+This will cause most HTTP clients to time out. Sent events will still be processed in this case. This
+behavior is not optimal and will be changed in a future release. In the future, this plugin will always 
+return a 429 if the queue is busy, and will not time out in the event of a busy queue. 
+
 ==== Security
 This plugin supports standard HTTP basic authentication headers to identify the requester.
 You can pass in a username, password combination while sending data to this input

data/lib/logstash/inputs/http.rb

@@ -120,16 +120,24 @@ class LogStash::Inputs::Http < LogStash::Inputs::Base
       @server.add_tcp_listener(@host, @port)
     end
     @server.min_threads = 0
-    @server.max_threads = @threads
+    # The actual number of threads is one higher to let us reject additional requests
+    @server.max_threads = @threads + 1
     @codecs = Hash.new
+
     @additional_codecs.each do |content_type, codec|
       @codecs[content_type] = LogStash::Plugin.lookup("codec", codec).new
     end
-    @codec_lock = java.util.concurrent.locks.ReentrantLock.new
+
+    @write_slots = java.util.concurrent.ArrayBlockingQueue.new(threads)
+    threads.times do
+      # Freeze these guys just in case, since they aren't threadsafe
+      @write_slots.put(Hash[@codecs.map {|k,v| [k.freeze, v.clone].freeze }.freeze].freeze)
+    end
+
   end # def register
 
+  BUSY_RESPONSE = ['Server busy, please retry request later'.freeze].freeze
   def run(queue)
-
     # proc needs to be defined at this context
     # to capture @codecs, @logger and lowercase_keys
     p = Proc.new do |req|
@@ -138,16 +146,20 @@ class LogStash::Inputs::Http < LogStash::Inputs::Base
         REJECTED_HEADERS.each {|k| req.delete(k) }
         req = lowercase_keys(req)
         body = req.delete("rack.input")
-        @codec_lock.lock
+        local_codecs = @write_slots.poll()
+        if !local_codecs # No free write slot
+          next [429, {}, BUSY_RESPONSE]
+        end
         begin
-          @codecs.fetch(req["content_type"], @codec).decode(body.read) do |event|
+          codec = local_codecs.fetch(req["content_type"], @codec)
+          codec.decode(body.read) do |event|
             event.set("host", remote_host)
             event.set("headers", req)
             decorate(event)
             queue << event
           end
         ensure
-          @codec_lock.unlock
+          @write_slots.put(local_codecs)
         end
         ['200', @response_headers, ['ok']]
       rescue => e

data/logstash-input-http.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-http'
-  s.version         = '3.0.7'
+  s.version         = '3.0.8'
   s.licenses = ['Apache License (2.0)']
   s.summary = "Receives events over HTTP or HTTPS"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/http_spec.rb

@@ -25,13 +25,39 @@ describe LogStash::Inputs::Http do
   end
 
   describe "request handling" do
-    subject { LogStash::Inputs::Http.new }
+    subject { LogStash::Inputs::Http.new() }
     before :each do
       subject.register
       t = Thread.new { subject.run(queue) }
       sleep 0.01 until subject.instance_variable_get(:@server).running == 0
     end
 
+    describe "handling overflowing requests with a 429" do
+      let(:queue) { SizedQueue.new(1) }
+      let(:options) { { "threads" => 2 } }
+
+      def do_post
+        FTW::Agent.new.post!("http://localhost:8080/meh.json",
+            :headers => { "content-type" => "text/plain" },
+            :body => "hello")
+      end
+
+      context "when sending more requests than than queue slots" do
+        it "should block when the queue is full" do
+          threads = (subject.threads+5).times.map do # Add one request to the queue then fill the two slots
+            Thread.new { do_post } # These threads should block
+          end
+
+          expect(do_post.status).to eq(429)
+
+          Thread.new do
+            while queue.pop
+            end
+          end
+        end
+      end
+    end
+
     it "should include remote host in \"host\" property" do
       agent.post!("http://localhost:8080/meh.json",
                   :headers => { "content-type" => "text/plain" },

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-http
 version: !ruby/object:Gem::Version
-  version: 3.0.7
+  version: 3.0.8
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2017-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -175,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Receives events over HTTP or HTTPS