logstash-output-http 5.1.2 → 5.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0def77507d3ea7b9d83ede431e0853c8f16e77f6249b11b62328e40cd3a0c403
-  data.tar.gz: 85108445896c3ea59793b997e2b73f28dbe7d4fa05cb028af09b3727a8df6e9e
+  metadata.gz: 74a3adbb0ae7188d50a8c025f7fd256cfb4adb36e4e17862c6d64ae6b6b6c8ee
+  data.tar.gz: 72e55c3f943a81d2e5e89e4a52ba54a39495e7f9ea3e684273822d7dc59d7ec4
 SHA512:
-  metadata.gz: ee4d0f5fc3152eed88be134bc5c72d99d6f2dccce346fd5bf45bcb3d4ebef72fba9b8b57c2f34dd5053f6b6550fd6a458837edc2bfd0c17d8ff3881f736a4be9
-  data.tar.gz: eaec797c6f1d257c0c4b8ad99d4e8fc329508ec088536f5a204dd169cbeb84efbf2649a753a8684924211be6b51eedd26acef822d8c38dea077dc9399bd634c1
+  metadata.gz: d7d0109785a2fe03f9936e6a7d56cf6406d3c81c1f66f7afdae0d45fe9b998dfdd95fe1e5970ccec15b69f6f884e9104aacfc7ca063a83d7feae3b35e278bb8a
+  data.tar.gz: 074e51a776ec7c1b12f3fbb33ccc9da9d6bd1188b32884bd6403506709c09083c930420550f9ee5ce2d8c5a4aff32dcc709972df71a53bb69c6a05044b1580fe

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 5.2.0
+  - Added json_batch format
+  - Make 429 responses log at debug, not error level. They are really just flow control
+
 ## 5.1.2
   - Add check to avoid hanging pipeline if an empty event array is passed in. #80
 

data/docs/index.asciidoc

@@ -46,7 +46,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-content_type>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cookies>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-follow_redirects>> |<<boolean,boolean>>|No
-| <<plugins-{type}s-{plugin}-format>> |<<string,string>>, one of `["json", "form", "message"]`|No
+| <<plugins-{type}s-{plugin}-format>> |<<string,string>>, one of `["json", "json_batch", "form", "message"]`|No
 | <<plugins-{type}s-{plugin}-headers>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-http_compression>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-http_method>> |<<string,string>>, one of `["put", "post", "patch", "delete", "get", "head"]`|Yes
@@ -130,6 +130,7 @@ Content type
 If not specified, this defaults to the following:
 
 * if format is "json", "application/json"
+* if format is "json_batch", "application/json". Each Logstash batch of events will be concatenated into a single array and sent in one request.
 * if format is "form", "application/x-www-form-urlencoded"
 
 [id="plugins-{type}s-{plugin}-cookies"]
@@ -152,11 +153,15 @@ Should redirects be followed? Defaults to `true`
 [id="plugins-{type}s-{plugin}-format"]
 ===== `format` 
 
-  * Value can be any of: `json`, `form`, `message`
+  * Value can be any of: `json`, `json_batch`, `form`, `message`
   * Default value is `"json"`
 
 Set the format of the http body.
 
+If json_batch, each batch of events received by this output will be placed
+into a single JSON array and sent in one request. This is particularly useful
+for high throughput scenarios such as sending data between Logstash instaces.
+
 If form, then the body will be the mapping (or whole event) converted
 into a query parameter string, e.g. `foo=bar&baz=fizz...`
 

data/lib/logstash/outputs/http.rb

@@ -42,7 +42,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
 
   # Custom headers to use
   # format is `headers => ["X-My-Header", "%{host}"]`
-  config :headers, :validate => :hash
+  config :headers, :validate => :hash, :default => {}
 
   # Content type
   #
@@ -79,7 +79,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
   # If message, then the body will be the result of formatting the event according to message
   #
   # Otherwise, the event is sent as json.
-  config :format, :validate => ["json", "form", "message"], :default => "json"
+  config :format, :validate => ["json", "json_batch", "form", "message"], :default => "json"
 
   # Set this to true if you want to enable gzip compression for your http requests
   config :http_compression, :validate => :boolean, :default => false
@@ -102,10 +102,14 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
       case @format
         when "form" ; @content_type = "application/x-www-form-urlencoded"
         when "json" ; @content_type = "application/json"
+        when "json_batch" ; @content_type = "application/json"
         when "message" ; @content_type = "text/plain"
       end
     end
 
+
+    @headers["Content-Type"] = @content_type
+
     validate_format!
     
     # Run named Timer as daemon thread
@@ -113,7 +117,12 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
   end # def register
 
   def multi_receive(events)
-    send_events(events) unless events.empty?
+    return if events.empty?
+    if @format == "json_batch"
+      send_json_batch(events)
+    else
+      send_events(events)
+    end
   end
   
   class RetryTimerTask < java.util.TimerTask
@@ -128,6 +137,46 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
       @pending << [@event, @attempt]
     end
   end
+
+  def send_json_batch(events)
+    attempt = 1
+    body = LogStash::Json.dump(events.map {|e| map_event(e) })
+    begin
+      while true
+        request = client.send(@http_method, @url, :body => body, :headers => @headers)
+        response = request.call
+        break if response_success?(response)
+        if retryable_response?(response)
+          log_retryable_response(response)
+          sleep_for_attempt attempt
+          attempt += 1
+        else
+          log_error_response(response, url, events)
+        end
+      end
+    rescue *RETRYABLE_MANTICORE_EXCEPTIONS => e
+      logger.warn("Encountered exception during http output send, will retry after delay",  :message => e.message, :class => e.class.name)
+      sleep_for_attempt attempt
+      retry
+    end
+  end
+
+  def log_retryable_response(response)
+    if (response.code == 429)
+      @logger.debug? && @logger.debug("Encountered a 429 response, will retry. This is not serious, just flow control via HTTP")
+    else
+      @logger.warn("Encountered a retryable HTTP request in HTTP output, will retry", :code => response.code, :body => response.body)
+    end
+  end
+
+  def log_error_response(response, url, event)
+    log_failure(
+              "Encountered non-2xx HTTP code #{response.code}",
+              :response_code => response.code,
+              :url => url,
+              :event => event
+            )
+  end
   
   def send_events(events)
     successes = java.util.concurrent.atomic.AtomicInteger.new(0)
@@ -212,18 +261,11 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
     request.on_success do |response|
       begin
         if !response_success?(response)
-          will_retry = retryable_response?(response)
-          log_failure(
-            "Encountered non-2xx HTTP code #{response.code}",
-            :response_code => response.code,
-            :url => url,
-            :event => event,
-            :will_retry => will_retry
-          )
-          
-          if will_retry 
+          if retryable_response?(response)
+            log_retryable_response(response)
             yield :retry, event, attempt
           else
+            log_error_response(response, url, event)
             yield :failure, event, attempt
           end
         else
@@ -343,9 +385,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
   end
 
   def event_headers(event)
-    headers = custom_headers(event) || {}
-    headers["Content-Type"] = @content_type
-    headers
+    custom_headers(event) || {}
   end
 
   def custom_headers(event)

data/logstash-output-http.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-http'
-  s.version         = '5.1.2'
+  s.version         = '5.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Sends events to a generic HTTP or HTTPS endpoint"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/http_spec.rb

@@ -47,7 +47,7 @@ class TestApp < Sinatra::Base
   end
 
   def self.retry_fail_count()
-    @retry_fail_count
+    @retry_fail_count || 2
   end
 
   multiroute(%w(get post put patch delete), "/good") do
@@ -126,6 +126,7 @@ describe LogStash::Outputs::Http do
                          with(expected_method, url, anything).
                          and_call_original
       allow(subject).to receive(:log_failure).with(any_args)
+      allow(subject).to receive(:log_retryable_response).with(any_args)
     end
 
     context 'sending no events' do
@@ -190,8 +191,8 @@ describe LogStash::Outputs::Http do
           subject.multi_receive([event])
         end
 
-        it "should log a failure 2 times" do
-          expect(subject).to have_received(:log_failure).with(any_args).twice
+        it "should log a retryable response 2 times" do
+          expect(subject).to have_received(:log_retryable_response).with(any_args).twice
         end
 
         it "should make three total requests" do
@@ -213,24 +214,43 @@ describe LogStash::Outputs::Http do
       TestApp.last_request = nil
     end
 
-    before do
-      subject.multi_receive([event])
-    end
+    let(:events) { [event] }
 
-    let(:last_request) { TestApp.last_request }
-    let(:body) { last_request.body.read }
-    let(:content_type) { last_request.env["CONTENT_TYPE"] }
+    describe "with a good code" do
+      before do
+        subject.multi_receive(events)
+      end
 
-    it "should receive the request" do
-      expect(last_request).to be_truthy
-    end
+      let(:last_request) { TestApp.last_request }
+      let(:body) { last_request.body.read }
+      let(:content_type) { last_request.env["CONTENT_TYPE"] }
+
+      it "should receive the request" do
+        expect(last_request).to be_truthy
+      end
+
+      it "should receive the event as a hash" do
+        expect(body).to eql(expected_body)
+      end
 
-    it "should receive the event as a hash" do
-      expect(body).to eql(expected_body)
+      it "should have the correct content type" do
+        expect(content_type).to eql(expected_content_type)
+      end
     end
 
-    it "should have the correct content type" do
-      expect(content_type).to eql(expected_content_type)
+    describe "a retryable code" do
+      let(:url) { "http://localhost:#{port}/retry" }
+
+      before do
+        TestApp.retry_fail_count=2
+        allow(subject).to receive(:send_event).and_call_original
+        allow(subject).to receive(:log_retryable_response)
+        subject.multi_receive(events)
+      end
+
+      it "should retry" do
+        expect(subject).to have_received(:log_retryable_response).with(any_args).twice
+      end
     end
   end
 
@@ -257,6 +277,19 @@ describe LogStash::Outputs::Http do
       include_examples("a received event")
     end
 
+    describe "sending the batch as JSON" do
+      let(:config) do
+        base_config.merge({"url" => url, "http_method" => "post", "format" => "json_batch"})
+      end
+
+      let(:expected_body) { ::LogStash::Json.dump events }
+      let(:events) { [::LogStash::Event.new("a" => 1), ::LogStash::Event.new("b" => 2)]}
+      let(:expected_content_type) { "application/json" }
+      
+      include_examples("a received event")
+
+    end
+
     describe "sending the event as a form" do
       let(:config) {
         base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "form"})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-http
 version: !ruby/object:Gem::Version
-  version: 5.1.2
+  version: 5.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-09 00:00:00.000000000 Z
+date: 2018-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement