logstash-output-http 5.2.1 → 5.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a0d07cbf8fdcd4bbd6f98617208c4446785b64be7f314865d1dcb3f84a8d00d
-  data.tar.gz: 280a1a1097797e2822ef442168aa1a5a77114e5d1c8627c2e811d56adeb9e3f7
+  metadata.gz: bd00218e30610f309f7870230ffeff01b5d1481b38d8723cb2e938f2067a561b
+  data.tar.gz: 6c31789a86e548649b8ae995cab4e5927fe071d4491570d98c3ef454c28649fd
 SHA512:
-  metadata.gz: 022e139b5bebe07707c0df01ed0392e6eb5c553d7f1201e5546e2a9f386ea938b71a1cc18421e3a2445031925d21d75ecd3b56a1e2c4b0477f82cbc5e01cf554
-  data.tar.gz: 3c84c3a83ddd00dd8ba8059d6080114c08269ea233d41d1d768aeb94db9789fca628b57bb6e8137dc0b32c9a57fdec92b0bc2022439810a996cc162e83e82679
+  metadata.gz: 0ed20344862d2b1587d8a3b5cb8bb6b71de3c51ac75a91c85f5f1326185438f899ff08573de4613394c0fbae252c572e8c4996f05497e9cf081fc92c227182c1
+  data.tar.gz: 11248626b51ec811d865f761e01a7adf219b9967211c20abd71c901ff16d0e551201c7d931d209f19cf40fa6d7dad164e4dbe06f2c13ee2bfa7549bb8a16b1b3

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 5.2.2
+  - Fix high CPU usage on retries in json_batch mode. [#89](https://github.com/logstash-plugins/logstash-output-http/pull/89)
+  - Enable compression in json_batch mode. [#89](https://github.com/logstash-plugins/logstash-output-http/pull/89)
+
 ## 5.2.1
   - Docs: Set the default_codec doc attribute.
 

data/lib/logstash/outputs/http.rb

@@ -11,6 +11,8 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
   
   concurrency :shared
 
+  attr_accessor :is_batch
+
   VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"]
   
   RETRYABLE_MANTICORE_EXCEPTIONS = [
@@ -107,6 +109,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
       end
     end
 
+    @is_batch = @format == "json_batch"
 
     @headers["Content-Type"] = @content_type
 
@@ -118,11 +121,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
 
   def multi_receive(events)
     return if events.empty?
-    if @format == "json_batch"
-      send_json_batch(events)
-    else
-      send_events(events)
-    end
+    send_events(events)
   end
   
   class RetryTimerTask < java.util.TimerTask
@@ -138,29 +137,6 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
     end
   end
 
-  def send_json_batch(events)
-    attempt = 1
-    body = LogStash::Json.dump(events.map {|e| map_event(e) })
-    begin
-      while true
-        request = client.send(@http_method, @url, :body => body, :headers => @headers)
-        response = request.call
-        break if response_success?(response)
-        if retryable_response?(response)
-          log_retryable_response(response)
-          sleep_for_attempt attempt
-          attempt += 1
-        else
-          log_error_response(response, url, events)
-        end
-      end
-    rescue *RETRYABLE_MANTICORE_EXCEPTIONS => e
-      logger.warn("Encountered exception during http output send, will retry after delay",  :message => e.message, :class => e.class.name)
-      sleep_for_attempt attempt
-      retry
-    end
-  end
-
   def log_retryable_response(response)
     if (response.code == 429)
       @logger.debug? && @logger.debug("Encountered a 429 response, will retry. This is not serious, just flow control via HTTP")
@@ -182,15 +158,20 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
     successes = java.util.concurrent.atomic.AtomicInteger.new(0)
     failures  = java.util.concurrent.atomic.AtomicInteger.new(0)
     retries = java.util.concurrent.atomic.AtomicInteger.new(0)
-    
+    event_count = @is_batch ? 1 : events.size
+
     pending = Queue.new
-    events.each {|e| pending << [e, 0]}
+    if @is_batch
+      pending << [events, 0]
+    else
+      events.each {|e| pending << [e, 0]}
+    end
 
     while popped = pending.pop
       break if popped == :done
       
       event, attempt = popped
-      
+
       send_event(event, attempt) do |action,event,attempt|
         begin 
           action = :failure if action == :retry && !@retry_failed
@@ -213,7 +194,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
           end
           
           if action == :success || action == :failure 
-            if successes.get+failures.get == events.size
+            if successes.get+failures.get == event_count
               pending << :done
             end
           end
@@ -246,8 +227,8 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
     body = event_body(event)
 
     # Send the request
-    url = event.sprintf(@url)
-    headers = event_headers(event)
+    url = @is_batch ? @url : event.sprintf(@url)
+    headers = @is_batch ? @headers : event_headers(event)
 
     # Compress the body and add appropriate header
     if @http_compression == true
@@ -347,6 +328,8 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
       LogStash::Json.dump(map_event(event))
     elsif @format == "message"
       event.sprintf(@message)
+    elsif @format == "json_batch"
+      LogStash::Json.dump(event.map {|e| map_event(e) })
     else
       encode(map_event(event))
     end

data/logstash-output-http.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-http'
-  s.version         = '5.2.1'
+  s.version         = '5.2.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Sends events to a generic HTTP or HTTPS endpoint"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-http
 version: !ruby/object:Gem::Version
-  version: 5.2.1
+  version: 5.2.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-06 00:00:00.000000000 Z
+date: 2018-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -133,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Sends events to a generic HTTP or HTTPS endpoint