logstash-output-http 2.0.5 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f9f8204839f653dead5f53cdf9de9158aeacd061
-  data.tar.gz: 68463deb8cad8e5f559604e379c8b5c7570178ff
+  metadata.gz: fae9178467d58576560fe35574cd6a2c66363d09
+  data.tar.gz: a631d6b89069d8c42a0bbf07a5e187cfdadd4bd0
 SHA512:
-  metadata.gz: 9ccf7a294708db2364966b117365001aa9c1f176efd85e1fc3f924409e565ccf6b0232e9418c50ce5a34c8944cd435e77fa28b3374b896501896ad47f7d4f280
-  data.tar.gz: 479985e604fabcf939fc34c93dab8adc3a98f600228a5474d79b0fd6478232f93511ba49822c97cb3a9a73e9c9f3003925f972198d0b3ae692d65b413bc06c8b
+  metadata.gz: 0232e0887bcda5e10903210fb6d0556aad188deca1f64f1415ce697024f06f2d387211d59e8446b6e6cdaa88074f9618452ed1b3108b53d2436f28b8e23b2cd4
+  data.tar.gz: 74ffbf0a24cb748f8ba2f3518ed321c6366295ab3a7de7f7cccd01540b757d723840075206f5a9c744be0ca8bd2784329f9be6f190feeeeaebe61f74090e3387

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.1.0
+ - Properly close the client on #close
+ - Optimized execution for Logstash 2.2 ng pipeline
+
 ## 2.0.5
  - fixed memory leak
 

data/README.md

@@ -1,5 +1,7 @@
 # Logstash Plugin
 
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-http.svg)](https://travis-ci.org/logstash-plugins/logstash-output-http)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/outputs/http.rb

@@ -89,31 +89,36 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
     validate_format!
   end # def register
 
-  # TODO: (colin) the request call sequence + async handling will have to be reworked when using
-  # Maticore >= 5.0. I will set a version constrain in the gemspec for this.
-  def receive(event)
+  def multi_receive(events)
+    events.each {|event| receive(event, :parallel)}
+    client.execute!
+  end
+
+  # Once we no longer need to support Logstash < 2.2 (pre-ng-pipeline)
+  # We don't need to handle :background style requests
+  #
+  # We use :background style requests for Logstash < 2.2 because before the microbatching
+  # pipeline performance is greatly improved by having some degree of async behavior.
+  #
+  # In Logstash 2.2 and after things are much simpler, we just run each batch in parallel
+  # This will make performance much easier to reason about, and more importantly let us guarantee
+  # that if `multi_receive` returns all items have been sent.
+  def receive(event, async_type=:background)
     body = event_body(event)
 
     # Block waiting for a token
-    token = @request_tokens.pop
+    token = @request_tokens.pop if async_type == :background
 
     # Send the request
     url = event.sprintf(@url)
     headers = event_headers(event)
 
     # Create an async request
-    request = client.send(@http_method, url, :body => body, :headers => headers, :async => true)
-
-    # with Maticore version < 0.5 using :async => true places the requests in an @async_requests
-    # list which is used & cleaned by Client#execute! but we are not using it here and we must
-    # purge it manually to avoid leaking requests.
-    client.clear_pending
-
-    # attach handlers before performing request
+    request = client.send(async_type).send(@http_method, url, :body => body, :headers => headers)
 
     request.on_complete do
       # Make sure we return the token to the pool
-      @request_tokens << token
+      @request_tokens << token  if async_type == :background
     end
 
     request.on_success do |response|
@@ -138,8 +143,11 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
       )
     end
 
-    # Invoke it using the Manticore Executor (CachedThreadPool) directly
-    request_async_background(request)
+    request.call if async_type == :background
+  end
+
+  def close
+    client.close
   end
 
   private

data/logstash-output-http.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-http'
-  s.version         = '2.0.5'
+  s.version         = '2.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This output lets you `PUT` or `POST` events to a generic HTTP(S) endpoint"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,12 +20,8 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
-  s.add_runtime_dependency "logstash-mixin-http_client", ">= 2.0.2", "< 3.0.0"
-
-  # Constrain Maticore dependency to less than 0.5.0 because of changes in the async handling
-  # see note in http.rb line 92-93
-  s.add_runtime_dependency "manticore", "< 0.5.0"
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 2.2.0", "< 3.0.0"
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'sinatra'

data/spec/outputs/http_spec.rb

@@ -92,6 +92,8 @@ describe LogStash::Outputs::Http do
     let(:pool_max) { 10 }
     let(:num_reqs) { pool_max / 2 }
     let(:client) { subject.client }
+    let(:client_proxy) { subject.client.background }
+
     subject {
       LogStash::Outputs::Http.new("url" => url,
                                   "http_method" => method,
@@ -99,11 +101,16 @@ describe LogStash::Outputs::Http do
     }
 
     before do
+      allow(client).to receive(:background).and_return(client_proxy)
       subject.register
     end
 
+    after do
+      subject.close
+    end
+
     it "should receive all the requests" do
-      expect(client).to receive(:send).
+      expect(client_proxy).to receive(:send).
                           with(method.to_sym, url, anything).
                           exactly(num_reqs).times.
                           and_call_original
@@ -112,15 +119,17 @@ describe LogStash::Outputs::Http do
     end
   end
 
-  shared_examples("verb behavior") do |method|
+  shared_examples("verb behavior") do |method, async_type|
     subject { LogStash::Outputs::Http.new("url" => url, "http_method" => method, "pool_max" => 1) }
 
     let(:expected_method) { method.clone.to_sym }
     let(:client) { subject.client }
+    let(:client_proxy) { subject.client.send(async_type) }
 
     before do
+      allow(client).to receive(async_type).and_return(client_proxy)
       subject.register
-      allow(client).to receive(:send).
+      allow(client_proxy).to receive(:send).
                          with(expected_method, url, anything).
                          and_call_original
       allow(subject).to receive(:log_failure).with(any_args)
@@ -129,11 +138,11 @@ describe LogStash::Outputs::Http do
     context "performing a get" do
       describe "invoking the request" do
         before do
-          subject.receive(event)
+          subject.receive(event, async_type)
         end
 
         it "should execute the request" do
-          expect(client).to have_received(:send).
+          expect(client_proxy).to have_received(:send).
                               with(expected_method, url, anything)
         end
       end
@@ -152,8 +161,13 @@ describe LogStash::Outputs::Http do
         let(:url) { "http://localhost:#{port}/bad"}
 
         before do
-          subject.receive(event)
-          wait_for_request
+          subject.receive(event, async_type)
+
+          if async_type == :background
+            wait_for_request
+          else
+            subject.client.execute!
+          end
         end
 
         it "should log a failure" do
@@ -164,8 +178,12 @@ describe LogStash::Outputs::Http do
   end
 
   LogStash::Outputs::Http::VALID_METHODS.each do |method|
-    context "when using '#{method}'" do
-      include_examples("verb behavior", method)
+    context "when using '#{method}' via :background" do
+      include_examples("verb behavior", method, :background)
+    end
+
+    context "when using '#{method}' via :parallel" do
+      include_examples("verb behavior", method, :parallel)
     end
   end
 

metadata

@@ -1,126 +1,112 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-http
 version: !ruby/object:Gem::Version
-  version: 2.0.5
+  version: 2.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-16 00:00:00.000000000 Z
+date: 2015-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
-      - !ruby/object:Gem::Version
-        version: 3.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
+        version: 2.0.0
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+  name: logstash-core
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-mixin-http_client
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 2.0.0
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 2.2.0
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+  name: logstash-mixin-http_client
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: manticore
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - <
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.5.0
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
+        version: 2.2.0
     - - <
       - !ruby/object:Gem::Version
-        version: 0.5.0
-  prerelease: false
-  type: :runtime
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: sinatra
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: sinatra
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: webrick
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: webrick
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/logstash/outputs/http.rb
-- spec/outputs/http_spec.rb
-- logstash-output-http.gemspec
-- README.md
 - CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
+- README.md
+- lib/logstash/outputs/http.rb
+- logstash-output-http.gemspec
+- spec/outputs/http_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -143,7 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This output lets you `PUT` or `POST` events to a generic HTTP(S) endpoint