logstash-input-http_poller 5.3.0 → 5.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59101199a9b6eda278187132d5e5fd719247a893a29917fb305178347f89bf1e
-  data.tar.gz: 10f4717d83cd6befd62a314939c27db5633e65cd7c83706478efe3c2f42b1983
+  metadata.gz: 510ac72c358c306a8377a8cd714a1f58c867283b792e68f0374c79737f758126
+  data.tar.gz: 8c375ba083f4d8e6b8b20ced9d2b5c81c7f1322e36c0115dcbedcdc91087d16d
 SHA512:
-  metadata.gz: e6eb58b9354d0e60aa5c2d5c1217f8fa769c3c70b8f207c453645b8e12dadd5b533193c8fcb60689319be0d6abdcba5e2de931f956a6b7974739d1cc516dad35
-  data.tar.gz: 15dcd344b59b3a14184153b82f5be9375427a5ff3cbdf2103922a4c8800624957ed86bff73c45bdf854ac52577e7a899f35c750d123ca8b8fc870e005e36cd5a
+  metadata.gz: 54b5171093fc938cb49e273cad550c6430179812b1c47d34b6fd1c307281612ad872a50aee10f420e39002a46e7a7b143162ae331a90bdc42fc4a534e2666762
+  data.tar.gz: 87bc152a391d0e76942ead58d80679ce5581275f89863e6e629248751713f17052ecef2d5ca7dcd778d776828be7f20e5fd9338ad4c181d0e5d9ec837faea775

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 5.3.1
+  - Fix: make sure plugin is closing the http client [#130](https://github.com/logstash-plugins/logstash-input-http_poller/pull/130)
+
 ## 5.3.0
   - Feat: added ssl_supported_protocols option [#133](https://github.com/logstash-plugins/logstash-input-http_poller/pull/133)
 

data/lib/logstash/inputs/http_poller.rb

@@ -49,19 +49,35 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   def register
     @host = Socket.gethostname.force_encoding(Encoding::UTF_8)
 
-    @logger.info("Registering http_poller Input", :type => @type, :schedule => @schedule, :timeout => @timeout)
-
     setup_ecs_field!
     setup_requests!
   end
 
   # @overload
   def stop
+    shutdown_scheduler_and_close_client(:wait)
+  end
+
+  # @overload
+  def close
+    shutdown_scheduler_and_close_client
+  end
+
+  def shutdown_scheduler_and_close_client(opt = nil)
+    @logger.debug("closing http client", client: client)
+    begin
+      client.close # since Manticore 0.9.0 this shuts-down/closes all resources
+    rescue => e
+      details = { exception: e.class, message: e.message }
+      details[:backtrace] = e.backtrace if @logger.debug?
+      @logger.warn "failed closing http client", details
+    end
     if @scheduler
-      @scheduler.shutdown # on newer Rufus (3.8) this joins on the scheduler thread
+      @logger.debug("shutting down scheduler", scheduler: @scheduler)
+      @scheduler.shutdown(opt) # on newer Rufus (3.8) this joins on the scheduler thread
     end
-    # TODO implement client.close as we as releasing it's pooled resources!
   end
+  private :shutdown_scheduler_and_close_client
 
   private
   def setup_requests!
@@ -179,15 +195,18 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
 
   def run_once(queue)
     @requests.each do |name, request|
+      # prevent executing a scheduler kick after the plugin has been stop-ed
+      # this could easily happen as the scheduler shutdown is not immediate
+      return if stop?
       request_async(queue, name, request)
     end
 
-    client.execute!
+    client.execute! unless stop?
   end
 
   private
   def request_async(queue, name, request)
-    @logger.debug? && @logger.debug("Fetching URL", :name => name, :url => request)
+    @logger.debug? && @logger.debug("async queueing fetching url", name: name, url: request)
     started = Time.now
 
     method, *request_opts = request
@@ -204,6 +223,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
 
   private
   def handle_success(queue, name, request, response, execution_time)
+    @logger.debug? && @logger.debug("success fetching url", name: name, url: request)
     body = response.body
     # If there is a usable response. HEAD requests are `nil` and empty get
     # responses come up as "" which will cause the codec to not yield anything
@@ -242,6 +262,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   private
   # Beware, on old versions of manticore some uncommon failures are not handled
   def handle_failure(queue, name, request, exception, execution_time)
+    @logger.debug? && @logger.debug("failed fetching url", name: name, url: request)
     event = event_factory.new_event
     event.tag("_http_request_failure")
     apply_metadata(event, name, request, nil, execution_time)

data/logstash-input-http_poller.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'logstash-input-http_poller'
-  s.version     = '5.3.0'
+  s.version     = '5.3.1'
   s.licenses    = ['Apache License (2.0)']
   s.summary     = "Decodes the output of an HTTP API into events"
   s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/http_poller_spec.rb

@@ -7,6 +7,15 @@ require "timecop"
 require 'rspec/matchers/built_in/raise_error.rb'
 require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 
+begin
+  # TODO: CI work-around - will most likely be moved to the scheduler mixin
+  require 'et-orbi.rb' # a dependency of rufus-scheduler since 3.4
+  ::EtOrbi::EoTime.now # might take a long time to initialize - loading time zone
+  # data (from tz-info) and thus gets un-predictable on CI, since the scheduler worker
+  # thread might be stuck starting while we attempt to shutdown in a given time frame
+rescue LoadError
+end
+
 describe LogStash::Inputs::HTTP_Poller do
   let(:metadata_target) { "_http_poller_metadata" }
   let(:queue) { Queue.new }
@@ -201,8 +210,8 @@ describe LogStash::Inputs::HTTP_Poller do
       end
 
       it "should run at the schedule" do
-        run_plugin_and_yield_queue(plugin, sleep: 3) do |queue|
-          try(5) { expect(queue.size).to be >= 2 }
+        run_plugin_and_yield_queue(plugin, sleep: 3.1) do |queue|
+          try(10) { expect(queue.size).to be >= 2 }
         end
       end
     end
@@ -227,8 +236,8 @@ describe LogStash::Inputs::HTTP_Poller do
       end
 
       it "should run at the schedule" do
-        run_plugin_and_yield_queue(plugin, sleep: 2) do |queue|
-          try(5) { expect(queue.size).to eq(1) }
+        run_plugin_and_yield_queue(plugin, sleep: 4.1) do |queue|
+          try(10) { expect(queue.size).to eq(1) }
         end
       end
     end
@@ -247,7 +256,7 @@ describe LogStash::Inputs::HTTP_Poller do
           #T       0123456
           #events  x x x x
           #expects 3 events at T=5
-          try(5) { expect(queue.size).to be_between(2, 3) }
+          try(10) { expect(queue.size).to be_between(2, 3) }
         end
       end
     end
@@ -255,15 +264,15 @@ describe LogStash::Inputs::HTTP_Poller do
     context "given 'in' expression" do
       let(:opts) {
         {
-          "schedule" => { "in" => "2s"},
+          "schedule" => { "in" => "1s"},
           "urls" => default_urls,
           "codec" => "json",
           "metadata_target" => metadata_target
         }
       }
       it "should run at the schedule" do
-        run_plugin_and_yield_queue(plugin, sleep: 2.5) do |queue|
-          try(5) { expect(queue.size).to eq(1) }
+        run_plugin_and_yield_queue(plugin, sleep: 2.05) do |queue|
+          try(10) { expect(queue.size).to eq(1) }
         end
       end
     end
@@ -410,14 +419,15 @@ describe LogStash::Inputs::HTTP_Poller do
         before do
           plugin.register
           u = url.is_a?(Hash) ? url["url"] : url # handle both complex specs and simple string URLs
-          plugin.client.stub(u,
-                               :body => response_body,
-                               :code => code
-          )
+          plugin.client.stub(u, :body => response_body, :code => code)
           allow(plugin).to receive(:decorate)
           plugin.send(:run_once, queue)
         end
 
+        after do
+          plugin.close
+        end
+
         it "should have a matching message" do
           expect(event.to_hash).to include(payload)
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-http_poller
 version: !ruby/object:Gem::Version
-  version: 5.3.0
+  version: 5.3.1
 platform: ruby
 authors:
 - Elastic
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-28 00:00:00.000000000 Z
+date: 2022-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement