logstash-input-http_poller 5.2.0 → 5.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 512535463aaa56747ba5e9d6a264887588966a6df43518955deccb19f2b7acae
-  data.tar.gz: 6c9b5b63198545b35e019b8e4f70edcec9f74ea9e8b4e53d6982b3841f48598b
+  metadata.gz: 510ac72c358c306a8377a8cd714a1f58c867283b792e68f0374c79737f758126
+  data.tar.gz: 8c375ba083f4d8e6b8b20ced9d2b5c81c7f1322e36c0115dcbedcdc91087d16d
 SHA512:
-  metadata.gz: 52a07cfe047ab7774d5ed1279ac9c5434031ccd86f57a6b93996345c8f7c1e6fe4792b8525dee40ef0be14832a2bfb85be192a80021f67a59adae507a0e01b5f
-  data.tar.gz: e614b85fd0558f4bfda62c8cb5df8811249cd6ed2cfa9ebfeb67caab08feb133a00e29737e2a806c471b59bf4efd75ac5404f06be38488f8e70987ebe8441ef6
+  metadata.gz: 54b5171093fc938cb49e273cad550c6430179812b1c47d34b6fd1c307281612ad872a50aee10f420e39002a46e7a7b143162ae331a90bdc42fc4a534e2666762
+  data.tar.gz: 87bc152a391d0e76942ead58d80679ce5581275f89863e6e629248751713f17052ecef2d5ca7dcd778d776828be7f20e5fd9338ad4c181d0e5d9ec837faea775

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 5.3.1
+  - Fix: make sure plugin is closing the http client [#130](https://github.com/logstash-plugins/logstash-input-http_poller/pull/130)
+
+## 5.3.0
+  - Feat: added ssl_supported_protocols option [#133](https://github.com/logstash-plugins/logstash-input-http_poller/pull/133)
+
+## 5.2.1
+  - Deps: unpin rufus-scheduler dependency [#130](https://github.com/logstash-plugins/logstash-input-http_poller/pull/130)
+
 ## 5.2.0
   - Feat: support ssl_verification_mode option [#131](https://github.com/logstash-plugins/logstash-input-http_poller/pull/131)
 

data/Gemfile

@@ -9,3 +9,5 @@ if Dir.exist?(logstash_path) && use_logstash_source
   gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
 end
+
+gem 'rufus-scheduler', ENV['RUFUS_SCHEDULER_VERSION'] if ENV['RUFUS_SCHEDULER_VERSION']

data/docs/index.asciidoc

@@ -146,6 +146,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-retry_non_idempotent>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-schedule>> |<<hash,hash>>|Yes
 | <<plugins-{type}s-{plugin}-socket_timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
@@ -414,6 +415,23 @@ See: rufus/scheduler for details about different schedule options and value stri
 
 Timeout (in seconds) to wait for data on the socket. Default is `10s`
 
+[id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
+===== `ssl_supported_protocols`
+
+  * Value type is <<string,string>>
+  * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'`
+  * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`.
+    `'TLSv1.1'` is not considered secure and is only provided for legacy applications.
+
+List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint.
+
+For Java 8 `'TLSv1.3'` is supported  only since **8u262** (AdoptOpenJDK), but requires that you set the
+`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash.
+
+NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash,
+the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in
+the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list.
+
 [id="plugins-{type}s-{plugin}-ssl_verification_mode"]
 ===== `ssl_verification_mode`
 

data/lib/logstash/inputs/http_poller.rb

@@ -49,16 +49,35 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   def register
     @host = Socket.gethostname.force_encoding(Encoding::UTF_8)
 
-    @logger.info("Registering http_poller Input", :type => @type, :schedule => @schedule, :timeout => @timeout)
-
     setup_ecs_field!
     setup_requests!
   end
 
+  # @overload
   def stop
-    Stud.stop!(@interval_thread) if @interval_thread
-    @scheduler.stop if @scheduler
+    shutdown_scheduler_and_close_client(:wait)
+  end
+
+  # @overload
+  def close
+    shutdown_scheduler_and_close_client
+  end
+
+  def shutdown_scheduler_and_close_client(opt = nil)
+    @logger.debug("closing http client", client: client)
+    begin
+      client.close # since Manticore 0.9.0 this shuts-down/closes all resources
+    rescue => e
+      details = { exception: e.class, message: e.message }
+      details[:backtrace] = e.backtrace if @logger.debug?
+      @logger.warn "failed closing http client", details
+    end
+    if @scheduler
+      @logger.debug("shutting down scheduler", scheduler: @scheduler)
+      @scheduler.shutdown(opt) # on newer Rufus (3.8) this joins on the scheduler thread
+    end
   end
+  private :shutdown_scheduler_and_close_client
 
   private
   def setup_requests!
@@ -163,29 +182,31 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
     #schedule hash must contain exactly one of the allowed keys
     msg_invalid_schedule = "Invalid config. schedule hash must contain " +
       "exactly one of the following keys - cron, at, every or in"
-    raise Logstash::ConfigurationError, msg_invalid_schedule if @schedule.keys.length !=1
+    raise Logstash::ConfigurationError, msg_invalid_schedule if @schedule.keys.length != 1
     schedule_type = @schedule.keys.first
     schedule_value = @schedule[schedule_type]
     raise LogStash::ConfigurationError, msg_invalid_schedule unless Schedule_types.include?(schedule_type)
 
     @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
-    #as of v3.0.9, :first_in => :now doesn't work. Use the following workaround instead
     opts = schedule_type == "every" ? { :first_in => 0.01 } : {} 
     @scheduler.send(schedule_type, schedule_value, opts) { run_once(queue) }
-    @scheduler.join
+    @scheduler.thread.join # due newer rufus (3.8) doing a blocking operation on scheduler.join
   end
 
   def run_once(queue)
     @requests.each do |name, request|
+      # prevent executing a scheduler kick after the plugin has been stop-ed
+      # this could easily happen as the scheduler shutdown is not immediate
+      return if stop?
       request_async(queue, name, request)
     end
 
-    client.execute!
+    client.execute! unless stop?
   end
 
   private
   def request_async(queue, name, request)
-    @logger.debug? && @logger.debug("Fetching URL", :name => name, :url => request)
+    @logger.debug? && @logger.debug("async queueing fetching url", name: name, url: request)
     started = Time.now
 
     method, *request_opts = request
@@ -202,6 +223,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
 
   private
   def handle_success(queue, name, request, response, execution_time)
+    @logger.debug? && @logger.debug("success fetching url", name: name, url: request)
     body = response.body
     # If there is a usable response. HEAD requests are `nil` and empty get
     # responses come up as "" which will cause the codec to not yield anything
@@ -240,6 +262,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   private
   # Beware, on old versions of manticore some uncommon failures are not handled
   def handle_failure(queue, name, request, exception, execution_time)
+    @logger.debug? && @logger.debug("failed fetching url", name: name, url: request)
     event = event_factory.new_event
     event.tag("_http_request_failure")
     apply_metadata(event, name, request, nil, execution_time)

data/logstash-input-http_poller.gemspec

@@ -1,9 +1,9 @@
 Gem::Specification.new do |s|
   s.name        = 'logstash-input-http_poller'
-  s.version     = '5.2.0'
+  s.version     = '5.3.1'
   s.licenses    = ['Apache License (2.0)']
   s.summary     = "Decodes the output of an HTTP API into events"
-  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors     = [ "Elastic", "andrewvc"]
   s.email       = 'info@elastic.co'
   s.homepage    = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -20,9 +20,8 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.1.0"
-  s.add_runtime_dependency 'stud', "~> 0.0.22"
-  s.add_runtime_dependency 'rufus-scheduler', "~>3.0.9"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.2.0"
+  s.add_runtime_dependency 'rufus-scheduler', ">= 3.0.9"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
   s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0', '>= 1.0.1'
   s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'

data/spec/inputs/http_poller_spec.rb

@@ -7,6 +7,15 @@ require "timecop"
 require 'rspec/matchers/built_in/raise_error.rb'
 require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 
+begin
+  # TODO: CI work-around - will most likely be moved to the scheduler mixin
+  require 'et-orbi.rb' # a dependency of rufus-scheduler since 3.4
+  ::EtOrbi::EoTime.now # might take a long time to initialize - loading time zone
+  # data (from tz-info) and thus gets un-predictable on CI, since the scheduler worker
+  # thread might be stuck starting while we attempt to shutdown in a given time frame
+rescue LoadError
+end
+
 describe LogStash::Inputs::HTTP_Poller do
   let(:metadata_target) { "_http_poller_metadata" }
   let(:queue) { Queue.new }
@@ -25,18 +34,19 @@ describe LogStash::Inputs::HTTP_Poller do
       "schedule" => default_schedule,
       "urls" => default_urls,
       "codec" => "json",
-      "metadata_target" => metadata_target
+      "metadata_target" => metadata_target,
+      "pool_max" => 3, "pool_max_per_route" => 1, 'keepalive' => false
     }
   }
-  let(:klass) { LogStash::Inputs::HTTP_Poller }
+  let(:opts) { default_opts }
 
+  subject(:plugin) { described_class.new(opts) }
 
   describe "instances" do
-    subject { klass.new(default_opts) }
+    subject { described_class.new(default_opts) }
 
-    before do
-      subject.register
-    end
+    before { subject.register }
+    after { subject.stop }
 
     describe "#run" do
       it "should setup a scheduler" do
@@ -189,22 +199,21 @@ describe LogStash::Inputs::HTTP_Poller do
           "metadata_target" => metadata_target
         }
       }
-      it "should run at the schedule" do
-        instance = klass.new(opts)
-        instance.register
+
+      before do
         Timecop.travel(Time.new(2000,1,1,0,0,0,'+00:00'))
         Timecop.scale(60)
-        queue = Queue.new
-        runner = Thread.new do
-          instance.run(queue)
-        end
-        sleep 3
-        instance.stop
-        runner.kill
-        runner.join
-        expect(queue.size).to eq(2)
+      end
+
+      after do
         Timecop.return
       end
+
+      it "should run at the schedule" do
+        run_plugin_and_yield_queue(plugin, sleep: 3.1) do |queue|
+          try(10) { expect(queue.size).to be >= 2 }
+        end
+      end
     end
 
     context "given 'at' expression" do
@@ -216,22 +225,21 @@ describe LogStash::Inputs::HTTP_Poller do
           "metadata_target" => metadata_target
         }
       }
-      it "should run at the schedule" do
-        instance = klass.new(opts)
-        instance.register
+
+      before do
         Timecop.travel(Time.new(2000,1,1,0,0,0,'+00:00'))
-        Timecop.scale(60 * 5)
-        queue = Queue.new
-        runner = Thread.new do
-          instance.run(queue)
-        end
-        sleep 2
-        instance.stop
-        runner.kill
-        runner.join
-        expect(queue.size).to eq(1)
+        Timecop.scale (60 * 5) / 2
+      end
+
+      after do
         Timecop.return
       end
+
+      it "should run at the schedule" do
+        run_plugin_and_yield_queue(plugin, sleep: 4.1) do |queue|
+          try(10) { expect(queue.size).to eq(1) }
+        end
+      end
     end
 
     context "given 'every' expression" do
@@ -244,48 +252,47 @@ describe LogStash::Inputs::HTTP_Poller do
         }
       }
       it "should run at the schedule" do
-        instance = klass.new(opts)
-        instance.register
-        queue = Queue.new
-        runner = Thread.new do
-          instance.run(queue)
+        run_plugin_and_yield_queue(plugin, sleep: 5) do |queue|
+          #T       0123456
+          #events  x x x x
+          #expects 3 events at T=5
+          try(10) { expect(queue.size).to be_between(2, 3) }
         end
-        #T       0123456
-        #events  x x x x
-        #expects 3 events at T=5
-        sleep 5
-        instance.stop
-        runner.kill
-        runner.join
-        expect(queue.size).to be_between(2, 3)
       end
     end
 
     context "given 'in' expression" do
       let(:opts) {
         {
-          "schedule" => { "in" => "2s"},
+          "schedule" => { "in" => "1s"},
           "urls" => default_urls,
           "codec" => "json",
           "metadata_target" => metadata_target
         }
       }
       it "should run at the schedule" do
-        instance = klass.new(opts)
-        instance.register
-        queue = Queue.new
-        runner = Thread.new do
-          instance.run(queue)
+        run_plugin_and_yield_queue(plugin, sleep: 2.05) do |queue|
+          try(10) { expect(queue.size).to eq(1) }
         end
-        sleep 3
-        instance.stop
-        runner.kill
-        runner.join
-        expect(queue.size).to eq(1)
       end
     end
   end
 
+  def run_plugin_and_yield_queue(plugin, sleep: nil)
+    plugin.register
+    queue = Queue.new
+    begin
+      runner = Thread.new do
+        plugin.run(queue)
+      end
+      sleep(sleep) if sleep
+      yield(queue)
+    ensure
+      plugin.stop
+      runner.join if runner
+    end
+  end
+
   describe "events", :ecs_compatibility_support, :aggregate_failures do
     ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
       before do
@@ -339,6 +346,8 @@ describe LogStash::Inputs::HTTP_Poller do
           event # materialize the subject
         end
 
+        after { poller.stop }
+
         it "should enqueue a message" do
           expect(event).to be_a(LogStash::Event)
         end
@@ -399,9 +408,6 @@ describe LogStash::Inputs::HTTP_Poller do
         let(:payload) { {"a" => 2, "hello" => ["a", "b", "c"]} }
         let(:response_body) { LogStash::Json.dump(payload) }
         let(:opts) { default_opts }
-        let(:instance) {
-          klass.new(opts)
-        }
         let(:name) { default_name }
         let(:url) { default_url }
         let(:code) { 202 }
@@ -411,14 +417,15 @@ describe LogStash::Inputs::HTTP_Poller do
         }
 
         before do
-          instance.register
+          plugin.register
           u = url.is_a?(Hash) ? url["url"] : url # handle both complex specs and simple string URLs
-          instance.client.stub(u,
-                               :body => response_body,
-                               :code => code
-          )
-          allow(instance).to receive(:decorate)
-          instance.send(:run_once, queue)
+          plugin.client.stub(u, :body => response_body, :code => code)
+          allow(plugin).to receive(:decorate)
+          plugin.send(:run_once, queue)
+        end
+
+        after do
+          plugin.close
         end
 
         it "should have a matching message" do
@@ -426,7 +433,7 @@ describe LogStash::Inputs::HTTP_Poller do
         end
 
         it "should decorate the event" do
-          expect(instance).to have_received(:decorate).once
+          expect(plugin).to have_received(:decorate).once
         end
 
         include_examples("matching metadata")
@@ -434,7 +441,7 @@ describe LogStash::Inputs::HTTP_Poller do
         context "with an empty body" do
           let(:response_body) { "" }
           it "should return an empty event" do
-            instance.send(:run_once, queue)
+            plugin.send(:run_once, queue)
             headers_field = ecs_select[disabled: "[#{metadata_target}][response_headers]",
                                       v1: "[#{metadata_target}][input][http_poller][response][headers]"]
             expect(event.get("#{headers_field}[content-length]")).to eql("0")
@@ -449,7 +456,7 @@ describe LogStash::Inputs::HTTP_Poller do
           }
 
           it "should not have any metadata on the event" do
-            instance.send(:run_once, queue)
+            plugin.send(:run_once, queue)
             expect(event.get(metadata_target)).to be_nil
           end
         end
@@ -555,6 +562,8 @@ describe LogStash::Inputs::HTTP_Poller do
 
   describe "stopping" do
     let(:config) { default_opts }
-    it_behaves_like "an interruptible input plugin"
+    it_behaves_like "an interruptible input plugin" do
+      let(:allowed_lag) { 10 } # CI: wait till scheduler shuts down
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-http_poller
 version: !ruby/object:Gem::Version
-  version: 5.2.0
+  version: 5.3.1
 platform: ruby
 authors:
 - Elastic
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-01 00:00:00.000000000 Z
+date: 2022-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.1.0
+        version: 7.2.0
   name: logstash-mixin-http_client
   prerelease: false
   type: :runtime
@@ -58,25 +58,11 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.1.0
+        version: 7.2.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.0.22
-  name: stud
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.0.22
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.0.9
   name: rufus-scheduler
@@ -84,7 +70,7 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.0.9
 - !ruby/object:Gem::Dependency