logstash-input-http_poller 5.2.1 → 5.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ffa63b5111349fab5d81dbc52cc005ab2ab6ed296aab3135859d941b3fd32c4b
-  data.tar.gz: 6ab7912818139b2c9595fe7b0797f0c4e02bb9af2722a32e5af1c9ca8c887536
+  metadata.gz: 959498284e07e414f7be7e02b9df125372526da763a4d273b55fc609186b777c
+  data.tar.gz: 6a6f4dc0da77770581920f719e9de03f3d81bdc079a61fdb43874e5c09d3afa8
 SHA512:
-  metadata.gz: 2093f0da0658436437293283a8425e2e497d2bb1e17a9b3910dff3e2ea5bf8a1ce546225abf4f97f24866abd812d0078e0c479ee7c3ef401ee3c3ec611002c21
-  data.tar.gz: 24210d613caee8ed465816b9865d75ad0547d8e81f5160858742d77bbe4ddef46284b798b4c1581bc706f0267f4316718318995dc5dbb3647c123d407d7fdf08
+  metadata.gz: d3af6dc69528053dcc66ccdef55fbd9dfc56bede4cd383ccd44689a884f601b14b630f1d3accc7d56e86ca41f4bbe544379e7dc8cc23d8794db8b2c6d9fd4711
+  data.tar.gz: 693a73cc6a442b7b78d15987ea99d84a6a3b5a6143b12261a95eda1a7bfec070650771f66c2a652709470e4c5cebdc5e8f0921c6e3265c3942d168d8bb32aea2

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 5.4.0
+  - Refactor: start using scheduler mixin [#134](https://github.com/logstash-plugins/logstash-input-http_poller/pull/134)
+
+## 5.3.1
+  - Fix: make sure plugin is closing the http client [#130](https://github.com/logstash-plugins/logstash-input-http_poller/pull/130)
+
+## 5.3.0
+  - Feat: added ssl_supported_protocols option [#133](https://github.com/logstash-plugins/logstash-input-http_poller/pull/133)
+
 ## 5.2.1
   - Deps: unpin rufus-scheduler dependency [#130](https://github.com/logstash-plugins/logstash-input-http_poller/pull/130)
 

data/docs/index.asciidoc

@@ -146,6 +146,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-retry_non_idempotent>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-schedule>> |<<hash,hash>>|Yes
 | <<plugins-{type}s-{plugin}-socket_timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
@@ -414,6 +415,23 @@ See: rufus/scheduler for details about different schedule options and value stri
 
 Timeout (in seconds) to wait for data on the socket. Default is `10s`
 
+[id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
+===== `ssl_supported_protocols`
+
+  * Value type is <<string,string>>
+  * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'`
+  * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`.
+    `'TLSv1.1'` is not considered secure and is only provided for legacy applications.
+
+List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint.
+
+For Java 8 `'TLSv1.3'` is supported  only since **8u262** (AdoptOpenJDK), but requires that you set the
+`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash.
+
+NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash,
+the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in
+the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list.
+
 [id="plugins-{type}s-{plugin}-ssl_verification_mode"]
 ===== `ssl_verification_mode`
 

data/lib/logstash/inputs/http_poller.rb

@@ -4,11 +4,11 @@ require "logstash/namespace"
 require "logstash/plugin_mixins/http_client"
 require "socket" # for Socket.gethostname
 require "manticore"
-require "rufus/scheduler"
 require "logstash/plugin_mixins/ecs_compatibility_support"
 require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
 require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 require 'logstash/plugin_mixins/event_support/event_factory_adapter'
+require 'logstash/plugin_mixins/scheduler'
 
 class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   include LogStash::PluginMixins::HttpClient
@@ -18,6 +18,8 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
 
   extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
 
+  include LogStash::PluginMixins::Scheduler
+
   config_name "http_poller"
 
   default :codec, "json"
@@ -45,23 +47,34 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   config :metadata_target, :validate => :string, :default => '@metadata'
 
   public
-  Schedule_types = %w(cron every at in)
   def register
     @host = Socket.gethostname.force_encoding(Encoding::UTF_8)
 
-    @logger.info("Registering http_poller Input", :type => @type, :schedule => @schedule, :timeout => @timeout)
-
     setup_ecs_field!
     setup_requests!
   end
 
   # @overload
   def stop
-    if @scheduler
-      @scheduler.shutdown # on newer Rufus (3.8) this joins on the scheduler thread
+    close_client
+  end
+
+  # @overload
+  def close
+    close_client
+  end
+
+  def close_client
+    @logger.debug("closing http client", client: client)
+    begin
+      client.close # since Manticore 0.9.0 this shuts-down/closes all resources
+    rescue => e
+      details = { exception: e.class, message: e.message }
+      details[:backtrace] = e.backtrace if @logger.debug?
+      @logger.warn "failed closing http client", details
     end
-    # TODO implement client.close as we as releasing it's pooled resources!
   end
+  private :close_client
 
   private
   def setup_requests!
@@ -169,25 +182,27 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
     raise Logstash::ConfigurationError, msg_invalid_schedule if @schedule.keys.length != 1
     schedule_type = @schedule.keys.first
     schedule_value = @schedule[schedule_type]
-    raise LogStash::ConfigurationError, msg_invalid_schedule unless Schedule_types.include?(schedule_type)
+    raise LogStash::ConfigurationError, msg_invalid_schedule unless %w(cron every at in).include?(schedule_type)
 
-    @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
-    opts = schedule_type == "every" ? { :first_in => 0.01 } : {} 
-    @scheduler.send(schedule_type, schedule_value, opts) { run_once(queue) }
-    @scheduler.thread.join # due newer rufus (3.8) doing a blocking operation on scheduler.join
+    opts = schedule_type == "every" ? { first_in: 0.01 } : {}
+    scheduler.public_send(schedule_type, schedule_value, opts) { run_once(queue) }
+    scheduler.join
   end
 
   def run_once(queue)
     @requests.each do |name, request|
+      # prevent executing a scheduler kick after the plugin has been stop-ed
+      # this could easily happen as the scheduler shutdown is not immediate
+      return if stop?
       request_async(queue, name, request)
     end
 
-    client.execute!
+    client.execute! unless stop?
   end
 
   private
   def request_async(queue, name, request)
-    @logger.debug? && @logger.debug("Fetching URL", :name => name, :url => request)
+    @logger.debug? && @logger.debug("async queueing fetching url", name: name, url: request)
     started = Time.now
 
     method, *request_opts = request
@@ -204,6 +219,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
 
   private
   def handle_success(queue, name, request, response, execution_time)
+    @logger.debug? && @logger.debug("success fetching url", name: name, url: request)
     body = response.body
     # If there is a usable response. HEAD requests are `nil` and empty get
     # responses come up as "" which will cause the codec to not yield anything
@@ -242,6 +258,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   private
   # Beware, on old versions of manticore some uncommon failures are not handled
   def handle_failure(queue, name, request, exception, execution_time)
+    @logger.debug? && @logger.debug("failed fetching url", name: name, url: request)
     event = event_factory.new_event
     event.tag("_http_request_failure")
     apply_metadata(event, name, request, nil, execution_time)

data/logstash-input-http_poller.gemspec

@@ -1,9 +1,9 @@
 Gem::Specification.new do |s|
   s.name        = 'logstash-input-http_poller'
-  s.version     = '5.2.1'
+  s.version     = '5.4.0'
   s.licenses    = ['Apache License (2.0)']
   s.summary     = "Decodes the output of an HTTP API into events"
-  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors     = [ "Elastic", "andrewvc"]
   s.email       = 'info@elastic.co'
   s.homepage    = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -20,8 +20,8 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.1.0"
-  s.add_runtime_dependency 'rufus-scheduler', ">= 3.0.9"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.2.0"
+  s.add_runtime_dependency 'logstash-mixin-scheduler', '~> 1.0'
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
   s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0', '>= 1.0.1'
   s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'

data/spec/inputs/http_poller_spec.rb

@@ -201,8 +201,8 @@ describe LogStash::Inputs::HTTP_Poller do
       end
 
       it "should run at the schedule" do
-        run_plugin_and_yield_queue(plugin, sleep: 3) do |queue|
-          try(5) { expect(queue.size).to be >= 2 }
+        run_plugin_and_yield_queue(plugin, sleep: 3.1) do |queue|
+          try(10) { expect(queue.size).to be >= 2 }
         end
       end
     end
@@ -227,8 +227,8 @@ describe LogStash::Inputs::HTTP_Poller do
       end
 
       it "should run at the schedule" do
-        run_plugin_and_yield_queue(plugin, sleep: 2) do |queue|
-          try(5) { expect(queue.size).to eq(1) }
+        run_plugin_and_yield_queue(plugin, sleep: 4.1) do |queue|
+          try(10) { expect(queue.size).to eq(1) }
         end
       end
     end
@@ -247,7 +247,7 @@ describe LogStash::Inputs::HTTP_Poller do
           #T       0123456
           #events  x x x x
           #expects 3 events at T=5
-          try(5) { expect(queue.size).to be_between(2, 3) }
+          try(10) { expect(queue.size).to be_between(2, 3) }
         end
       end
     end
@@ -255,15 +255,15 @@ describe LogStash::Inputs::HTTP_Poller do
     context "given 'in' expression" do
       let(:opts) {
         {
-          "schedule" => { "in" => "2s"},
+          "schedule" => { "in" => "1s"},
           "urls" => default_urls,
           "codec" => "json",
           "metadata_target" => metadata_target
         }
       }
       it "should run at the schedule" do
-        run_plugin_and_yield_queue(plugin, sleep: 2.5) do |queue|
-          try(5) { expect(queue.size).to eq(1) }
+        run_plugin_and_yield_queue(plugin, sleep: 2.05) do |queue|
+          try(10) { expect(queue.size).to eq(1) }
         end
       end
     end
@@ -410,14 +410,15 @@ describe LogStash::Inputs::HTTP_Poller do
         before do
           plugin.register
           u = url.is_a?(Hash) ? url["url"] : url # handle both complex specs and simple string URLs
-          plugin.client.stub(u,
-                               :body => response_body,
-                               :code => code
-          )
+          plugin.client.stub(u, :body => response_body, :code => code)
           allow(plugin).to receive(:decorate)
           plugin.send(:run_once, queue)
         end
 
+        after do
+          plugin.close
+        end
+
         it "should have a matching message" do
           expect(event.to_hash).to include(payload)
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-http_poller
 version: !ruby/object:Gem::Version
-  version: 5.2.1
+  version: 5.4.0
 platform: ruby
 authors:
 - Elastic
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-16 00:00:00.000000000 Z
+date: 2022-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.1.0
+        version: 7.2.0
   name: logstash-mixin-http_client
   prerelease: false
   type: :runtime
@@ -58,21 +58,21 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.1.0
+        version: 7.2.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.9
-  name: rufus-scheduler
+        version: '1.0'
+  name: logstash-mixin-scheduler
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.9
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: