logstash-input-http_poller 5.0.1 → 5.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25cfbb3da58886a59098ad1df73d19209dbc497a31566d6fe800ed7d6b14b83d
-  data.tar.gz: 9be5d6ef406f2912c74d30efc800793c01ef0fa67301e786298b54bbafcb30f4
+  metadata.gz: ffa63b5111349fab5d81dbc52cc005ab2ab6ed296aab3135859d941b3fd32c4b
+  data.tar.gz: 6ab7912818139b2c9595fe7b0797f0c4e02bb9af2722a32e5af1c9ca8c887536
 SHA512:
-  metadata.gz: d1b6f1165a6d15f7ec692820c769e29ad2be9924252ae4274a348db43ce82ac4046199ebbc8cae84058e8fc21806dcc4874e020cbee11c5eaf11806c60bfe24b
-  data.tar.gz: 76bf3600e28dea4e6a8418a7285fb5f47e5301629bb4648c53fc897db7b5288f1da900707f36feccd81a8f6d951bc64140129e1cd70a623e5789ad7334918db0
+  metadata.gz: 2093f0da0658436437293283a8425e2e497d2bb1e17a9b3910dff3e2ea5bf8a1ce546225abf4f97f24866abd812d0078e0c479ee7c3ef401ee3c3ec611002c21
+  data.tar.gz: 24210d613caee8ed465816b9865d75ad0547d8e81f5160858742d77bbe4ddef46284b798b4c1581bc706f0267f4316718318995dc5dbb3647c123d407d7fdf08

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 5.2.1
+  - Deps: unpin rufus-scheduler dependency [#130](https://github.com/logstash-plugins/logstash-input-http_poller/pull/130)
+
+## 5.2.0
+  - Feat: support ssl_verification_mode option [#131](https://github.com/logstash-plugins/logstash-input-http_poller/pull/131)
+
+## 5.1.0
+  - Add ECS support [#129](https://github.com/logstash-plugins/logstash-input-http_poller/pull/129)
+
+## 5.0.2
+ - [DOC]Expanded url option to include Manticore keys [#119](https://github.com/logstash-plugins/logstash-input-http_poller/pull/119)
+
 ## 5.0.1
  - Fixed minor doc and doc formatting issues [#107](https://github.com/logstash-plugins/logstash-input-http_poller/pull/107)
 
@@ -18,7 +30,7 @@
   - Docs: Remove row in overview table to fix build error
   
 ## 4.0.2
-  - Don't bleed URLs credentials on startup and on exception #82
+  - Don't bleed URLs credentials on startup and on exception [#82](https://github.com/logstash-plugins/logstash-input-http_poller/pull/82)
 
 ## 4.0.1
   - Fix some documentation issues

data/Gemfile

@@ -9,3 +9,5 @@ if Dir.exist?(logstash_path) && use_logstash_source
   gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
 end
+
+gem 'rufus-scheduler', ENV['RUFUS_SCHEDULER_VERSION'] if ENV['RUFUS_SCHEDULER_VERSION']

data/LICENSE

@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -178,7 +179,7 @@
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
+      boilerplate notice, with the fields enclosed by brackets "[]"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
@@ -186,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright 2020 Elastic and contributors
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -199,4 +200,3 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
-

data/README.md

@@ -1,6 +1,6 @@
 # Logstash HTTP input plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-http_poller.svg)](https://travis-ci.org/logstash-plugins/logstash-input-http_poller)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-http_poller.svg)](https://travis-ci.com/logstash-plugins/logstash-input-http_poller)
 
 This plugin is based off [logstash-input-rest](https://github.com/maximede/logstash-input-rest) by @maximede.
 

data/docs/index.asciidoc

@@ -87,6 +87,36 @@ The above snippet will create two files `downloaded_cert.pem` and `downloaded_tr
 ----------------------------------
 
 
+[id="plugins-{type}s-{plugin}-ecs_metadata"]
+==== Event Metadata and the Elastic Common Schema (ECS)
+
+This input will add metadata about the HTTP connection itself to each event.
+
+When ECS compatibility is disabled, metadata was added to a variety of non-standard top-level fields, which has the potential to create confusion and schema conflicts downstream.
+
+With ECS Compatibility Mode, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace.
+
+Here’s how ECS compatibility mode affects output.
+[cols="<l,<l,e,<e"]
+|=======================================================================
+| ECS disabled | ECS v1 | Availability | Description
+
+| [@metadata][host] | [@metadata][input][http_poller][request][host][hostname] | Always | Hostname
+| [@metadata][code] | [@metadata][input][http_poller][response][status_code] | When server responds a valid status code | HTTP response code
+| [@metadata][response_headers] | [@metadata][input][http_poller][response][headers] | When server responds with headers | HTTP headers of the response
+| [@metadata][response_message] | [@metadata][input][http_poller][response][status_message] | When server responds with status line | message of status line of HTTP headers
+| [@metadata][runtime_seconds] | [@metadata][input][http_poller][response][elapsed_time_ns] | When server responds a valid status code | elapsed time of calling endpoint. ECS v1 shows in nanoseconds.
+| [http_request_failure][runtime_seconds] | [event][duration] | When server throws exception | elapsed time of calling endpoint. ECS v1 shows in nanoseconds.
+| [@metadata][times_retried] | [@metadata][input][http_poller][request][retry_count] | When the poller calls server successfully | retry count from http client library
+| [@metadata][name] / [http_request_failure][name] | [@metadata][input][http_poller][request][name] | Always | The key of `urls` from poller config
+| [@metadata][request] / [http_request_failure][request]| [@metadata][input][http_poller][request][original] | Always | The whole object of `urls` from poller config
+| [http_request_failure][error] | [error][message] | When server throws exception | Error message
+| [http_request_failure][backtrace] | [error][stack_trace] | When server throws exception | Stack trace of error
+| -- | [url][full] | When server throws exception | The URL of the endpoint
+| -- | [http][request][method] | When server throws exception | HTTP request method
+| -- | [host][hostname] | When server throws exception | Hostname
+|=======================================================================
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Http_poller Input Configuration Options
 
@@ -95,20 +125,20 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
-| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|no
-| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-automatic_retries>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-connect_timeout>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-cookies>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-follow_redirects>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-keepalive>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-keystore_type>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-metadata_target>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-pool_max>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-pool_max_per_route>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-proxy>> |<<,>>|No
@@ -116,11 +146,13 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-retry_non_idempotent>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-schedule>> |<<hash,hash>>|Yes
 | <<plugins-{type}s-{plugin}-socket_timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-truststore_type>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-urls>> |<<hash,hash>>|Yes
+| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-validate_after_inactivity>> |<<number,number>>|No
 |=======================================================================
 
@@ -129,23 +161,6 @@ input plugins.
 
 &nbsp;
 
-[id="plugins-{type}s-{plugin}-user"]
-===== `user` 
-
-  * Value type is <<string,string>>
-  * There is no default value for this setting.
-
-Username to use with HTTP authentication for ALL requests. Note that you can also set this per-URL.
-If you set this you must also set the `password` option.
-
-[id="plugins-{type}s-{plugin}-password"]
-===== `password` 
-
-  * Value type is <<password,password>>
-  * There is no default value for this setting.
-
-Password to be used in conjunction with the username for HTTP authentication.
-
 [id="plugins-{type}s-{plugin}-automatic_retries"]
 ===== `automatic_retries` 
 
@@ -197,6 +212,81 @@ Timeout (in seconds) to wait for a connection to be established. Default is `10s
 Enable cookie support. With this enabled the client will persist cookies
 across requests as a normal web browser would. Enabled by default
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+* Value type is <<string,string>>
+* Supported values are:
+** `disabled`: unstructured data added at root level
+** `v1`: uses `error`, `url` and `http` fields that are compatible with Elastic Common Schema
+
+Controls this plugin's compatibility with the
+{ecs-ref}[Elastic Common Schema (ECS)].
+See <<plugins-{type}s-{plugin}-ecs_metadata>> for detailed information.
+
+Example output:
+
+**Sample output: ECS disabled**
+[source,text]
+-----
+{
+    "http_poller_data" => {
+        "@version" => "1",
+        "@timestamp" => 2021-01-01T00:43:22.388Z,
+        "status" => "UP"
+    },
+    "@version" => "1",
+    "@timestamp" => 2021-01-01T00:43:22.389Z,
+}
+-----
+
+**Sample output: ECS enabled**
+[source,text]
+-----
+{
+    "http_poller_data" => {
+        "status" => "UP",
+        "@version" => "1",
+        "event" => {
+            "original" => "{\"status\":\"UP\"}"
+        },
+        "@timestamp" => 2021-01-01T00:40:59.558Z
+    },
+    "@version" => "1",
+    "@timestamp" => 2021-01-01T00:40:59.559Z
+}
+-----
+
+**Sample error output: ECS enabled**
+[source,text]
+----
+{
+    "@timestamp" => 2021-07-09T09:53:48.721Z,
+    "@version" => "1",
+    "host" => {
+        "hostname" => "MacBook-Pro"
+    },
+    "http" => {
+        "request" => {
+            "method" => "get"
+        }
+    },
+    "event" => {
+        "duration" => 259019
+    },
+    "error" => {
+        "stack_trace" => nil,
+        "message" => "Connection refused (Connection refused)"
+    },
+    "url" => {
+        "full" => "http://localhost:8080/actuator/health"
+    },
+    "tags" => [
+        [0] "_http_request_failure"
+    ]
+}
+----
+
 [id="plugins-{type}s-{plugin}-follow_redirects"]
 ===== `follow_redirects` 
 
@@ -249,6 +339,14 @@ If you'd like to work with the request/response metadata.
 Set this value to the name of the field you'd like to store a nested
 hash of metadata.
 
+[id="plugins-{type}s-{plugin}-password"]
+===== `password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Password to be used in conjunction with <<plugins-{type}s-{plugin}-user>> for HTTP authentication.
+
 [id="plugins-{type}s-{plugin}-pool_max"]
 ===== `pool_max` 
 
@@ -316,6 +414,22 @@ See: rufus/scheduler for details about different schedule options and value stri
 
 Timeout (in seconds) to wait for data on the socket. Default is `10s`
 
+[id="plugins-{type}s-{plugin}-ssl_verification_mode"]
+===== `ssl_verification_mode`
+
+  * Value type is <<string,string>>
+  * Supported values are: `full`, `none`
+  * Default value is `full`
+
+Controls the verification of server certificates.
+The `full` option verifies that the provided certificate is signed by a trusted authority (CA)
+and also that the server’s hostname (or IP address) matches the names identified within the certificate.
+
+The `none` setting performs no verification of the server’s certificate.
+This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration.
+It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors.
+Using `none`  in production environments is strongly discouraged.
+
 [id="plugins-{type}s-{plugin}-target"]
 ===== `target` 
 
@@ -324,6 +438,10 @@ Timeout (in seconds) to wait for data on the socket. Default is `10s`
 
 Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event.
 
+TIP: When ECS is enabled, set `target` in the codec (if the codec has a `target` option).
+Example: `codec => json { target => "TARGET_FIELD_NAME" }`
+
+
 [id="plugins-{type}s-{plugin}-truststore"]
 ===== `truststore` 
 
@@ -357,7 +475,45 @@ Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS`
   * There is no default value for this setting.
 
 A Hash of urls in this format : `"name" => "url"`.
-The name and the url will be passed in the outputed event
+The name and the url will be passed in the outputted event.
+
+The values in urls can be either:
+
+* a string url (which will be issued as an HTTP GET).
+* a sub-hash containing many useful keys provided by the Manticore backend:
+** url: the String url
+** method: (optional) the HTTP method to use (defaults to GET)
+** user: (optional) the HTTP Basic Auth user. The user must be under 
+   an auth sub-hash for Manticore, but this plugin also accepts it either way.
+** password: (optional) the HTTP Basic Auth password. The password 
+   must be under an auth sub-hash for Manticore, but this plugin accepts it either way.
+** headers: a hash containing key-value pairs of headers.
+** body: a string (supported only on POST and PUT requests)
+** possibly other options mentioned in the 
+   https://www.rubydoc.info/github/cheald/manticore/Manticore/Client#http-instance_method[Manticore docs].
+   Note that Manticore options that are not explicitly documented above are not
+   thoroughly tested and therefore liable to break in unexpected ways if we
+   replace the backend.
+
+*Notes:*
+
+* Passwords specified as a part of `urls` are prone to exposure in plugin log output. 
+The plugin does not declare them as passwords, and therefore doesn't wrap them in
+leak-reducing wrappers as we do elsewhere.
+* We don't guarantee that boolean-type options like Manticore's `follow_redirects` are supported
+correctly. The strings `true` or `false` may get passed through, and in ruby any
+string is "truthy."
+* Our implementation of this plugin precludes the ability to specify auth[:eager]
+as anything other than true
+
+[id="plugins-{type}s-{plugin}-user"]
+===== `user` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Username to use with HTTP authentication for ALL requests. Note that you can also set this per-URL.
+If you set this you must also set the <<plugins-{type}s-{plugin}-password>> option.
 
 [id="plugins-{type}s-{plugin}-validate_after_inactivity"]
 ===== `validate_after_inactivity` 

data/lib/logstash/inputs/http_poller.rb

@@ -5,9 +5,18 @@ require "logstash/plugin_mixins/http_client"
 require "socket" # for Socket.gethostname
 require "manticore"
 require "rufus/scheduler"
+require "logstash/plugin_mixins/ecs_compatibility_support"
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
+require 'logstash/plugin_mixins/event_support/event_factory_adapter'
 
 class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   include LogStash::PluginMixins::HttpClient
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+  include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
+  include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
+
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
 
   config_name "http_poller"
 
@@ -28,7 +37,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   config :schedule, :validate => :hash, :required => true
 
   # Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event.
-  config :target, :validate => :string
+  config :target, :validate => :field_reference
 
   # If you'd like to work with the request/response metadata.
   # Set this value to the name of the field you'd like to store a nested
@@ -42,12 +51,16 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
 
     @logger.info("Registering http_poller Input", :type => @type, :schedule => @schedule, :timeout => @timeout)
 
+    setup_ecs_field!
     setup_requests!
   end
 
+  # @overload
   def stop
-    Stud.stop!(@interval_thread) if @interval_thread
-    @scheduler.stop if @scheduler
+    if @scheduler
+      @scheduler.shutdown # on newer Rufus (3.8) this joins on the scheduler thread
+    end
+    # TODO implement client.close as we as releasing it's pooled resources!
   end
 
   private
@@ -55,6 +68,35 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
     @requests = Hash[@urls.map {|name, url| [name, normalize_request(url)] }]
   end
 
+  private
+  # In the context of ECS, there are two type of events in this plugin, valid HTTP response and failure
+  # For a valid HTTP response, `url`, `request_method` and `host` are metadata of request.
+  #   The call could retrieve event which contain `[url]`, `[http][request][method]`, `[host][hostname]` data
+  #   Therefore, metadata should not write to those fields
+  # For a failure, `url`, `request_method` and `host` are primary data of the event because the plugin owns this event,
+  #   so it writes to url.*, http.*, host.*
+  def setup_ecs_field!
+    @request_host_field = ecs_select[disabled: "[#{metadata_target}][host]", v1: "[#{metadata_target}][input][http_poller][request][host][hostname]"]
+    @response_code_field = ecs_select[disabled: "[#{metadata_target}][code]", v1: "[#{metadata_target}][input][http_poller][response][status_code]"]
+    @response_headers_field = ecs_select[disabled: "[#{metadata_target}][response_headers]", v1: "[#{metadata_target}][input][http_poller][response][headers]"]
+    @response_message_field = ecs_select[disabled: "[#{metadata_target}][response_message]", v1: "[#{metadata_target}][input][http_poller][response][status_message]"]
+    @response_time_s_field = ecs_select[disabled: "[#{metadata_target}][runtime_seconds]", v1: nil]
+    @response_time_ns_field = ecs_select[disabled: nil, v1: "[#{metadata_target}][input][http_poller][response][elapsed_time_ns]"]
+    @request_retry_count_field = ecs_select[disabled: "[#{metadata_target}][times_retried]", v1: "[#{metadata_target}][input][http_poller][request][retry_count]"]
+    @request_name_field = ecs_select[disabled: "[#{metadata_target}][name]", v1: "[#{metadata_target}][input][http_poller][request][name]"]
+    @original_request_field = ecs_select[disabled: "[#{metadata_target}][request]", v1: "[#{metadata_target}][input][http_poller][request][original]"]
+
+    @error_msg_field = ecs_select[disabled: "[http_request_failure][error]", v1: "[error][message]"]
+    @stack_trace_field = ecs_select[disabled: "[http_request_failure][backtrace]", v1: "[error][stack_trace]"]
+    @fail_original_request_field = ecs_select[disabled: "[http_request_failure][request]", v1: nil]
+    @fail_request_name_field = ecs_select[disabled: "[http_request_failure][name]", v1: nil]
+    @fail_response_time_s_field = ecs_select[disabled: "[http_request_failure][runtime_seconds]", v1: nil]
+    @fail_response_time_ns_field = ecs_select[disabled: nil, v1: "[event][duration]"]
+    @fail_request_url_field = ecs_select[disabled: nil, v1: "[url][full]"]
+    @fail_request_method_field = ecs_select[disabled: nil, v1: "[http][request][method]"]
+    @fail_request_host_field = ecs_select[disabled: nil, v1: "[host][hostname]"]
+  end
+
   private
   def normalize_request(url_or_spec)
     if url_or_spec.is_a?(String)
@@ -124,16 +166,15 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
     #schedule hash must contain exactly one of the allowed keys
     msg_invalid_schedule = "Invalid config. schedule hash must contain " +
       "exactly one of the following keys - cron, at, every or in"
-    raise Logstash::ConfigurationError, msg_invalid_schedule if @schedule.keys.length !=1
+    raise Logstash::ConfigurationError, msg_invalid_schedule if @schedule.keys.length != 1
     schedule_type = @schedule.keys.first
     schedule_value = @schedule[schedule_type]
     raise LogStash::ConfigurationError, msg_invalid_schedule unless Schedule_types.include?(schedule_type)
 
     @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
-    #as of v3.0.9, :first_in => :now doesn't work. Use the following workaround instead
     opts = schedule_type == "every" ? { :first_in => 0.01 } : {} 
     @scheduler.send(schedule_type, schedule_value, opts) { run_once(queue) }
-    @scheduler.join
+    @scheduler.thread.join # due newer rufus (3.8) doing a blocking operation on scheduler.join
   end
 
   def run_once(queue)
@@ -151,10 +192,14 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
 
     method, *request_opts = request
     client.async.send(method, *request_opts).
-      on_success {|response| handle_success(queue, name, request, response, Time.now - started)}.
-      on_failure {|exception|
-      handle_failure(queue, name, request, exception, Time.now - started)
-    }
+      on_success {|response| handle_success(queue, name, request, response, Time.now - started) }.
+      on_failure {|exception| handle_failure(queue, name, request, exception, Time.now - started) }
+  end
+
+  private
+  # time diff in float to nanoseconds
+  def to_nanoseconds(time_diff)
+    (time_diff * 1000000).to_i
   end
 
   private
@@ -164,11 +209,11 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
     # responses come up as "" which will cause the codec to not yield anything
     if body && body.size > 0
       decode_and_flush(@codec, body) do |decoded|
-        event = @target ? LogStash::Event.new(@target => decoded.to_hash) : decoded
+        event = @target ? targeted_event_factory.new_event(decoded.to_hash) : decoded
         handle_decoded_event(queue, name, request, response, event, execution_time)
       end
     else
-      event = ::LogStash::Event.new
+      event = event_factory.new_event
       handle_decoded_event(queue, name, request, response, event, execution_time)
     end
   end
@@ -197,20 +242,10 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   private
   # Beware, on old versions of manticore some uncommon failures are not handled
   def handle_failure(queue, name, request, exception, execution_time)
-    event = LogStash::Event.new
-    apply_metadata(event, name, request)
-
+    event = event_factory.new_event
     event.tag("_http_request_failure")
-
-    # This is also in the metadata, but we send it anyone because we want this
-    # persisted by default, whereas metadata isn't. People don't like mysterious errors
-    event.set("http_request_failure", {
-      "request" => structure_request(request),
-      "name" => name,
-      "error" => exception.to_s,
-      "backtrace" => exception.backtrace,
-      "runtime_seconds" => execution_time
-   })
+    apply_metadata(event, name, request, nil, execution_time)
+    apply_failure_fields(event, name, request, exception, execution_time)
 
     queue << event
   rescue StandardError, java.lang.Exception => e
@@ -231,29 +266,39 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   end
 
   private
-  def apply_metadata(event, name, request, response=nil, execution_time=nil)
+  def apply_metadata(event, name, request, response, execution_time)
     return unless @metadata_target
-    event.set(@metadata_target, event_metadata(name, request, response, execution_time))
-  end
-
-  private
-  def event_metadata(name, request, response=nil, execution_time=nil)
-    m = {
-        "name" => name,
-        "host" => @host,
-        "request" => structure_request(request),
-      }
 
-    m["runtime_seconds"] = execution_time
+    event.set(@request_host_field, @host)
+    event.set(@response_time_s_field, execution_time) if @response_time_s_field
+    event.set(@response_time_ns_field, to_nanoseconds(execution_time)) if @response_time_ns_field
+    event.set(@request_name_field, name)
+    event.set(@original_request_field, structure_request(request))
 
     if response
-      m["code"] = response.code
-      m["response_headers"] = response.headers
-      m["response_message"] = response.message
-      m["times_retried"] = response.times_retried
+      event.set(@response_code_field, response.code)
+      event.set(@response_headers_field, response.headers)
+      event.set(@response_message_field, response.message)
+      event.set(@request_retry_count_field, response.times_retried)
     end
+  end
 
-    m
+  private
+  def apply_failure_fields(event, name, request, exception, execution_time)
+    # This is also in the metadata, but we send it anyone because we want this
+    # persisted by default, whereas metadata isn't. People don't like mysterious errors
+    event.set(@fail_original_request_field, structure_request(request)) if @fail_original_request_field
+    event.set(@fail_request_name_field, name) if @fail_request_name_field
+
+    method, url, _ = request
+    event.set(@fail_request_url_field, url) if @fail_request_url_field
+    event.set(@fail_request_method_field, method.to_s) if @fail_request_method_field
+    event.set(@fail_request_host_field, @host) if @fail_request_host_field
+
+    event.set(@fail_response_time_s_field, execution_time) if @fail_response_time_s_field
+    event.set(@fail_response_time_ns_field, to_nanoseconds(execution_time)) if @fail_response_time_ns_field
+    event.set(@error_msg_field, exception.to_s)
+    event.set(@stack_trace_field, exception.backtrace)
   end
 
   private

data/logstash-input-http_poller.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'logstash-input-http_poller'
-  s.version         = '5.0.1'
+  s.version     = '5.2.1'
   s.licenses    = ['Apache License (2.0)']
   s.summary     = "Decodes the output of an HTTP API into events"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,9 +20,11 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency 'logstash-mixin-http_client', "~> 7"
-  s.add_runtime_dependency 'stud', "~> 0.0.22"
-  s.add_runtime_dependency 'rufus-scheduler', "~>3.0.9"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.1.0"
+  s.add_runtime_dependency 'rufus-scheduler', ">= 3.0.9"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
+  s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0', '>= 1.0.1'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
 
   s.add_development_dependency 'logstash-codec-json'
   s.add_development_dependency 'logstash-codec-line'