logstash-input-http_poller 5.0.2 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50ae088dbf54c3b101fdbb36f6743d9525c4664ef9dc592ba3d351580a8712a8
-  data.tar.gz: 0ea8ab2cfae1a16dd3faadf4a1af9ccac2b9e235d1ac0a83353ed21e95c70b78
+  metadata.gz: d7223eb133a657b108ccadc547d953ea940d9c7f814b32e69048bee665ff89fa
+  data.tar.gz: ea919bb6bd64fd83cedf9421e2457e60717357ae2ad41798feb2edb227403f40
 SHA512:
-  metadata.gz: 47457f51157fb74494d11b03567af00c7855f1824b020740c836977fd99f3b91a06bbde2724148205971c4ff02dc539580537e90088dd36a14e5daf09311d94d
-  data.tar.gz: 982dd17052f30f21f8a4d6d1e852260f0955d11f0d42eae0dcc003a3478522fb4cb4a8310e57db821238127aefbe728cd758154e689045dc7dbc221ca29f617b
+  metadata.gz: 205e05fa3dc29773537dbfa9f58eb9efe489b518b3e210c1668434221b69b3f7c339f57c70951ee83f8ddffa55eaf22e3a269479e079e453080f4089301e9071
+  data.tar.gz: a1b9acc31715ea764b3f50a8388da6578d1d3e5380503d154c9d5e8afb347992b8ce7c8dd5fc60c4783c3dfe81b7e722691e70c5631fc9823c8ea979f55d87a9

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 5.1.0
+  - Add ECS support [#129](https://github.com/logstash-plugins/logstash-input-http_poller/pull/129)
+
 ## 5.0.2
  - [DOC]Expanded url option to include Manticore keys [#119](https://github.com/logstash-plugins/logstash-input-http_poller/pull/119)
 

data/README.md

@@ -1,6 +1,6 @@
 # Logstash HTTP input plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-http_poller.svg)](https://travis-ci.org/logstash-plugins/logstash-input-http_poller)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-http_poller.svg)](https://travis-ci.com/logstash-plugins/logstash-input-http_poller)
 
 This plugin is based off [logstash-input-rest](https://github.com/maximede/logstash-input-rest) by @maximede.
 

data/docs/index.asciidoc

@@ -87,6 +87,36 @@ The above snippet will create two files `downloaded_cert.pem` and `downloaded_tr
 ----------------------------------
 
 
+[id="plugins-{type}s-{plugin}-ecs_metadata"]
+==== Event Metadata and the Elastic Common Schema (ECS)
+
+This input will add metadata about the HTTP connection itself to each event.
+
+When ECS compatibility is disabled, metadata was added to a variety of non-standard top-level fields, which has the potential to create confusion and schema conflicts downstream.
+
+With ECS Compatibility Mode, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace.
+
+Here’s how ECS compatibility mode affects output.
+[cols="<l,<l,e,<e"]
+|=======================================================================
+| ECS disabled | ECS v1 | Availability | Description
+
+| [@metadata][host] | [@metadata][input][http_poller][request][host][hostname] | Always | Hostname
+| [@metadata][code] | [@metadata][input][http_poller][response][status_code] | When server responds a valid status code | HTTP response code
+| [@metadata][response_headers] | [@metadata][input][http_poller][response][headers] | When server responds with headers | HTTP headers of the response
+| [@metadata][response_message] | [@metadata][input][http_poller][response][status_message] | When server responds with status line | message of status line of HTTP headers
+| [@metadata][runtime_seconds] | [@metadata][input][http_poller][response][elapsed_time_ns] | When server responds a valid status code | elapsed time of calling endpoint. ECS v1 shows in nanoseconds.
+| [http_request_failure][runtime_seconds] | [event][duration] | When server throws exception | elapsed time of calling endpoint. ECS v1 shows in nanoseconds.
+| [@metadata][times_retried] | [@metadata][input][http_poller][request][retry_count] | When the poller calls server successfully | retry count from http client library
+| [@metadata][name] / [http_request_failure][name] | [@metadata][input][http_poller][request][name] | Always | The key of `urls` from poller config
+| [@metadata][request] / [http_request_failure][request]| [@metadata][input][http_poller][request][original] | Always | The whole object of `urls` from poller config
+| [http_request_failure][error] | [error][message] | When server throws exception | Error message
+| [http_request_failure][backtrace] | [error][stack_trace] | When server throws exception | Stack trace of error
+| -- | [url][full] | When server throws exception | The URL of the endpoint
+| -- | [http][request][method] | When server throws exception | HTTP request method
+| -- | [host][hostname] | When server throws exception | Hostname
+|=======================================================================
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Http_poller Input Configuration Options
 
@@ -101,6 +131,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-connect_timeout>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-cookies>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-follow_redirects>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-keepalive>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No
@@ -180,6 +211,81 @@ Timeout (in seconds) to wait for a connection to be established. Default is `10s
 Enable cookie support. With this enabled the client will persist cookies
 across requests as a normal web browser would. Enabled by default
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+* Value type is <<string,string>>
+* Supported values are:
+** `disabled`: unstructured data added at root level
+** `v1`: uses `error`, `url` and `http` fields that are compatible with Elastic Common Schema
+
+Controls this plugin's compatibility with the
+{ecs-ref}[Elastic Common Schema (ECS)].
+See <<plugins-{type}s-{plugin}-ecs_metadata>> for detailed information.
+
+Example output:
+
+**Sample output: ECS disabled**
+[source,text]
+-----
+{
+    "http_poller_data" => {
+        "@version" => "1",
+        "@timestamp" => 2021-01-01T00:43:22.388Z,
+        "status" => "UP"
+    },
+    "@version" => "1",
+    "@timestamp" => 2021-01-01T00:43:22.389Z,
+}
+-----
+
+**Sample output: ECS enabled**
+[source,text]
+-----
+{
+    "http_poller_data" => {
+        "status" => "UP",
+        "@version" => "1",
+        "event" => {
+            "original" => "{\"status\":\"UP\"}"
+        },
+        "@timestamp" => 2021-01-01T00:40:59.558Z
+    },
+    "@version" => "1",
+    "@timestamp" => 2021-01-01T00:40:59.559Z
+}
+-----
+
+**Sample error output: ECS enabled**
+[source,text]
+----
+{
+    "@timestamp" => 2021-07-09T09:53:48.721Z,
+    "@version" => "1",
+    "host" => {
+        "hostname" => "MacBook-Pro"
+    },
+    "http" => {
+        "request" => {
+            "method" => "get"
+        }
+    },
+    "event" => {
+        "duration" => 259019
+    },
+    "error" => {
+        "stack_trace" => nil,
+        "message" => "Connection refused (Connection refused)"
+    },
+    "url" => {
+        "full" => "http://localhost:8080/actuator/health"
+    },
+    "tags" => [
+        [0] "_http_request_failure"
+    ]
+}
+----
+
 [id="plugins-{type}s-{plugin}-follow_redirects"]
 ===== `follow_redirects` 
 
@@ -315,6 +421,10 @@ Timeout (in seconds) to wait for data on the socket. Default is `10s`
 
 Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event.
 
+TIP: When ECS is enabled, set `target` in the codec (if the codec has a `target` option).
+Example: `codec => json { target => "TARGET_FIELD_NAME" }`
+
+
 [id="plugins-{type}s-{plugin}-truststore"]
 ===== `truststore` 
 

data/lib/logstash/inputs/http_poller.rb

@@ -5,9 +5,18 @@ require "logstash/plugin_mixins/http_client"
 require "socket" # for Socket.gethostname
 require "manticore"
 require "rufus/scheduler"
+require "logstash/plugin_mixins/ecs_compatibility_support"
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
+require 'logstash/plugin_mixins/event_support/event_factory_adapter'
 
 class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   include LogStash::PluginMixins::HttpClient
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+  include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
+  include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
+
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
 
   config_name "http_poller"
 
@@ -28,7 +37,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   config :schedule, :validate => :hash, :required => true
 
   # Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event.
-  config :target, :validate => :string
+  config :target, :validate => :field_reference
 
   # If you'd like to work with the request/response metadata.
   # Set this value to the name of the field you'd like to store a nested
@@ -42,6 +51,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
 
     @logger.info("Registering http_poller Input", :type => @type, :schedule => @schedule, :timeout => @timeout)
 
+    setup_ecs_field!
     setup_requests!
   end
 
@@ -55,6 +65,35 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
     @requests = Hash[@urls.map {|name, url| [name, normalize_request(url)] }]
   end
 
+  private
+  # In the context of ECS, there are two type of events in this plugin, valid HTTP response and failure
+  # For a valid HTTP response, `url`, `request_method` and `host` are metadata of request.
+  #   The call could retrieve event which contain `[url]`, `[http][request][method]`, `[host][hostname]` data
+  #   Therefore, metadata should not write to those fields
+  # For a failure, `url`, `request_method` and `host` are primary data of the event because the plugin owns this event,
+  #   so it writes to url.*, http.*, host.*
+  def setup_ecs_field!
+    @request_host_field = ecs_select[disabled: "[#{metadata_target}][host]", v1: "[#{metadata_target}][input][http_poller][request][host][hostname]"]
+    @response_code_field = ecs_select[disabled: "[#{metadata_target}][code]", v1: "[#{metadata_target}][input][http_poller][response][status_code]"]
+    @response_headers_field = ecs_select[disabled: "[#{metadata_target}][response_headers]", v1: "[#{metadata_target}][input][http_poller][response][headers]"]
+    @response_message_field = ecs_select[disabled: "[#{metadata_target}][response_message]", v1: "[#{metadata_target}][input][http_poller][response][status_message]"]
+    @response_time_s_field = ecs_select[disabled: "[#{metadata_target}][runtime_seconds]", v1: nil]
+    @response_time_ns_field = ecs_select[disabled: nil, v1: "[#{metadata_target}][input][http_poller][response][elapsed_time_ns]"]
+    @request_retry_count_field = ecs_select[disabled: "[#{metadata_target}][times_retried]", v1: "[#{metadata_target}][input][http_poller][request][retry_count]"]
+    @request_name_field = ecs_select[disabled: "[#{metadata_target}][name]", v1: "[#{metadata_target}][input][http_poller][request][name]"]
+    @original_request_field = ecs_select[disabled: "[#{metadata_target}][request]", v1: "[#{metadata_target}][input][http_poller][request][original]"]
+
+    @error_msg_field = ecs_select[disabled: "[http_request_failure][error]", v1: "[error][message]"]
+    @stack_trace_field = ecs_select[disabled: "[http_request_failure][backtrace]", v1: "[error][stack_trace]"]
+    @fail_original_request_field = ecs_select[disabled: "[http_request_failure][request]", v1: nil]
+    @fail_request_name_field = ecs_select[disabled: "[http_request_failure][name]", v1: nil]
+    @fail_response_time_s_field = ecs_select[disabled: "[http_request_failure][runtime_seconds]", v1: nil]
+    @fail_response_time_ns_field = ecs_select[disabled: nil, v1: "[event][duration]"]
+    @fail_request_url_field = ecs_select[disabled: nil, v1: "[url][full]"]
+    @fail_request_method_field = ecs_select[disabled: nil, v1: "[http][request][method]"]
+    @fail_request_host_field = ecs_select[disabled: nil, v1: "[host][hostname]"]
+  end
+
   private
   def normalize_request(url_or_spec)
     if url_or_spec.is_a?(String)
@@ -151,10 +190,14 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
 
     method, *request_opts = request
     client.async.send(method, *request_opts).
-      on_success {|response| handle_success(queue, name, request, response, Time.now - started)}.
-      on_failure {|exception|
-      handle_failure(queue, name, request, exception, Time.now - started)
-    }
+      on_success {|response| handle_success(queue, name, request, response, Time.now - started) }.
+      on_failure {|exception| handle_failure(queue, name, request, exception, Time.now - started) }
+  end
+
+  private
+  # time diff in float to nanoseconds
+  def to_nanoseconds(time_diff)
+    (time_diff * 1000000).to_i
   end
 
   private
@@ -164,11 +207,11 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
     # responses come up as "" which will cause the codec to not yield anything
     if body && body.size > 0
       decode_and_flush(@codec, body) do |decoded|
-        event = @target ? LogStash::Event.new(@target => decoded.to_hash) : decoded
+        event = @target ? targeted_event_factory.new_event(decoded.to_hash) : decoded
         handle_decoded_event(queue, name, request, response, event, execution_time)
       end
     else
-      event = ::LogStash::Event.new
+      event = event_factory.new_event
       handle_decoded_event(queue, name, request, response, event, execution_time)
     end
   end
@@ -197,20 +240,10 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   private
   # Beware, on old versions of manticore some uncommon failures are not handled
   def handle_failure(queue, name, request, exception, execution_time)
-    event = LogStash::Event.new
-    apply_metadata(event, name, request)
-
+    event = event_factory.new_event
     event.tag("_http_request_failure")
-
-    # This is also in the metadata, but we send it anyone because we want this
-    # persisted by default, whereas metadata isn't. People don't like mysterious errors
-    event.set("http_request_failure", {
-      "request" => structure_request(request),
-      "name" => name,
-      "error" => exception.to_s,
-      "backtrace" => exception.backtrace,
-      "runtime_seconds" => execution_time
-   })
+    apply_metadata(event, name, request, nil, execution_time)
+    apply_failure_fields(event, name, request, exception, execution_time)
 
     queue << event
   rescue StandardError, java.lang.Exception => e
@@ -231,29 +264,39 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   end
 
   private
-  def apply_metadata(event, name, request, response=nil, execution_time=nil)
+  def apply_metadata(event, name, request, response, execution_time)
     return unless @metadata_target
-    event.set(@metadata_target, event_metadata(name, request, response, execution_time))
-  end
-
-  private
-  def event_metadata(name, request, response=nil, execution_time=nil)
-    m = {
-        "name" => name,
-        "host" => @host,
-        "request" => structure_request(request),
-      }
 
-    m["runtime_seconds"] = execution_time
+    event.set(@request_host_field, @host)
+    event.set(@response_time_s_field, execution_time) if @response_time_s_field
+    event.set(@response_time_ns_field, to_nanoseconds(execution_time)) if @response_time_ns_field
+    event.set(@request_name_field, name)
+    event.set(@original_request_field, structure_request(request))
 
     if response
-      m["code"] = response.code
-      m["response_headers"] = response.headers
-      m["response_message"] = response.message
-      m["times_retried"] = response.times_retried
+      event.set(@response_code_field, response.code)
+      event.set(@response_headers_field, response.headers)
+      event.set(@response_message_field, response.message)
+      event.set(@request_retry_count_field, response.times_retried)
     end
+  end
 
-    m
+  private
+  def apply_failure_fields(event, name, request, exception, execution_time)
+    # This is also in the metadata, but we send it anyone because we want this
+    # persisted by default, whereas metadata isn't. People don't like mysterious errors
+    event.set(@fail_original_request_field, structure_request(request)) if @fail_original_request_field
+    event.set(@fail_request_name_field, name) if @fail_request_name_field
+
+    method, url, _ = request
+    event.set(@fail_request_url_field, url) if @fail_request_url_field
+    event.set(@fail_request_method_field, method.to_s) if @fail_request_method_field
+    event.set(@fail_request_host_field, @host) if @fail_request_host_field
+
+    event.set(@fail_response_time_s_field, execution_time) if @fail_response_time_s_field
+    event.set(@fail_response_time_ns_field, to_nanoseconds(execution_time)) if @fail_response_time_ns_field
+    event.set(@error_msg_field, exception.to_s)
+    event.set(@stack_trace_field, exception.backtrace)
   end
 
   private

data/logstash-input-http_poller.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'logstash-input-http_poller'
-  s.version         = '5.0.2'
+  s.version         = '5.1.0'
   s.licenses    = ['Apache License (2.0)']
   s.summary     = "Decodes the output of an HTTP API into events"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -23,6 +23,9 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-mixin-http_client', "~> 7"
   s.add_runtime_dependency 'stud', "~> 0.0.22"
   s.add_runtime_dependency 'rufus-scheduler', "~>3.0.9"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
+  s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0', '>= 1.0.1'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
 
   s.add_development_dependency 'logstash-codec-json'
   s.add_development_dependency 'logstash-codec-line'

data/spec/inputs/http_poller_spec.rb

@@ -5,6 +5,7 @@ require 'flores/random'
 require "timecop"
 # Workaround for the bug reported in https://github.com/jruby/jruby/issues/4637
 require 'rspec/matchers/built_in/raise_error.rb'
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 
 describe LogStash::Inputs::HTTP_Poller do
   let(:metadata_target) { "_http_poller_metadata" }
@@ -29,6 +30,7 @@ describe LogStash::Inputs::HTTP_Poller do
   }
   let(:klass) { LogStash::Inputs::HTTP_Poller }
 
+
   describe "instances" do
     subject { klass.new(default_opts) }
 
@@ -133,10 +135,10 @@ describe LogStash::Inputs::HTTP_Poller do
 
             it "should properly set the auth parameter" do
               expect(normalized[2][:auth]).to eq({
-                :user => auth["user"], 
-                :pass => auth["password"],
-                :eager => true
-              })
+                                                   :user => auth["user"],
+                                                   :pass => auth["password"],
+                                                   :eager => true
+                                                 })
             end
           end
         end
@@ -144,14 +146,14 @@ describe LogStash::Inputs::HTTP_Poller do
         # Legacy way of doing things, kept for backwards compat.
         describe "auth with nested auth hash" do
           let(:url) { {"url" => "http://localhost", "method" => "get", "auth" => auth} }
-          
+
           include_examples("auth")
         end
 
         # The new 'right' way to do things
         describe "auth with direct auth options" do
           let(:url) { {"url" => "http://localhost", "method" => "get", "user" => auth["user"], "password" => auth["password"]} }
-          
+
           include_examples("auth")
         end
       end
@@ -255,7 +257,7 @@ describe LogStash::Inputs::HTTP_Poller do
         instance.stop
         runner.kill
         runner.join
-        expect(queue.size).to eq(3)
+        expect(queue.size).to be_between(2, 3)
       end
     end
 
@@ -284,216 +286,267 @@ describe LogStash::Inputs::HTTP_Poller do
     end
   end
 
-  describe "events" do
-    shared_examples("matching metadata") {
-      let(:metadata) { event.get(metadata_target) }
-
-      it "should have the correct name" do
-        expect(metadata["name"]).to eql(name)
+  describe "events", :ecs_compatibility_support, :aggregate_failures do
+    ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
+      before do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
       end
 
-      it "should have the correct request url" do
-        if url.is_a?(Hash) # If the url was specified as a complex test the whole thing
-          expect(metadata["request"]).to eql(url)
-        else # Otherwise we have to make some assumptions
-          expect(metadata["request"]["url"]).to eql(url)
-        end
-      end
+      shared_examples("matching metadata") {
+        let(:metadata) { event.get(metadata_target) }
 
-      it "should have the correct code" do
-        expect(metadata["code"]).to eql(code)
-      end
-    }
+        it "should have the correct name" do
+          field = ecs_select[disabled: "[#{metadata_target}][name]", v1: "[#{metadata_target}][input][http_poller][request][name]"]
+          expect(event.get(field)).to eql(name)
+        end
 
-    shared_examples "unprocessable_requests" do
-      let(:poller) { LogStash::Inputs::HTTP_Poller.new(settings) }
-      subject(:event) {
-        poller.send(:run_once, queue)
-        queue.pop(true)
-      }
+        it "should have the correct request url" do
+          if url.is_a?(Hash) # If the url was specified as a complex test the whole thing
+            http_client_field = ecs_select[disabled: "[#{metadata_target}][request]",
+                                         v1: "[#{metadata_target}][input][http_poller][request][original]"]
+            expect(event.get(http_client_field)).to eql(url)
+          else # Otherwise we have to make some assumptions
+            url_field = ecs_select[disabled: "[#{metadata_target}][request][url]",
+                                   v1: "[#{metadata_target}][input][http_poller][request][original][url]"]
+            expect(event.get(url_field)).to eql(url)
+          end
+        end
 
-      before do
-        poller.register
-        allow(poller).to receive(:handle_failure).and_call_original
-        allow(poller).to receive(:handle_success)
-        event # materialize the subject
-      end
+        it "should have the correct code" do
+          expect(event.get(ecs_select[disabled: "[#{metadata_target}][code]",
+                                      v1: "[#{metadata_target}][input][http_poller][response][status_code]"]))
+            .to eql(code)
+        end
 
-      it "should enqueue a message" do
-        expect(event).to be_a(LogStash::Event)
-      end
+        it "should have the correct host" do
+          expect(event.get(ecs_select[disabled: "[#{metadata_target}][host]",
+                                      v1: "[#{metadata_target}][input][http_poller][request][host][hostname]"]))
+            .not_to be_nil
+        end
+      }
 
-      it "should enqueue a message with 'http_request_failure' set" do
-        expect(event.get("http_request_failure")).to be_a(Hash)
-      end
+      shared_examples "unprocessable_requests" do
+        let(:poller) { LogStash::Inputs::HTTP_Poller.new(settings) }
+        subject(:event) {
+          poller.send(:run_once, queue)
+          queue.pop(true)
+        }
 
-      it "should tag the event with '_http_request_failure'" do
-        expect(event.get("tags")).to include('_http_request_failure')
-      end
+        before do
+          poller.register
+          allow(poller).to receive(:handle_failure).and_call_original
+          allow(poller).to receive(:handle_success)
+          event # materialize the subject
+        end
 
-      it "should invoke handle failure exactly once" do
-        expect(poller).to have_received(:handle_failure).once
-      end
+        it "should enqueue a message" do
+          expect(event).to be_a(LogStash::Event)
+        end
 
-      it "should not invoke handle success at all" do
-        expect(poller).not_to have_received(:handle_success)
-      end
+        it "should enqueue a message with 'http_request_failure' set" do
+          if ecs_compatibility == :disabled
+            expect(event.get("http_request_failure")).to be_a(Hash)
+            expect(event.get("[http_request_failure][runtime_seconds]")).to be_a(Float)
+          else
+            expect(event.get("http_request_failure")).to be_nil
+            expect(event.get("error")).to be_a(Hash)
+            expect(event.get("[event][duration]")).to be_a(Integer)
+            expect(event.get("[url][full]")).to eq(url)
+            expect(event.get("[http][request][method]")).to be_a(String)
+            expect(event.get("[host][hostname]")).to be_a(String)
+          end
+        end
 
-      include_examples("matching metadata")
-    end
+        it "should tag the event with '_http_request_failure'" do
+          expect(event.get("tags")).to include('_http_request_failure')
+        end
 
-    context "with a non responsive server" do
-      context "due to a non-existant host" do # Fail with handlers
-        let(:name) { default_name }
-        let(:url) { "http://thouetnhoeu89ueoueohtueohtneuohn" }
-        let(:code) { nil } # no response expected
+        it "should invoke handle failure exactly once" do
+          expect(poller).to have_received(:handle_failure).once
+        end
 
-        let(:settings) { default_opts.merge("urls" => { name => url}) }
+        it "should not invoke handle success at all" do
+          expect(poller).not_to have_received(:handle_success)
+        end
 
-        include_examples("unprocessable_requests")
+        include_examples("matching metadata")
       end
 
-      context "due to a bogus port number" do # fail with return?
-        let(:invalid_port) { Flores::Random.integer(65536..1000000) }
+      context "with a non responsive server" do
+        context "due to a non-existant host" do # Fail with handlers
+          let(:name) { default_name }
+          let(:url) { "http://thouetnhoeu89ueoueohtueohtneuohn" }
+          let(:code) { nil } # no response expected
 
-        let(:name) { default_name }
-        let(:url) { "http://127.0.0.1:#{invalid_port}" }
-        let(:settings) { default_opts.merge("urls" => {name => url}) }
-        let(:code) { nil } # No response expected
+          let(:settings) { default_opts.merge("urls" => { name => url}) }
 
-        include_examples("unprocessable_requests")
-      end
-    end
+          include_examples("unprocessable_requests")
+        end
 
-    describe "a valid request and decoded response" do
-      let(:payload) { {"a" => 2, "hello" => ["a", "b", "c"]} }
-      let(:response_body) { LogStash::Json.dump(payload) }
-      let(:opts) { default_opts }
-      let(:instance) {
-        klass.new(opts)
-      }
-      let(:name) { default_name }
-      let(:url) { default_url }
-      let(:code) { 202 }
+        context "due to a bogus port number" do # fail with return?
+          let(:invalid_port) { Flores::Random.integer(65536..1000000) }
 
-      subject(:event) {
-        queue.pop(true)
-      }
+          let(:name) { default_name }
+          let(:url) { "http://127.0.0.1:#{invalid_port}" }
+          let(:settings) { default_opts.merge("urls" => {name => url}) }
+          let(:code) { nil } # No response expected
 
-      before do
-        instance.register
-        u = url.is_a?(Hash) ? url["url"] : url # handle both complex specs and simple string URLs
-        instance.client.stub(u,
-                             :body => response_body,
-                             :code => code
-        )
-        allow(instance).to receive(:decorate)
-        instance.send(:run_once, queue)
+          include_examples("unprocessable_requests")
+        end
       end
 
-      it "should have a matching message" do
-        expect(event.to_hash).to include(payload)
-      end
+      describe "a valid request and decoded response" do
+        let(:payload) { {"a" => 2, "hello" => ["a", "b", "c"]} }
+        let(:response_body) { LogStash::Json.dump(payload) }
+        let(:opts) { default_opts }
+        let(:instance) {
+          klass.new(opts)
+        }
+        let(:name) { default_name }
+        let(:url) { default_url }
+        let(:code) { 202 }
 
-      it "should decorate the event" do
-        expect(instance).to have_received(:decorate).once
-      end
+        subject(:event) {
+          queue.pop(true)
+        }
 
-      include_examples("matching metadata")
-      
-      context "with an empty body" do
-        let(:response_body) { "" }
-        it "should return an empty event" do
+        before do
+          instance.register
+          u = url.is_a?(Hash) ? url["url"] : url # handle both complex specs and simple string URLs
+          instance.client.stub(u,
+                               :body => response_body,
+                               :code => code
+          )
+          allow(instance).to receive(:decorate)
           instance.send(:run_once, queue)
-          expect(event.get("[_http_poller_metadata][response_headers][content-length]")).to eql("0")
         end
-      end
 
-      context "with metadata omitted" do
-        let(:opts) {
-          opts = default_opts.clone
-          opts.delete("metadata_target")
-          opts
-        }
+        it "should have a matching message" do
+          expect(event.to_hash).to include(payload)
+        end
 
-        it "should not have any metadata on the event" do
-          instance.send(:run_once, queue)
-          expect(event.get(metadata_target)).to be_nil
+        it "should decorate the event" do
+          expect(instance).to have_received(:decorate).once
         end
-      end
 
-      context "with a complex URL spec" do
-        let(:url) {
-          {
-            "method" => "get",
-            "url" => default_url,
-            "headers" => {
-              "X-Fry" => "I'm having one of those things, like a headache, with pictures..."
+        include_examples("matching metadata")
+
+        context "with an empty body" do
+          let(:response_body) { "" }
+          it "should return an empty event" do
+            instance.send(:run_once, queue)
+            headers_field = ecs_select[disabled: "[#{metadata_target}][response_headers]",
+                                      v1: "[#{metadata_target}][input][http_poller][response][headers]"]
+            expect(event.get("#{headers_field}[content-length]")).to eql("0")
+          end
+        end
+
+        context "with metadata omitted" do
+          let(:opts) {
+            opts = default_opts.clone
+            opts.delete("metadata_target")
+            opts
+          }
+
+          it "should not have any metadata on the event" do
+            instance.send(:run_once, queue)
+            expect(event.get(metadata_target)).to be_nil
+          end
+        end
+
+        context "with a complex URL spec" do
+          let(:url) {
+            {
+              "method" => "get",
+              "url" => default_url,
+              "headers" => {
+                "X-Fry" => "I'm having one of those things, like a headache, with pictures..."
+              }
             }
           }
-        }
-        let(:opts) {
-          {
-            "schedule" => {
-              "cron" => "* * * * * UTC"
+          let(:opts) {
+            {
+              "schedule" => {
+                "cron" => "* * * * * UTC"
+              },
+              "urls" => {
+                default_name => url
               },
-            "urls" => {
-              default_name => url
-            },
-            "codec" => "json",
-            "metadata_target" => metadata_target
+              "codec" => "json",
+              "metadata_target" => metadata_target
+            }
           }
-        }
 
-        include_examples("matching metadata")
+          include_examples("matching metadata")
 
-        it "should have a matching message" do
-          expect(event.to_hash).to include(payload)
+          it "should have a matching message" do
+            expect(event.to_hash).to include(payload)
+          end
         end
-      end
 
-      context "with a specified target" do
-        let(:target) { "mytarget" }
-        let(:opts) { default_opts.merge("target" => target) }
+        context "with a specified target" do
+          let(:target) { "mytarget" }
+          let(:opts) { default_opts.merge("target" => target) }
 
-        it "should store the event info in the target" do
-          # When events go through the pipeline they are java-ified
-          # this normalizes the payload to java types
-          payload_normalized = LogStash::Json.load(LogStash::Json.dump(payload))
-          expect(event.get(target)).to include(payload_normalized)
+          it "should store the event info in the target" do
+            # When events go through the pipeline they are java-ified
+            # this normalizes the payload to java types
+            payload_normalized = LogStash::Json.load(LogStash::Json.dump(payload))
+            expect(event.get(target)).to include(payload_normalized)
+          end
         end
-      end
 
-      context 'using a line codec' do
-        let(:opts) do
-          default_opts.merge({"codec" => "line"})
-        end
-        subject(:events) do
-          [].tap do |events|
-            events << queue.pop until queue.empty?
+        context "with default metadata target" do
+          let(:metadata_target) { "@metadata" }
+
+          it "should store the metadata info in @metadata" do
+            if ecs_compatibility == :disabled
+              expect(event.get("[@metadata][response_headers]")).to be_a(Hash)
+              expect(event.get("[@metadata][runtime_seconds]")).to be_a(Float)
+              expect(event.get("[@metadata][times_retried]")).to eq(0)
+              expect(event.get("[@metadata][name]")).to eq(default_name)
+              expect(event.get("[@metadata][request]")).to be_a(Hash)
+            else
+              expect(event.get("[@metadata][input][http_poller][response][headers]")).to be_a(Hash)
+              expect(event.get("[@metadata][input][http_poller][response][elapsed_time_ns]")).to be_a(Integer)
+              expect(event.get("[@metadata][input][http_poller][request][retry_count]")).to eq(0)
+              expect(event.get("[@metadata][input][http_poller][request][name]")).to eq(default_name)
+              expect(event.get("[@metadata][input][http_poller][request][original]")).to be_a(Hash)
+            end
           end
         end
 
-        context 'when response has a trailing newline' do
-          let(:response_body) { "one\ntwo\nthree\nfour\n" }
-          it 'emits all events' do
-            expect(events.size).to equal(4)
-            messages = events.map{|e| e.get('message')}
-            expect(messages).to include('one')
-            expect(messages).to include('two')
-            expect(messages).to include('three')
-            expect(messages).to include('four')
+        context 'using a line codec' do
+          let(:opts) do
+            default_opts.merge({"codec" => "line"})
           end
-        end
-        context 'when response has no trailing newline' do
-          let(:response_body) { "one\ntwo\nthree\nfour" }
-          it 'emits all events' do
-            expect(events.size).to equal(4)
-            messages = events.map{|e| e.get('message')}
-            expect(messages).to include('one')
-            expect(messages).to include('two')
-            expect(messages).to include('three')
-            expect(messages).to include('four')
+          subject(:events) do
+            [].tap do |events|
+              events << queue.pop until queue.empty?
+            end
+          end
+
+          context 'when response has a trailing newline' do
+            let(:response_body) { "one\ntwo\nthree\nfour\n" }
+            it 'emits all events' do
+              expect(events.size).to equal(4)
+              messages = events.map{|e| e.get('message')}
+              expect(messages).to include('one')
+              expect(messages).to include('two')
+              expect(messages).to include('three')
+              expect(messages).to include('four')
+            end
+          end
+          context 'when response has no trailing newline' do
+            let(:response_body) { "one\ntwo\nthree\nfour" }
+            it 'emits all events' do
+              expect(events.size).to equal(4)
+              messages = events.map{|e| e.get('message')}
+              expect(messages).to include('one')
+              expect(messages).to include('two')
+              expect(messages).to include('three')
+              expect(messages).to include('four')
+            end
           end
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-http_poller
 version: !ruby/object:Gem::Version
-  version: 5.0.2
+  version: 5.1.0
 platform: ruby
 authors:
 - Elastic
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-05 00:00:00.000000000 Z
+date: 2021-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -87,6 +87,54 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.0.9
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  name: logstash-mixin-ecs_compatibility_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  name: logstash-mixin-event_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-validator_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -195,8 +243,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Decodes the output of an HTTP API into events