logstash-filter-http 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c0b4f1ef2952372087bacf06f50c64bf513cb1866b5bfffb4ed5c7fc1ceca5b
-  data.tar.gz: acadbea6ea5bfd20daa64d5788482ff84754489a54a8e49acba69732798af03e
+  metadata.gz: 676e5abcdac3f6a16a8274eb18793cd7d75fe26545f7b2bfee9f707b6cf1fb90
+  data.tar.gz: 3ed6f080ba7323a07621cc69380f693a07b51ab5b374d0172d660a8682396f90
 SHA512:
-  metadata.gz: 9702baf244c832b58d42ce44efeeacf972257e709ccb137960134f5fe0f8d427124b45e06dd584edc8d15021d6cf93f7a26afd6d50b32bb5433dedb539e875b0
-  data.tar.gz: 5c7d1bf276909ca595c3c95c5ab779a502c01072ab227112aca590cc282062865e7b5b73956b7e196711369c3b78d2bc0e9442e8e25dff2251fdff7250732b5d
+  metadata.gz: 76adf00a0498d9f2f1d43b898457d744dce0a8ecef5bc15b559682513e2fc3cf09747925dea0a01b515762cbba9f9e5106ee316ea7c44a420a13b67596c24656
+  data.tar.gz: ceeaadb9afcc5678f87f7b1d3ccf6446d56826db73354655bc5f4976d0bf07698bc061db234fd27cf569de9fd1b467dbb6a75c918a04929918235ef42a08869e

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.2.0
+  - Feat: improve ECS compatibility [#35](https://github.com/logstash-plugins/logstash-filter-http/pull/35)
+
 ## 1.1.0
   - Add support for PUT requests [#34](https://github.com/logstash-plugins/logstash-filter-http/pull/34)
 

data/lib/logstash/filters/http.rb

@@ -1,13 +1,21 @@
 # encoding: utf-8
 require 'logstash/filters/base'
+require 'logstash/json'
 require 'logstash/namespace'
 require 'logstash/plugin_mixins/http_client'
-require 'logstash/json'
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 
 # Logstash HTTP Filter
 # This filter calls a defined URL and saves the answer into a specified field.
 #
 class LogStash::Filters::Http < LogStash::Filters::Base
+
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
   include LogStash::PluginMixins::HttpClient
 
   config_name 'http'
@@ -21,14 +29,28 @@ class LogStash::Filters::Http < LogStash::Filters::Base
   config :body, :required => false
   config :body_format, :validate => ['text', 'json'], :default => "text"
 
-  config :target_body, :validate => :string, :default => "body"
-  config :target_headers, :validate => :string, :default => "headers"
+  # default [body] (legacy) required to be specified in ECS mode
+  config :target_body, :validate => :field_reference
+  # default [headers] (legacy) or [@metadata][filter][http][response][headers] in ECS mode
+  config :target_headers, :validate => :field_reference
 
   # Append values to the `tags` field when there has been no
   # successful match or json parsing error
   config :tag_on_request_failure, :validate => :array, :default => ['_httprequestfailure']
   config :tag_on_json_failure, :validate => :array, :default => ['_jsonparsefailure']
 
+  def initialize(*params)
+    super
+
+    @target_body ||= ecs_select[disabled: '[body]', v1: false]
+    if @target_body.eql? false # user needs to specify target in ECS mode
+      @logger.error missing_config_message(:target_body)
+      raise LogStash::ConfigurationError.new "Wrong configuration (in ecs_compatibility mode #{ecs_compatibility.inspect})"
+    end
+
+    @target_headers ||= ecs_select[disabled: '[headers]', v1: '[@metadata][filter][http][response][headers]']
+  end
+
   def register
     # nothing to see here
     @verb = verb.downcase
@@ -79,6 +101,20 @@ class LogStash::Filters::Http < LogStash::Filters::Base
   end # def filter
 
   private
+
+  def missing_config_message(name)
+    <<-EOF
+Missing a required setting for the http filter plugin:
+
+  filter {
+    http {
+      #{name} => # SETTING MISSING
+      ...
+    }
+  }
+EOF
+  end
+
   def request_http(verb, url, options = {})
     response = client.http(verb, url, options)
     [response.code, response.headers, response.body]

data/logstash-filter-http.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-http'
-  s.version = '1.1.0'
+  s.version = '1.2.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = 'This filter requests data from a RESTful Web Service.'
   s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-filter-http. This gem is not a stand-alone program'
@@ -27,7 +27,9 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.2'
   s.add_runtime_dependency 'logstash-mixin-http_client', '>= 5.0.0', '< 9.0.0'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/filters/http_spec.rb

@@ -1,4 +1,5 @@
 require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 require 'logstash/filters/http'
 
 describe LogStash::Filters::Http do
@@ -6,34 +7,74 @@ describe LogStash::Filters::Http do
   let(:event) { LogStash::Event.new(data) }
   let(:data) { { "message" => "test" } }
 
-  describe 'response body handling' do
-    before(:each) { subject.register }
-    let(:url) { 'http://laceholder.typicode.com/users/10' }
-    let(:config) do
-      { "url" => url, "target_body" => 'rest' }
+  let(:url) { 'https://a-non-existent.url1' }
+  let(:config) { { "url" => url, 'target_body' => '[the][body]' } }
+
+  let(:response) { [200, {}, "Bom dia"] }
+
+  describe 'target_body default', :ecs_compatibility_support do
+
+    let(:config) { { "url" => url, "ecs_compatibility" => ecs_compatibility } }
+
+    ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
+
+      it "has a default body_target (in legacy mode)" do
+        subject.register
+        allow(subject).to receive(:request_http).and_return(response)
+        subject.filter(event)
+
+        expect(event.get('body')).to eq("Bom dia")
+      end if ecs_select.active_mode == :disabled
+
+      it "fails due missing body_target (in ECS mode)" do
+        expect { subject }.to raise_error(LogStash::ConfigurationError)
+      end if ecs_select.active_mode != :disabled
+
     end
+
+  end
+
+  describe 'response body handling', :ecs_compatibility_support do
+
+    let(:url) { 'http://laceholder.typicode.com/users/10' }
+
     before(:each) do
+      subject.register
+
       allow(subject).to receive(:request_http).and_return(response)
       subject.filter(event)
     end
 
-    context "when body is text" do
-      let(:response) { [200, {}, "Bom dia"] }
+    ecs_compatibility_matrix(:disabled, :v1, :v8) do
+
+      let(:config) { super().merge "ecs_compatibility" => ecs_compatibility, 'target_body' => 'gw-response' }
+
+      context "when body is JSON" do
+        context "and headers are set correctly" do
 
-      it "fetches and writes body to target" do
-        expect(event.get('rest')).to eq("Bom dia")
+          let(:response) { [200, {"content-type" => "application/json"}, "{\"id\": 10}"] }
+
+          it "fetches and writes body to target" do
+            expect(event.get('[gw-response][id]')).to eq(10)
+          end
+
+        end
       end
-    end
-    context "when body is JSON" do
-      context "and headers are set correctly" do
-        let(:response) { [200, {"content-type" => "application/json"}, "{\"id\": 10}"] }
+
+      context 'with body target' do
+
+        let(:config) { super().merge "target_body" => '[rest]' }
 
         it "fetches and writes body to target" do
-          expect(event.get('[rest][id]')).to eq(10)
+          expect(event.get('rest')).to eq("Bom dia")
+          expect(event.include?('body')).to be false
         end
+
       end
+
     end
   end
+
   describe 'URL parameter' do
     before(:each) { subject.register }
     context "when url contains field references" do
@@ -45,6 +86,7 @@ describe LogStash::Filters::Http do
       it "interpolates request url using event data" do
         expect(subject).to receive(:request_http).with(anything, "http://stringsize.com/test", anything).and_return(response)
         subject.filter(event)
+        expect(event.get('size')).to eql '4'
       end
     end
   end
@@ -68,38 +110,77 @@ describe LogStash::Filters::Http do
       expect(event.get('tags')).to include('_httprequestfailure')
     end
   end
-  describe "headers" do
+
+  describe "headers", :ecs_compatibility_support do
     before(:each) { subject.register }
-    let(:response) { [200, {}, "Bom dia"] }
-    context "when set" do
-      let(:headers) { { "Cache-Control" => "nocache" } }
-      let(:config) do
-        {
-          "url" => "http://stringsize.com",
-          "target_body" => "size",
-          "headers" => headers
-        }
+    let(:response) do
+      response_headers = {
+          'Server' => 'Apache',
+          'Last-Modified' => 'Mon, 18 Jul 2016 02:36:04 GMT',
+          'X-Backend-Server' => 'logstash.elastic.co',
+      }
+      [200, response_headers, "Bom dia"]
+    end
+
+    let(:url) { "http://stringsize.com" }
+
+    ecs_compatibility_matrix(:disabled, :v1) do |ecs_select|
+
+      let(:config) { super().merge "ecs_compatibility" => ecs_compatibility }
+
+      it "sets response headers in the event" do
+        expect(subject).to receive(:request_http).with(anything, config['url'], anything).and_return(response)
+
+        subject.filter(event)
+
+        if ecs_select.active_mode == :disabled
+          expect(event.get('headers')).to include "Server" => "Apache"
+          expect(event.get('headers')).to include "X-Backend-Server" => "logstash.elastic.co"
+        else
+          expect(event.include?('headers')).to be false
+          expect(event.get('[@metadata][filter][http][response][headers]')).to include "Server" => "Apache"
+          expect(event.get('[@metadata][filter][http][response][headers]')).to include "X-Backend-Server" => "logstash.elastic.co"
+        end
+      end
+
+      context 'with a headers target' do
+
+        let(:config) { super().merge "target_headers" => '[res][headers]' }
+
+        it "sets response headers in the event" do
+          expect(subject).to receive(:request_http).with(anything, config['url'], anything).and_return(response)
+
+          subject.filter(event)
+
+          expect(event.get('[res][headers]')).to include "Server" => "Apache"
+          expect(event.get('[res][headers]').keys).to include "Last-Modified"
+        end
+
       end
+
+    end
+
+    context "(extra) request headers" do
+      let(:headers) { { "Cache-Control" => "nocache" } }
+      let(:config) { super().merge "headers" => headers }
+
       it "are included in the request" do
         expect(subject).to receive(:request_http) do |verb, url, options|
-          expect(options.fetch(:headers, {})).to include(headers)
+          expect( options.fetch(:headers, {}) ).to include(headers)
         end.and_return(response)
+
         subject.filter(event)
       end
     end
   end
+
   describe "query string parameters" do
     before(:each) { subject.register }
     let(:response) { [200, {}, "Bom dia"] }
     context "when set" do
       let(:query) { { "color" => "green" } }
-      let(:config) do
-        {
-          "url" => "http://stringsize.com/%{message}",
-          "target_body" => "size",
-          "query" => query
-        }
-      end
+      let(:config) { super().merge "query" => query }
+
       it "are included in the request" do
         expect(subject).to receive(:request_http).with(anything, anything, include(:query => query)).and_return(response)
         subject.filter(event)
@@ -109,21 +190,11 @@ describe LogStash::Filters::Http do
   describe "request body" do
     before(:each) { subject.register }
     let(:response) { [200, {}, "Bom dia"] }
-    let(:config) do
-      {
-        "url" => "http://stringsize.com",
-        "body" => body
-      }
-    end
+    let(:url) { "http://stringsize.com" }
+    let(:config) { super().merge "body" => body }
 
     describe "format" do
-      let(:config) do
-        {
-          "url" => "http://stringsize.com",
-          "body_format" => body_format,
-          "body" => body
-        }
-      end
+      let(:config) { super().merge "body_format" => body_format, "body" => body }
 
       context "when is json" do
         let(:body_format) { "json" }
@@ -190,13 +261,8 @@ describe LogStash::Filters::Http do
   end
   describe "verb" do
     let(:response) { [200, {}, "Bom dia"] }
-    let(:config) do
-      {
-        "verb" => verb,
-        "url" => "http://stringsize.com",
-        "target_body" => "size"
-      }
-    end
+    let(:config) { super().merge "verb" => verb }
+
     ["GET", "HEAD", "POST", "DELETE", "PATCH", "PUT"].each do |verb_string|
       let(:verb) { verb_string }
       context "when verb #{verb_string} is set" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-http
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-03 00:00:00.000000000 Z
+date: 2021-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,20 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  name: logstash-mixin-ecs_compatibility_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -50,6 +64,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 9.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-validator_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: