logstash-filter-http 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d5da94956185d0453f30766ff885a90eaa8f0e90954e5f1fc8e8c3654baccb01
+  data.tar.gz: 217cdde8ce00bb9812531ac65edbe6cb0abcacbb07a8aeff8477f35b13904b3d
+SHA512:
+  metadata.gz: feb63179da4d54e1d0b38091b8fd54ccd3a41ffc9d3617c807930523144c7410d389de665bc119cedecf7c97e7f189f75f7806a01791b3882d57303f9b5e6cc3
+  data.tar.gz: 6dba46bc255f7a7205cadf94160eb0b31ab1d202592031dfd9f98b7f631f4d3afaac4470280a460024054d4f288680dc080028f0b78297d4d43d679e0d0b86bd

data/CHANGELOG.md

@@ -0,0 +1,28 @@
+## 0.5.4
+
+  - update `gemspec` to work with logstash 5.5
+
+## 0.5.3
+  - freeze all instance variables
+  - fix parallel processing by creating a `deep_clone` for each event
+  - use `LogStash::Util.deep_clone` for object cloning
+  - only dump body as json, if json is enabled in config (default)
+  - delete empty target testcase, as catched by upper logstash `LogStash::ConfigurationError`
+  - fix `sprintf` find and merge for more complex structures
+
+## 0.5.2
+  - Fix behavior, where a referenced field (`%{...}`) has `ruby` chars
+  (i.e., consisting of `:`)
+  - Field interpolation is done by assigning explicit values instead
+  of converting the `sprintf` string back into a `hash`
+
+## 0.5.0
+  - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
+  - Require devutils >= 0 to make `bundler` update the package
+  - Use Event API for LS-5
+  - Implicit `sprintf`, deprecating the setting
+  - `target` is now required, dropping support to write into top-level in favor of only using new Event API
+    - this follows other logstash-plugins like `logstash-filter-json`
+  - if the response is empty, add the restfailure tags
+  - Some logging moved before code
+  - Testcases adapted to new behavior with error check

data/CONTRIBUTORS

@@ -0,0 +1,11 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* Aaron Mildenstein (untergeek)
+* Pier-Hugues Pellerin (ph)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/Gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,92 @@
+# Logstash REST Filter [![Build Status](https://travis-ci.org/logstash-plugins/logstash-filter-http.svg?branch=master)](https://travis-ci.org/logstash-plugins/logstash-filter-http)
+
+This is a filter plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+This logstash filter provides an easy way to access RESTful Resources within logstash. It can be used to post data to a REST API or to gather data and save it in your log file.
+
+## Usage
+### 1. Installation
+You can use the built-in plugin tool of Logstash to install the filter:
+```
+$LS_HOME/bin/logstash-plugin install logstash-filter-rest
+```
+
+Or you can build it yourself:
+```
+git clone https://github.com/lucashenning/logstash-filter-rest.git
+bundle install
+gem build logstash-filter-rest.gemspec
+$LS_HOME/bin/logstash-plugin install logstash-filter-rest-0.1.0.gem
+```
+
+### 2. Filter Configuration
+Add the following inside the filter section of your logstash configuration:
+
+```sh
+filter {
+  rest {
+    request => {
+      url => "http://example.com"        # string (required, with field reference: "http://example.com?id=%{id}" or params, if defined)
+      method => "post"                   # string (optional, default = "get")
+      headers => {                       # hash (optional)
+        "key1" => "value1"
+        "key2" => "value2"
+      }
+      auth => {
+        user => "AzureDiamond"
+        password => "hunter2"
+      }
+      params => {                        # hash (optional, available for method => "get" and "post"; if post it will be transformed into body hash and posted as json)
+        "key1" => "value1"
+        "key2" => "value2"
+        "key3" => "%{somefield}"         # sprintf is used implicitly
+      }
+    }
+    json => true                         # boolean (optional, default = true)
+    target => "my_key"                   # string (mandatory, no default)
+    fallback => {                        # hash describing a default in case of error
+      "key1" => "value1"
+      "key2" => "value2"
+    }
+  }
+}
+```
+
+Print plugin version:
+
+``` bash
+bin/logstash-plugin list --verbose | grep rest
+```
+
+Examples for running logstash from `cli`:
+
+``` bash
+bin/logstash --debug -e 'input { stdin{} } filter { rest { request => { url => "https://jsonplaceholder.typicode.com/posts" method => "post" params => { "userId" => "%{message}" } headers => { "Content-Type" => "application/json" } } target => 'rest' } } output {stdout { codec => rubydebug }}'
+```
+
+``` bash
+bin/logstash --debug -e 'input { stdin{} } filter { rest { request => { url => "https://jsonplaceholder.typicode.com/posts" method => "post" body => { "userId" => "%{message}" } headers => { "Content-Type" => "application/json" } } target => 'rest' } } output {stdout { codec => rubydebug }}'
+```
+
+``` bash
+bin/logstash --debug -e 'input { stdin{} } filter { rest { request => { url => "http://jsonplaceholder.typicode.com/users/%{message}" } target => 'rest' } } output {stdout { codec => rubydebug }}'
+```
+
+``` bash
+bin/logstash --debug -e 'input { stdin{} } filter { rest { request => { url => "https://jsonplaceholder.typicode.com/posts" method => "get" params => { "userId" => "%{message}" } headers => { "Content-Type" => "application/json" } } target => 'rest' } } output {stdout { codec => rubydebug }}'
+```
+
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/filters/http.rb

@@ -0,0 +1,120 @@
+# encoding: utf-8
+require 'logstash/filters/base'
+require 'logstash/namespace'
+require 'logstash/plugin_mixins/http_client'
+require 'logstash/json'
+
+# Logstash HTTP Filter
+# This filter calls a defined URL and saves the answer into a specified field.
+#
+class LogStash::Filters::Http < LogStash::Filters::Base
+  include LogStash::PluginMixins::HttpClient
+
+  config_name 'http'
+
+  VALID_VERBS = ['GET', 'HEAD', 'PATCH', 'DELETE', 'POST']
+
+  config :url, :validate => :string, :required => true
+  config :verb, :validate => VALID_VERBS, :required => false, :default => 'GET'
+  config :headers, :validate => :hash, :required => false, :default => {}
+  config :query, :validate => :hash, :required => false, :default => {}
+  config :body, :required => false
+  config :body_format, :validate => ['text', 'json'], :default => "text"
+
+  config :target_body, :validate => :string, :default => "body"
+  config :target_headers, :validate => :string, :default => "headers"
+
+  # Append values to the `tags` field when there has been no
+  # successful match or json parsing error
+  config :tag_on_request_failure, :validate => :array, :default => ['_httprequestfailure']
+  config :tag_on_json_failure, :validate => :array, :default => ['_jsonparsefailure']
+
+  def register
+    # nothing to see here
+    @verb = verb.downcase
+  end
+
+  def filter(event)
+    url_for_event = event.sprintf(@url)
+    headers_sprintfed = sprintf_object(event, @headers)
+    if body_format == "json"
+      headers_sprintfed["content-type"] = "application/json"
+    else
+      headers_sprintfed["content-type"] = "text/plain"
+    end
+    query_sprintfed = sprintf_object(event, @query)
+    body_sprintfed = sprintf_object(event, @body)
+    # we don't need to serialize strings and numbers
+    if @body_format == "json" && body_sprintfed.kind_of?(Enumerable)
+      body_sprintfed = LogStash::Json.dump(body_sprintfed)
+    end
+
+    options = { :headers => headers_sprintfed, :query => query_sprintfed, :body => body_sprintfed }
+
+    @logger.debug? && @logger.debug('processing request', :url => url_for_event, :headers => headers_sprintfed, :parameters => parameters_sprintfed)
+    client_error = nil
+
+    begin
+      code, response_headers, response_body = request_http(@verb, url_for_event, options)
+    rescue => e
+      client_error = e
+    end
+
+    if client_error
+      @logger.error('error during HTTP request',
+                    :url => url_for_event, :body => @body,
+                    :client_error => client_error.message)
+      @tag_on_request_failure.each { |tag| event.tag(tag) }
+    elsif !code.between?(200, 299)
+      @logger.error('error during HTTP request',
+                    :url => url_for_event, :code => code,
+                    :response => response_body)
+      @tag_on_request_failure.each { |tag| event.tag(tag) }
+    else
+      @logger.debug? && @logger.debug('success received',
+                                      :code => code, :body => response_body)
+      process_response(response_body, response_headers, event)
+      filter_matched(event)
+    end
+  end # def filter
+
+  private
+  def request_http(verb, url, options = {})
+    response = client.http(verb, url, options)
+    [response.code, response.headers, response.body]
+  end
+
+  def sprintf_object(event, obj)
+    case obj
+    when Array
+      obj.map {|el| sprintf_object(event, el) }
+    when Hash
+      obj.inject({}) {|acc, (k,v)| acc[sprintf_object(event, k)] = sprintf_object(event, v); acc }
+    when String
+      event.sprintf(obj)
+    else
+      obj
+    end
+  end
+
+  def process_response(body, headers, event)
+    content_type, _ = headers.fetch("content-type", "").split(";")
+    event.set(@target_headers, headers)
+    if content_type == "application/json"
+      begin
+        parsed = LogStash::Json.load(body)
+        event.set(@target_body, parsed)
+      rescue => e
+        if @logger.debug?
+          @logger.warn('JSON parsing error', :message => e.message, :body => body)
+        else
+          @logger.warn('JSON parsing error', :message => e.message)
+        end
+        @tag_on_json_failure.each { |tag| event.tag(tag) }
+      end
+    else
+      event.set(@target_body, body.strip)
+    end
+  end
+
+end # class LogStash::Filters::Rest

data/logstash-filter-http.gemspec

@@ -0,0 +1,33 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-filter-http'
+  s.version = '0.1.0'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = 'This filter requests data from a RESTful Web Service.'
+  s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-filter-http. This gem is not a stand-alone program'
+  s.authors = ['Lucas Henning', 'Gandalf Buscher']
+  s.email = 'mail@hurb.de'
+  s.homepage = 'https://github.com/lucashenning/logstash-filter-http/'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*',
+                'spec/**/*',
+                'vendor/**/*',
+                '*.gemspec',
+                '*.md',
+                'CONTRIBUTORS',
+                'Gemfile',
+                'LICENSE',
+                'NOTICE.TXT']
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'filter' }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
+  s.add_runtime_dependency 'logstash-mixin-http_client', '>= 5.0.0', '< 9.0.0'
+
+  s.add_development_dependency 'logstash-devutils', '>= 0', '< 2.0.0'
+end

data/spec/filters/http_spec.rb

@@ -0,0 +1,507 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/filters/http'
+
+describe LogStash::Filters::Http do
+  subject { described_class.new(config) }
+  let(:event) { LogStash::Event.new(data) }
+  let(:data) { { "message" => "test" } }
+
+  describe 'response body handling' do
+    before(:each) { subject.register }
+    let(:url) { 'http://laceholder.typicode.com/users/10' }
+    let(:config) do
+      { "url" => url, "target_body" => 'rest' }
+    end
+    before(:each) do
+      allow(subject).to receive(:request_http).and_return(response)
+      subject.filter(event)
+    end
+
+    context "when body is text" do
+      let(:response) { [200, {}, "Bom dia"] }
+
+      it "fetches and writes body to target" do
+        expect(event.get('rest')).to eq("Bom dia")
+      end
+    end
+    context "when body is JSON" do
+      context "and headers are set correctly" do
+        let(:response) { [200, {"content-type" => "application/json"}, "{\"id\": 10}"] }
+
+        it "fetches and writes body to target" do
+          expect(event.get('[rest][id]')).to eq(10)
+        end
+      end
+    end
+  end
+  describe 'URL parameter' do
+    before(:each) { subject.register }
+    context "when url contains field references" do
+      let(:config) do
+        { "url" => "http://stringsize.com/%{message}", "target_body" => "size" }
+      end
+      let(:response) { [200, {}, "4"] }
+
+      it "interpolates request url using event data" do
+        expect(subject).to receive(:request_http).with(anything, "http://stringsize.com/test", anything).and_return(response)
+        subject.filter(event)
+      end
+    end
+  end
+  context 'when request returns 404' do
+    before(:each) { subject.register }
+    let(:config) do
+      {
+        'url' => 'http://httpstat.us/404',
+        'target_body' => 'rest'
+      }
+    end
+    let(:response) { [404, {}, ""] }
+
+    before(:each) do
+      allow(subject).to receive(:request_http).and_return(response)
+      subject.filter(event)
+    end
+
+    it "tags the event with _httprequestfailure" do
+      expect(event).to_not include('rest')
+      expect(event.get('tags')).to include('_httprequestfailure')
+    end
+  end
+  describe "headers" do
+    before(:each) { subject.register }
+    let(:response) { [200, {}, "Bom dia"] }
+    context "when set" do
+      let(:headers) { { "Cache-Control" => "nocache" } }
+      let(:config) do
+        {
+          "url" => "http://stringsize.com",
+          "target_body" => "size",
+          "headers" => headers
+        }
+      end
+      it "are included in the request" do
+        expect(subject).to receive(:request_http) do |verb, url, options|
+          expect(options.fetch(:headers, {})).to include(headers)
+        end.and_return(response)
+        subject.filter(event)
+      end
+    end
+  end
+  describe "query string parameters" do
+    before(:each) { subject.register }
+    let(:response) { [200, {}, "Bom dia"] }
+    context "when set" do
+      let(:query) { { "color" => "green" } }
+      let(:config) do
+        {
+          "url" => "http://stringsize.com/%{message}",
+          "target_body" => "size",
+          "query" => query
+        }
+      end
+      it "are included in the request" do
+        expect(subject).to receive(:request_http).with(anything, anything, include(:query => query)).and_return(response)
+        subject.filter(event)
+      end
+    end
+  end
+  describe "request body" do
+    before(:each) { subject.register }
+    let(:response) { [200, {}, "Bom dia"] }
+    let(:config) do
+      {
+        "url" => "http://stringsize.com",
+        "body" => body
+      }
+    end
+
+    describe "format" do
+      let(:config) do
+        {
+          "url" => "http://stringsize.com",
+          "body_format" => body_format,
+          "body" => body
+        }
+      end
+
+      context "when is json" do
+        let(:body_format) { "json" }
+        let(:body) do
+          { "hey" => "you" }
+        end
+        let(:body_json) { LogStash::Json.dump(body) }
+
+        it "serializes the body to json" do
+          expect(subject).to receive(:request_http) do |verb, url, options|
+            expect(options).to include(:body => body_json)
+          end.and_return(response)
+          subject.filter(event)
+        end
+        it "sets content-type to application/json" do
+          expect(subject).to receive(:request_http) do |verb, url, options|
+            expect(options).to include(:headers => { "content-type" => "application/json"})
+          end.and_return(response)
+          subject.filter(event)
+        end
+      end
+      context "when is text" do
+        let(:body_format) { "text" }
+        let(:body) { "Hey, you!" }
+
+        it "uses the text as body for the request" do
+          expect(subject).to receive(:request_http) do |verb, url, options|
+            expect(options).to include(:body => body)
+          end.and_return(response)
+          subject.filter(event)
+        end
+        it "sets content-type to text/plain" do
+          expect(subject).to receive(:request_http) do |verb, url, options|
+            expect(options).to include(:headers => { "content-type" => "text/plain"})
+          end.and_return(response)
+          subject.filter(event)
+        end
+      end
+    end
+    context "when using field references" do
+      let(:body_format) { "json" }
+      let(:body) do
+        { "%{key1}" => [ "%{[field1]}", "another_value", { "key" => "other-%{[nested][field2]}" } ] }
+      end
+      let(:body_json) { LogStash::Json.dump(body) }
+      let(:data) do
+        {
+          "message" => "ola",
+          "key1" => "mykey",
+          "field1" => "normal value",
+          "nested" => { "field2" => "value2" }
+        }
+      end
+
+      it "fills the body with event data" do
+        expect(subject).to receive(:request_http) do |verb, url, options|
+          body = options.fetch(:body, {})
+          expect(body.keys).to include("mykey")
+          expect(body.fetch("mykey")).to eq(["normal value", "another_value", { "key" => "other-value2" }])
+        end.and_return(response)
+        subject.filter(event)
+      end
+    end
+  end
+  describe "verb" do
+    let(:response) { [200, {}, "Bom dia"] }
+    let(:config) do
+      {
+        "verb" => verb,
+        "url" => "http://stringsize.com",
+        "target_body" => "size"
+      }
+    end
+    ["GET", "HEAD", "POST", "DELETE"].each do |verb_string|
+      let(:verb) { verb_string }
+      context "when verb #{verb_string} is set" do
+        before(:each) { subject.register }
+        it "it is used in the request" do
+          expect(subject).to receive(:request_http).with(verb.downcase, anything, anything).and_return(response)
+          subject.filter(event)
+        end
+      end
+    end
+    context "when using an invalid verb" do
+      let(:verb) { "something else" }
+      it "it is used in the request" do
+        expect { described_class.new(config) }.to raise_error ::LogStash::ConfigurationError
+      end
+    end
+  end
+end
+
+=begin
+  # TODO refactor remaning tests to avoid insist + whole pipeline instantiation
+  describe 'empty response' do
+    let(:config) do <<-CONFIG
+      filter {
+        rest {
+          request => {
+            url => 'https://jsonplaceholder.typicode.com/posts'
+            params => {
+              userId => 0
+            }
+            headers => {
+              'Content-Type' => 'application/json'
+            }
+          }
+          target => 'rest'
+        }
+      }
+    CONFIG
+    end
+
+    sample('message' => 'some text') do
+      expect(subject).to_not include('rest')
+      expect(subject.get('tags')).to include('_restfailure')
+    end
+  end
+  describe 'Set to Rest Filter Get with params sprintf' do
+    let(:config) do <<-CONFIG
+      filter {
+        rest {
+          request => {
+            url => 'https://jsonplaceholder.typicode.com/posts'
+            params => {
+              userId => "%{message}"
+              id => "%{message}"
+            }
+            headers => {
+              'Content-Type' => 'application/json'
+            }
+          }
+          json => true
+          target => 'rest'
+        }
+      }
+    CONFIG
+    end
+
+    sample('message' => '1') do
+      expect(subject).to include('rest')
+      expect(subject.get('[rest][0]')).to include('userId')
+      expect(subject.get('[rest][0][userId]')).to eq(1)
+      expect(subject.get('[rest][0][id]')).to eq(1)
+      expect(subject.get('rest').length).to eq(1)
+      expect(subject.get('rest')).to_not include('fallback')
+    end
+  end
+  describe 'Set to Rest Filter Post with params' do
+    let(:config) do <<-CONFIG
+      filter {
+        rest {
+          request => {
+            url => 'https://jsonplaceholder.typicode.com/posts'
+            method => 'post'
+            params => {
+              title => 'foo'
+              body => 'bar'
+              userId => 42
+            }
+            headers => {
+              'Content-Type' => 'application/json'
+            }
+          }
+          json => true
+          target => 'rest'
+        }
+      }
+    CONFIG
+    end
+
+    sample('message' => 'some text') do
+      expect(subject).to include('rest')
+      expect(subject.get('rest')).to include('id')
+      expect(subject.get('[rest][userId]')).to eq(42)
+      expect(subject.get('rest')).to_not include('fallback')
+    end
+  end
+  describe 'Set to Rest Filter Post with params sprintf' do
+    let(:config) do <<-CONFIG
+      filter {
+        rest {
+          request => {
+            url => 'https://jsonplaceholder.typicode.com/posts'
+            method => 'post'
+            params => {
+              title => '%{message}'
+              body => 'bar'
+              userId => "%{message}"
+            }
+            headers => {
+              'Content-Type' => 'application/json'
+            }
+          }
+          json => true
+          target => 'rest'
+        }
+      }
+    CONFIG
+    end
+
+    sample('message' => '42') do
+      expect(subject).to include('rest')
+      expect(subject.get('rest')).to include('id')
+      expect(subject.get('[rest][title]')).to eq(42)
+      expect(subject.get('[rest][userId]')).to eq(42)
+      expect(subject.get('rest')).to_not include('fallback')
+    end
+    sample('message' => ':5e?#!-_') do
+      expect(subject).to include('rest')
+      expect(subject.get('rest')).to include('id')
+      expect(subject.get('[rest][title]')).to eq(':5e?#!-_')
+      expect(subject.get('[rest][userId]')).to eq(':5e?#!-_')
+      expect(subject.get('rest')).to_not include('fallback')
+    end
+    sample('message' => ':4c43=>') do
+      expect(subject).to include('rest')
+      expect(subject.get('rest')).to include('id')
+      expect(subject.get('[rest][title]')).to eq(':4c43=>')
+      expect(subject.get('[rest][userId]')).to eq(':4c43=>')
+      expect(subject.get('rest')).to_not include('fallback')
+    end
+  end
+  describe 'Set to Rest Filter Post with body sprintf' do
+    let(:config) do <<-CONFIG
+      filter {
+        rest {
+          request => {
+            url => 'https://jsonplaceholder.typicode.com/posts'
+            method => 'post'
+            body => {
+              title => 'foo'
+              body => 'bar'
+              userId => "%{message}"
+            }
+            headers => {
+              'Content-Type' => 'application/json'
+            }
+          }
+          json => true
+          target => 'rest'
+        }
+      }
+    CONFIG
+    end
+
+    sample('message' => '42') do
+      expect(subject).to include('rest')
+      expect(subject.get('rest')).to include('id')
+      expect(subject.get('[rest][userId]')).to eq(42)
+      expect(subject.get('rest')).to_not include('fallback')
+    end
+  end
+  describe 'Set to Rest Filter Post with body sprintf nested params' do
+    let(:config) do <<-CONFIG
+      filter {
+        rest {
+          request => {
+            url => 'https://jsonplaceholder.typicode.com/posts'
+            method => 'post'
+            body => {
+              key1 => [
+                {
+                  "filterType" => "text"
+                  "text" => "salmon"
+                  "boolean" => false
+                },
+                {
+                  "filterType" => "unique"
+                }
+              ]
+              key2 => [
+                {
+                  "message" => "123%{message}"
+                  "boolean" => true
+                }
+              ]
+              key3 => [
+                {
+                  "text" => "%{message}123"
+                  "filterType" => "text"
+                  "number" => 44
+                },
+                {
+                  "filterType" => "unique"
+                  "null" => nil
+                }
+              ]
+              userId => "%{message}"
+            }
+            headers => {
+              'Content-Type' => 'application/json'
+            }
+          }
+          target => 'rest'
+        }
+      }
+    CONFIG
+    end
+
+    sample('message' => '42') do
+      expect(subject).to include('rest')
+      expect(subject.get('rest')).to include('key1')
+      expect(subject.get('[rest][key1][0][boolean]')).to eq('false')
+      expect(subject.get('[rest][key1][1][filterType]')).to eq('unique')
+      expect(subject.get('[rest][key2][0][message]')).to eq('12342')
+      expect(subject.get('[rest][key2][0][boolean]')).to eq('true')
+      expect(subject.get('[rest][key3][0][text]')).to eq('42123')
+      expect(subject.get('[rest][key3][0][filterType]')).to eq('text')
+      expect(subject.get('[rest][key3][0][number]')).to eq(44)
+      expect(subject.get('[rest][key3][1][filterType]')).to eq('unique')
+      expect(subject.get('[rest][key3][1][null]')).to eq('nil')
+      expect(subject.get('[rest][userId]')).to eq(42)
+      expect(subject.get('rest')).to_not include('fallback')
+    end
+  end
+  describe 'fallback' do
+    let(:config) do <<-CONFIG
+      filter {
+        rest {
+          request => {
+            url => 'http://jsonplaceholder.typicode.com/users/0'
+          }
+          json => true
+          fallback => {
+            'fallback1' => true
+            'fallback2' => true
+          }
+          target => 'rest'
+        }
+      }
+    CONFIG
+    end
+
+    sample('message' => 'some text') do
+      expect(subject).to include('rest')
+      expect(subject.get('rest')).to include('fallback1')
+      expect(subject.get('rest')).to include('fallback2')
+      expect(subject.get('rest')).to_not include('id')
+    end
+  end
+  describe 'empty target exception' do
+    let(:config) do <<-CONFIG
+      filter {
+        rest {
+          request => {
+            url => 'http://jsonplaceholder.typicode.com/users/0'
+          }
+          json => true
+          fallback => {
+            'fallback1' => true
+            'fallback2' => true
+          }
+          target => ''
+        }
+      }
+    CONFIG
+    end
+    sample('message' => 'some text') do
+      expect { subject }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+  describe 'http client throws exception' do
+    let(:config) do <<-CONFIG
+      filter {
+        rest {
+          request => {
+            url => 'invalid_url'
+          }
+          target => 'rest'
+        }
+      }
+    CONFIG
+    end
+    sample('message' => 'some text') do
+      expect(subject).to_not include('rest')
+      expect(subject.get('tags')).to include('_restfailure')
+    end
+  end
+end
+=end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-http
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Lucas Henning
+- Gandalf Buscher
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-12-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 9.0.0
+  name: logstash-mixin-http_client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 9.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-filter-http.
+  This gem is not a stand-alone program
+email: mail@hurb.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/filters/http.rb
+- logstash-filter-http.gemspec
+- spec/filters/http_spec.rb
+homepage: https://github.com/lucashenning/logstash-filter-http/
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.13
+signing_key:
+specification_version: 4
+summary: This filter requests data from a RESTful Web Service.
+test_files:
+- spec/filters/http_spec.rb