logstash-output-bcdb 0.1.0

data/logstash-output-bcdb.gemspec

@@ -0,0 +1,43 @@
+#
+# Copyright ©2020. MODEX (Gibraltar) LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Gem::Specification.new do |s|
+  s.name            = 'logstash-output-bcdb'
+  s.version         = '0.1.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Sends events to a BCDB Database endpoint"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Modex"]
+  s.email           = 'support@modex.tech'
+  s.homepage        = "https://github.com/modex-bcdb/logstash-output-bcdb"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 6.0.0", "< 8.0.0"
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'sinatra'
+  s.add_development_dependency 'webrick'
+end

data/spec/outputs/bcdb_spec.rb

@@ -0,0 +1,364 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/http"
+require "logstash/codecs/plain"
+require "thread"
+require "sinatra"
+require_relative "../supports/compressed_requests"
+
+PORT = rand(65535-1024) + 1025
+
+class LogStash::Outputs::Http
+  attr_writer :agent
+  attr_reader :request_tokens
+end
+
+# note that Sinatra startup and shutdown messages are directly logged to stderr so
+# it is not really possible to disable them without reopening stderr which is not advisable.
+#
+# == Sinatra (v1.4.6) has taken the stage on 51572 for development with backup from WEBrick
+# == Sinatra has ended his set (crowd applauds)
+#
+class TestApp < Sinatra::Base
+  # on the fly uncompress gzip content
+  use CompressedRequests
+
+  # disable WEBrick logging
+  def self.server_settings
+    { :AccessLog => [], :Logger => WEBrick::BasicLog::new(nil, WEBrick::BasicLog::FATAL) }
+  end
+
+  def self.multiroute(methods, path, &block)
+    methods.each do |method|
+      method.to_sym
+      self.send method, path, &block
+    end
+  end
+
+  def self.last_request=(request)
+    @last_request = request
+  end
+
+  def self.last_request
+    @last_request
+  end
+
+  def self.retry_fail_count=(count)
+    @retry_fail_count = count
+  end
+
+  def self.retry_fail_count()
+    @retry_fail_count || 2
+  end
+
+  multiroute(%w(get post put patch delete), "/good") do
+    self.class.last_request = request
+    [200, "YUP"]
+  end
+
+  multiroute(%w(get post put patch delete), "/bad") do
+    self.class.last_request = request
+    [400, "YUP"]
+  end
+
+  multiroute(%w(get post put patch delete), "/retry") do
+    self.class.last_request = request
+
+    if self.class.retry_fail_count > 0
+      self.class.retry_fail_count -= 1
+      [429, "Will succeed in #{self.class.retry_fail_count}"]
+    else
+      [200, "Done Retrying"]
+    end
+  end
+end
+
+RSpec.configure do |config|
+  #http://stackoverflow.com/questions/6557079/start-and-call-ruby-http-server-in-the-same-script
+  def sinatra_run_wait(app, opts)
+    queue = Queue.new
+
+    t = java.lang.Thread.new(
+      proc do
+        begin
+          app.run!(opts) do |server|
+            queue.push("started")
+          end
+        rescue => e
+          puts "Error in webserver thread #{e}"
+          # ignore
+        end
+      end
+    )
+    t.daemon = true
+    t.start
+    queue.pop # blocks until the run! callback runs
+  end
+
+  config.before(:suite) do
+    sinatra_run_wait(TestApp, :port => PORT, :server => 'webrick')
+    puts "Test webserver on port #{PORT}"
+  end
+end
+
+describe LogStash::Outputs::Http do
+  # Wait for the async request to finish in this spinlock
+  # Requires pool_max to be 1
+
+  let(:port) { PORT }
+  let(:event) {
+    LogStash::Event.new({"message" => "hi"})
+  }
+  let(:url) { "http://localhost:#{port}/good" }
+  let(:method) { "post" }
+
+  shared_examples("verb behavior") do |method|
+    let(:verb_behavior_config) { {"url" => url, "http_method" => method, "pool_max" => 1} }
+    subject { LogStash::Outputs::Http.new(verb_behavior_config) }
+
+    let(:expected_method) { method.clone.to_sym }
+    let(:client) { subject.client }
+
+    before do
+      subject.register
+      allow(client).to receive(:send).
+                         with(expected_method, url, anything).
+                         and_call_original
+      allow(subject).to receive(:log_failure).with(any_args)
+      allow(subject).to receive(:log_retryable_response).with(any_args)
+    end
+
+    context 'sending no events' do
+      it 'should not block the pipeline' do
+        subject.multi_receive([])
+      end
+    end
+
+    context "performing a get" do
+      describe "invoking the request" do
+        before do
+          subject.multi_receive([event])
+        end
+
+        it "should execute the request" do
+          expect(client).to have_received(:send).
+                              with(expected_method, url, anything)
+        end
+      end
+
+      context "with passing requests" do
+        before do
+          subject.multi_receive([event])
+        end
+
+        it "should not log a failure" do
+          expect(subject).not_to have_received(:log_failure).with(any_args)
+        end
+      end
+
+      context "with failing requests" do
+        let(:url) { "http://localhost:#{port}/bad"}
+
+        before do
+          subject.multi_receive([event])
+        end
+
+        it "should log a failure" do
+          expect(subject).to have_received(:log_failure).with(any_args)
+        end
+      end
+
+      context "with ignorable failing requests" do
+        let(:url) { "http://localhost:#{port}/bad"}
+        let(:verb_behavior_config) { super.merge("ignorable_codes" => [400]) }
+
+        before do
+          subject.multi_receive([event])
+        end
+
+        it "should log a failure" do
+          expect(subject).not_to have_received(:log_failure).with(any_args)
+        end
+      end
+
+      context "with retryable failing requests" do
+        let(:url) { "http://localhost:#{port}/retry"}
+
+        before do
+          TestApp.retry_fail_count=2
+          allow(subject).to receive(:send_event).and_call_original
+          subject.multi_receive([event])
+        end
+
+        it "should log a retryable response 2 times" do
+          expect(subject).to have_received(:log_retryable_response).with(any_args).twice
+        end
+
+        it "should make three total requests" do
+          expect(subject).to have_received(:send_event).exactly(3).times
+        end
+      end
+
+    end
+  end
+
+  LogStash::Outputs::Http::VALID_METHODS.each do |method|
+    context "when using '#{method}'" do
+      include_examples("verb behavior", method)
+    end
+  end
+
+  shared_examples("a received event") do
+    before do
+      TestApp.last_request = nil
+    end
+
+    let(:events) { [event] }
+
+    describe "with a good code" do
+      before do
+        subject.multi_receive(events)
+      end
+
+      let(:last_request) { TestApp.last_request }
+      let(:body) { last_request.body.read }
+      let(:content_type) { last_request.env["CONTENT_TYPE"] }
+
+      it "should receive the request" do
+        expect(last_request).to be_truthy
+      end
+
+      it "should receive the event as a hash" do
+        expect(body).to eql(expected_body)
+      end
+
+      it "should have the correct content type" do
+        expect(content_type).to eql(expected_content_type)
+      end
+    end
+
+    describe "a retryable code" do
+      let(:url) { "http://localhost:#{port}/retry" }
+
+      before do
+        TestApp.retry_fail_count=2
+        allow(subject).to receive(:send_event).and_call_original
+        allow(subject).to receive(:log_retryable_response)
+        subject.multi_receive(events)
+      end
+
+      it "should retry" do
+        expect(subject).to have_received(:log_retryable_response).with(any_args).twice
+      end
+    end
+  end
+
+  shared_examples "integration tests" do
+    let(:base_config) { {} }
+    let(:url) { "http://localhost:#{port}/good" }
+    let(:event) {
+      LogStash::Event.new("foo" => "bar", "baz" => "bot", "user" => "McBest")
+    }
+
+    subject { LogStash::Outputs::Http.new(config) }
+
+    before do
+      subject.register
+    end
+
+    describe "sending with the default (JSON) config" do
+      let(:config) {
+        base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1})
+      }
+      let(:expected_body) { LogStash::Json.dump(event) }
+      let(:expected_content_type) { "application/json" }
+
+      include_examples("a received event")
+    end
+
+    describe "sending the batch as JSON" do
+      let(:config) do
+        base_config.merge({"url" => url, "http_method" => "post", "format" => "json_batch"})
+      end
+
+      let(:expected_body) { ::LogStash::Json.dump events }
+      let(:events) { [::LogStash::Event.new("a" => 1), ::LogStash::Event.new("b" => 2)]}
+      let(:expected_content_type) { "application/json" }
+      
+      include_examples("a received event")
+
+    end
+
+    describe "sending the event as a form" do
+      let(:config) {
+        base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "form"})
+      }
+      let(:expected_body) { subject.send(:encode, event.to_hash) }
+      let(:expected_content_type) { "application/x-www-form-urlencoded" }
+
+      include_examples("a received event")
+    end
+
+    describe "sending the event as a message" do
+      let(:config) {
+        base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "message", "message" => "%{foo} AND %{baz}"})
+      }
+      let(:expected_body) { "#{event.get("foo")} AND #{event.get("baz")}" }
+      let(:expected_content_type) { "text/plain" }
+
+      include_examples("a received event")
+    end
+
+    describe "sending a mapped event" do
+      let(:config) {
+        base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "mapping" => {"blah" => "X %{foo}"} })
+      }
+      let(:expected_body) { LogStash::Json.dump("blah" => "X #{event.get("foo")}") }
+      let(:expected_content_type) { "application/json" }
+
+      include_examples("a received event")
+    end
+
+    describe "sending a mapped, nested event" do
+      let(:config) {
+        base_config.merge({
+          "url" => url,
+          "http_method" => "post",
+          "pool_max" => 1,
+          "mapping" => {
+            "host" => "X %{foo}",
+            "event" => {
+              "user" => "Y %{user}"
+            },
+            "arrayevent" => [{
+              "user" => "Z %{user}"
+            }]
+          }
+        })
+      }
+      let(:expected_body) {
+        LogStash::Json.dump({
+          "host" => "X #{event.get("foo")}",
+          "event" => {
+            "user" => "Y #{event.get("user")}"
+          },
+          "arrayevent" => [{
+            "user" => "Z #{event.get("user")}"
+          }]
+        })
+      }
+      let(:expected_content_type) { "application/json" }
+
+      include_examples("a received event")
+    end
+  end
+
+  describe "integration test without gzip compression" do
+    include_examples("integration tests")
+  end
+
+  describe "integration test with gzip compression" do
+    include_examples("integration tests") do
+      let(:base_config) { { "http_compression" => true } }
+    end
+  end
+end

data/spec/supports/compressed_requests.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+#
+# based on relistan's rack handler
+# out of the box rack only gives use the rack deflater handler to return compressed
+# response, this gist offer the inverse and should work on all rack based app like sinatra or rails.
+#
+# original source: https://gist.github.com/relistan/2109707
+require "zlib"
+
+class CompressedRequests
+  def initialize(app)
+    @app = app
+  end
+
+  def encoding_handled?(env)
+    ['gzip', 'deflate'].include? env['HTTP_CONTENT_ENCODING']
+  end
+
+  def call(env)
+    if encoding_handled?(env)
+      extracted = decode(env['rack.input'], env['HTTP_CONTENT_ENCODING'])
+
+      env.delete('HTTP_CONTENT_ENCODING')
+      env['CONTENT_LENGTH'] = extracted.bytesize
+      env['rack.input'] = StringIO.new(extracted)
+    end
+
+    status, headers, response = @app.call(env)
+    return [status, headers, response]
+  end
+
+  def decode(input, content_encoding)
+    case content_encoding
+      when 'gzip' then Zlib::GzipReader.new(input).read
+      when 'deflate' then Zlib::Inflate.inflate(input.read)
+    end
+  end
+end

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-bcdb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Modex
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-06-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  name: logstash-mixin-http_client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: support@modex.tech
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- docs/index.asciidoc
+- lib/logstash/outputs/bcdb.rb
+- logstash-output-bcdb.gemspec
+- spec/outputs/bcdb_spec.rb
+- spec/supports/compressed_requests.rb
+homepage: https://github.com/modex-bcdb/logstash-output-bcdb
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Sends events to a BCDB Database endpoint
+test_files:
+- spec/outputs/bcdb_spec.rb
+- spec/supports/compressed_requests.rb