logstash-input-http 3.1.0-java

data/logstash-input-http.gemspec

@@ -0,0 +1,33 @@
+HTTP_INPUT_VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "VERSION"))).strip unless defined?(HTTP_INPUT_VERSION)
+
+Gem::Specification.new do |s|
+  s.name = 'logstash-input-http'
+  s.version         = HTTP_INPUT_VERSION
+  s.licenses = ['Apache License (2.0)']
+  s.summary = "Receives events over HTTP or HTTPS"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["Elastic"]
+  s.email = 'info@elastic.co'
+  s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib", "vendor/jar-dependencies"]
+
+  # Files
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'jar-dependencies', '~> 0.3.4'
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-codec-json'
+  s.add_development_dependency 'logstash-codec-json_lines'
+  s.add_development_dependency 'manticore'
+
+  s.platform = "java"
+end

data/spec/inputs/http_spec.rb

@@ -0,0 +1,277 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/http"
+require "json"
+require "manticore"
+require "stud/temporary"
+require "zlib"
+require "stringio"
+
+describe LogStash::Inputs::Http do
+
+  before do
+    srand(RSpec.configuration.seed)
+  end
+
+  let(:client) { Manticore::Client.new(client_options) }
+  let(:client_options) { { } }
+  let(:logstash_queue) { Queue.new }
+  let(:port) { rand(5000) + 1025 }
+
+  it_behaves_like "an interruptible input plugin" do
+    let(:config) { { "port" => port } }
+  end
+
+  after :each do
+    client.clear_pending
+    client.close
+    subject.stop
+  end
+
+  describe "request handling" do
+    subject { LogStash::Inputs::Http.new("port" => port) }
+    before :each do
+      subject.register
+      t = Thread.new { subject.run(logstash_queue) }
+      ok = false
+      until ok
+        begin
+          client.post("http://127.0.0.1:#{port}", :body => '{}').call
+        rescue => e
+          # retry
+        else
+          ok = true
+        end
+        sleep 0.01
+      end
+      logstash_queue.pop if logstash_queue.size == 1 # pop test event
+    end
+
+    describe "handling overflowing requests with a 429" do
+      let(:logstash_queue_size) { rand(10) + 1 }
+      let(:max_pending_requests) { rand(5) + 1 }
+      let(:threads) { rand(4) + 1 }
+      let(:logstash_queue) { SizedQueue.new(logstash_queue_size) }
+      let(:client_options) { {
+        "request_timeout" => 0.1,
+        "connect_timeout" => 3,
+        "socket_timeout" => 0.1
+      } }
+
+      subject { described_class.new("port" => port, "threads" => threads, "max_pending_requests" => max_pending_requests) }
+
+      context "when sending more requests than queue slots" do
+        it "should block when the queue is full" do
+          # these will queue and return 200
+          logstash_queue_size.times.each do |i|
+            response = client.post("http://127.0.0.1:#{port}", :body => '{}').call
+            expect(response.code).to eq(200)
+          end
+
+          # these will block
+          (threads + max_pending_requests).times.each do |i|
+            expect {
+              client.post("http://127.0.0.1:#{port}", :body => '{}').call
+            }.to raise_error(Manticore::SocketTimeout)
+          end
+
+          # by now we should be rejecting with 429
+          response = client.post("http://127.0.0.1:#{port}", :body => '{}').call
+          expect(response.code).to eq(429)
+        end
+      end
+    end
+
+    it "should include remote host in \"host\" property" do
+      client.post("http://127.0.0.1:#{port}/meh.json",
+                  :headers => { "content-type" => "text/plain" },
+                  :body => "hello").call
+      event = logstash_queue.pop
+      expect(event.get("host")).to eq("127.0.0.1")
+    end
+
+    context "with default codec" do
+      subject { LogStash::Inputs::Http.new("port" => port) }
+      context "when receiving a text/plain request" do
+        it "should process the request normally" do
+          client.post("http://127.0.0.1:#{port}/meh.json",
+                      :headers => { "content-type" => "text/plain" },
+                      :body => "hello").call
+          event = logstash_queue.pop
+          expect(event.get("message")).to eq("hello")
+        end
+      end
+      context "when receiving a deflate compressed text/plain request" do
+        it "should process the request normally" do
+          client.post("http://127.0.0.1:#{port}/meh.json",
+                      :headers => { "content-type" => "text/plain", "content-encoding" => "deflate" },
+                      :body => Zlib::Deflate.deflate("hello")).call
+          event = logstash_queue.pop
+          expect(event.get("message")).to eq("hello")
+        end
+      end
+      context "when receiving a deflate text/plain request that cannot be decompressed" do
+        let(:response) do
+          response = client.post("http://127.0.0.1:#{port}/meh.json",
+                                 :headers => { "content-type" => "text/plain", "content-encoding" => "deflate" },
+                                   :body => "hello").call
+        end
+        it "should respond with 400" do
+          expect(response.code).to eq(400)
+        end
+      end
+      context "when receiving a gzip compressed text/plain request" do
+        it "should process the request normally" do
+          wio = StringIO.new("w")
+          z = Zlib::GzipWriter.new(wio)
+          z.write("hello")
+          z.close
+          entity = org.apache.http.entity.ByteArrayEntity.new(wio.string.to_java_bytes)
+          response = client.post("http://127.0.0.1:#{port}",
+                      :headers => { "Content-Encoding" => "gzip" },
+                      :entity => entity).call
+          expect(response.code).to eq(200)
+          event = logstash_queue.pop
+          expect(event.get("message")).to eq("hello")
+        end
+      end
+      context "when receiving a gzip text/plain request that cannot be decompressed" do
+        let(:response) do
+          client.post("http://127.0.0.1:#{port}",
+                      :headers => { "Content-Encoding" => "gzip" },
+                      :body => Zlib::Deflate.deflate("hello")).call
+        end
+        it "should respond with 400" do
+          expect(response.code).to eq(400)
+        end
+      end
+      context "when receiving an application/json request" do
+        it "should parse the json body" do
+          client.post("http://127.0.0.1:#{port}/meh.json",
+                      :headers => { "content-type" => "application/json" },
+                      :body => { "message_body" => "Hello" }.to_json).call
+          event = logstash_queue.pop
+          expect(event.get("message_body")).to eq("Hello")
+        end
+      end
+    end
+
+    context "with json codec" do
+      subject { LogStash::Inputs::Http.new("port" => port, "codec" => "json") }
+      it "should parse the json body" do
+        response = client.post("http://127.0.0.1:#{port}/meh.json", :body => { "message" => "Hello" }.to_json).call
+        event = logstash_queue.pop
+        expect(event.get("message")).to eq("Hello")
+      end
+    end
+
+    context "with json_lines codec without final delimiter" do
+      subject { LogStash::Inputs::Http.new("port" => port, "codec" => "json_lines") }
+      let(:line1) { '{"foo": 1}' }
+      let(:line2) { '{"foo": 2}' }
+      it "should parse all json_lines in body including last one" do
+        client.post("http://localhost:#{port}/meh.json", :body => "#{line1}\n#{line2}").call
+        expect(logstash_queue.size).to eq(2)
+        event = logstash_queue.pop
+        expect(event.get("foo")).to eq(1)
+        event = logstash_queue.pop
+        expect(event.get("foo")).to eq(2)
+      end
+    end
+
+    context "when using a custom codec mapping" do
+      subject { LogStash::Inputs::Http.new("port" => port,
+                                           "additional_codecs" => { "application/json" => "plain" }) }
+      it "should decode the message accordingly" do
+        body = { "message" => "Hello" }.to_json
+        client.post("http://127.0.0.1:#{port}/meh.json",
+                    :headers => { "content-type" => "application/json" },
+                      :body => body).call
+        event = logstash_queue.pop
+        expect(event.get("message")).to eq(body)
+      end
+    end
+
+    context "when using custom headers" do
+      let(:custom_headers) { { 'access-control-allow-origin' => '*' } }
+      subject { LogStash::Inputs::Http.new("port" => port, "response_headers" => custom_headers) }
+
+      describe "the response" do
+        it "should include the custom headers" do
+          response = client.post("http://127.0.0.1:#{port}/meh", :body => "hello")
+          expect(response.headers.to_hash).to include(custom_headers)
+        end
+      end
+    end
+    describe "basic auth" do
+      user = "test"; password = "pwd"
+      subject { LogStash::Inputs::Http.new("port" => port, "user" => user, "password" => password) }
+      let(:auth_token) { Base64.strict_encode64("#{user}:#{password}") }
+      context "when client doesn't present auth token" do
+        let!(:response) { client.post("http://127.0.0.1:#{port}/meh", :body => "hi").call }
+        it "should respond with 401" do
+          expect(response.code).to eq(401)
+        end
+        it "should not generate an event" do
+          expect(logstash_queue).to be_empty
+        end
+      end
+      context "when client presents incorrect auth token" do
+        let!(:response) do
+          client.post("http://127.0.0.1:#{port}/meh",
+                      :headers => {
+                        "content-type" => "text/plain",
+                        "authorization" => "Basic meh"
+                      },
+                      :body => "hi").call
+        end
+        it "should respond with 401" do
+          expect(response.code).to eq(401)
+        end
+        it "should not generate an event" do
+          expect(logstash_queue).to be_empty
+        end
+      end
+      context "when client presents correct auth token" do
+        let!(:response) do
+          client.post("http://127.0.0.1:#{port}/meh",
+                      :headers => {
+                        "content-type" => "text/plain",
+                        "authorization" => "Basic #{auth_token}"
+                      }, :body => "hi").call
+        end
+        it "should respond with 200" do
+          expect(response.code).to eq(200)
+        end
+        it "should generate an event" do
+          expect(logstash_queue).to_not be_empty
+        end
+      end
+    end
+
+  end
+
+  context "with :ssl => false" do
+    subject { LogStash::Inputs::Http.new("port" => port, "ssl" => false) }
+    it "should not raise exception" do
+      expect { subject.register }.to_not raise_exception
+    end
+  end
+  context "with :ssl => true" do
+    context "without :ssl_certificate" do
+      subject { LogStash::Inputs::Http.new("port" => port, "ssl" => true) }
+      it "should raise exception" do
+        expect { subject.register }.to raise_exception(LogStash::ConfigurationError)
+      end
+    end
+    context "with :ssl_certificate" do
+      let(:ssl_certificate) { Stud::Temporary.file }
+      let(:ssl_key) { Stud::Temporary.file }
+      subject { LogStash::Inputs::Http.new("port" => port, "ssl" => true,
+                                           "ssl_certificate" => ssl_certificate.path,
+                                           "ssl_key" => ssl_key.path) }
+      it "should not raise exception" do
+        expect { subject.register }.to_not raise_exception
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,173 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-http
+version: !ruby/object:Gem::Version
+  version: 3.1.0
+platform: java
+authors:
+- Elastic
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-04-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.4
+  name: jar-dependencies
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.4
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json_lines
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: manticore
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: info@elastic.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- VERSION
+- docs/index.asciidoc
+- lib/logstash-input-http_jars.rb
+- lib/logstash/inputs/http.rb
+- lib/logstash/inputs/http/message_handler.rb
+- lib/logstash/inputs/http/tls.rb
+- lib/logstash/util/http_compressed_requests.rb
+- lib/tasks/build.rake
+- logstash-input-http.gemspec
+- spec/inputs/http_spec.rb
+- vendor/jar-dependencies/io/netty/netty-all/4.1.18.Final/netty-all-4.1.18.Final.jar
+- vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/2.0.7.Final/netty-tcnative-boringssl-static-2.0.7.Final.jar
+- vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
+- vendor/jar-dependencies/org/logstash/plugins/input/http/logstash-input-http/3.1.0/logstash-input-http-3.1.0.jar
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+- vendor/jar-dependencies
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.11
+signing_key:
+specification_version: 4
+summary: Receives events over HTTP or HTTPS
+test_files:
+- spec/inputs/http_spec.rb