logstash-input-http 3.1.0-java → 3.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 445d3da2f390e8891b32e2090945f28677d6257018545983859c26665c21af84
-  data.tar.gz: 585afff812773b166fd44370764bd0837958218dd994335da024216faf2cd352
+  metadata.gz: dd4ab63a8da778ab6b98669f405773f04defba698bef77a3430d0ea4eb778cf0
+  data.tar.gz: 8bc8d4d7909fa768424d63142198644bb97e6b8ea44bef779288267d48044052
 SHA512:
-  metadata.gz: a9915f4e38147a0b97c2376b9fda49412b9c2aa995d2a6d5f4b5153a957b54baaa045375a0cbc8ef53d32cd3b1f97731729b73aee3b5cd34dbda4dfa468afc0d
-  data.tar.gz: cf4d9aa38eda357bf7bba97c6f696221eaf2d09f990eb36ff9e1857c26f6fb96118905fc0e4aff16ce583476af9e095d9a0121e15884b7d34449e1d99cb36c1e
+  metadata.gz: 91ee9ad30eaf383efbf38b2f8a6720a3a9bba045ed7dda602734600dfa63b6d7c532bbee12b078a6c1444632e7edb3afe19569d72e9f2f4e94405092849d4a6c
+  data.tar.gz: b09f089df723d4240a6a89d8c78cb46c54a914784d8a15491604be7b31238c4f668044ea15f3cceb74e557f70678783ff0b4c9444ba8d61c56d55f0f8bd844c8

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.2.0
+  - Add `request_headers_target_field` and `remote_host_target_field` configuration options with default to `host` and `headers` respectively #68
+  - Sanitize content-type header with getMimeType #87
+  - Move most message handling code to java #85
+  - Fix: respond with correct http protocol version #84
+
 ## 3.1.0
   - Replace Puma web server with Netty
   - Support crt/key certificates

data/VERSION

@@ -1 +1 @@
-3.1.0
+3.2.0

data/docs/index.asciidoc

@@ -184,6 +184,22 @@ Persistent Queue for the logstash pipeline.
 
 specify a custom set of response headers
 
+[id="plugins-{type}s-{plugin}-remote_host_target_field"]
+===== `remote_host_target_field` 
+
+  * Value type is <<string,string>>
+  * Default value is `"host"`
+
+specify a target field for the client host of the http request
+
+[id="plugins-{type}s-{plugin}-request_headers_target_field"]
+===== `request_headers_target_field` 
+
+  * Value type is <<string,string>>
+  * Default value is `"headers"`
+
+specify target field for the client host of the http request
+
 [id="plugins-{type}s-{plugin}-ssl"]
 ===== `ssl` 
 

data/lib/logstash-input-http_jars.rb

@@ -4,4 +4,4 @@ require 'jar_dependencies'
 require_jar('io.netty', 'netty-all', '4.1.18.Final')
 require_jar('io.netty', 'netty-tcnative-boringssl-static', '2.0.7.Final')
 require_jar('org.apache.logging.log4j', 'log4j-api', '2.6.2')
-require_jar('org.logstash.plugins.input.http', 'logstash-input-http', '3.1.0')
+require_jar('org.logstash.plugins.input.http', 'logstash-input-http', '3.2.0')

data/lib/logstash/inputs/http.rb

@@ -4,6 +4,8 @@ require "logstash/namespace"
 require "stud/interval"
 require "logstash-input-http_jars"
 
+java_import "io.netty.handler.codec.http.HttpUtil"
+
 # Using this input you can receive single or multiline events over http(s).
 # Applications can send a HTTP POST request with a body to the endpoint started by this
 # input and Logstash will convert it into an event for subsequent processing. Users 
@@ -101,6 +103,12 @@ class LogStash::Inputs::Http < LogStash::Inputs::Base
   # specify a custom set of response headers
   config :response_headers, :validate => :hash, :default => { 'Content-Type' => 'text/plain' }
 
+  # target field for the client host of the http request
+  config :remote_host_target_field, :validate => :string, :default => "host"
+
+  # target field for the client host of the http request
+  config :request_headers_target_field, :validate => :string, :default => "headers"
+
   config :threads, :validate => :number, :required => false, :default => ::LogStash::Config::CpuCoreStrategy.maximum
 
   config :max_pending_requests, :validate => :number, :required => false, :default => 200
@@ -133,7 +141,7 @@ class LogStash::Inputs::Http < LogStash::Inputs::Base
     end
 
     require "logstash/inputs/http/message_handler"
-    message_handler = MessageHandler.new(self, @codec, @codecs)
+    message_handler = MessageHandler.new(self, @codec, @codecs, @auth_token)
     @http_server = create_http_server(message_handler)
   end # def register
 
@@ -151,23 +159,12 @@ class LogStash::Inputs::Http < LogStash::Inputs::Base
     @http_server.close() rescue nil
   end
 
-  def decode_body(remote_address, http_full_request, default_codec, additional_codecs)
-    body = read_bytes(http_full_request.content)
-    headers = {}
-
-    http_full_request.headers.each do |header|
-      headers[header.key.downcase.tr('-', '_')] = header.value
-    end
-    headers["request_method"] = http_full_request.getMethod.name
-    headers["request_path"] = http_full_request.getUri
-    headers["http_version"] = http_full_request.getProtocolVersion.text
-    headers["http_accept"] = headers.delete("accept")
-    headers["http_host"] = headers.delete("host")
-    headers["http_user_agent"] = headers.delete("user_agent")
-    codec = additional_codecs.fetch(headers["content_type"], default_codec)
+  def decode_body(headers, remote_address, body, default_codec, additional_codecs)
+    content_type = headers.fetch("content_type", "")
+    codec = additional_codecs.fetch(HttpUtil.getMimeType(content_type), default_codec)
     codec.decode(body) { |event| push_decoded_event(headers, remote_address, event) }
     codec.flush { |event| push_decoded_event(headers, remote_address, event) }
-    [200, @response_headers, 'ok']
+    true
   rescue => e
     @logger.error(
       "unable to process event.",
@@ -175,31 +172,16 @@ class LogStash::Inputs::Http < LogStash::Inputs::Base
       :class => e.class.name,
       :backtrace => e.backtrace
     )
-    [500, @response_headers, 'internal error']
+    false
   end
 
   def push_decoded_event(headers, remote_address, event)
-    event.set("headers", headers)
-    event.set("host", remote_address)
+    event.set(@request_headers_target_field, headers)
+    event.set(@remote_host_target_field, remote_address)
     decorate(event)
     @queue << event
   end
 
-  def valid_auth?(token)
-    if @auth_token
-      @auth_token == token
-    else
-      true
-    end
-  end
-
-  private
-  def read_bytes(bytebuffer)
-    bytes = Java::byte[bytebuffer.readableBytes].new
-    bytebuffer.getBytes(0, bytes)
-    String.from_java_bytes(bytes, "ASCII-8BIT")
-  end
-
   def validate_ssl_settings!
     if !@ssl
       @logger.warn("SSL Certificate will not be used") if @ssl_certificate

data/lib/logstash/inputs/http/message_handler.rb

@@ -14,37 +14,27 @@ module LogStash module Inputs class Http
 
     attr_reader :input
 
-    def initialize(input, default_codec, additional_codecs)
+    def initialize(input, default_codec, additional_codecs, auth_token)
       @input = input
       @default_codec = default_codec
       @additional_codecs = additional_codecs
+      @auth_token = auth_token
     end
 
-    def onNewMessage(remote_address, message)
-      if valid_auth?(message)
-        message.headers.remove(HttpHeaderNames::AUTHORIZATION)
-        status, headers, content = @input.decode_body(remote_address, message, @default_codec, @additional_codecs)
+    def validates_token(token)
+      if @auth_token
+        @auth_token == token
       else
-        status, headers, content = 401, {}, 'failed to authenticate'
+        true
       end
-      generate_response(status, headers, content)
     end
 
-    private
-    def generate_response(status, headers, content)
-      payload = Unpooled.copiedBuffer(content.to_java_string, CharsetUtil::UTF_8)
-      response = DefaultFullHttpResponse.new(
-        HttpVersion::HTTP_1_1,
-        HttpResponseStatus.valueOf(status),
-        payload)
-      response.headers().set(HttpHeaderNames::CONTENT_LENGTH, payload.readable_bytes());
-      response.headers().set(HttpHeaderNames::CONTENT_TYPE, "text/plain");
-      headers.each { |k, v| response.headers().set(k, v) }
-      response
+    def onNewMessage(remote_address, headers, body)
+      @input.decode_body(headers, remote_address, body, @default_codec, @additional_codecs)
     end
 
     def copy
-      MessageHandler.new(@input, @default_codec.clone, clone_additional_codecs())
+      MessageHandler.new(@input, @default_codec.clone, clone_additional_codecs(), @auth_token)
     end
 
     def clone_additional_codecs
@@ -55,8 +45,8 @@ module LogStash module Inputs class Http
       clone_additional_codecs
     end
 
-    def valid_auth?(message)
-      @input.valid_auth?(message.headers.get(HttpHeaderNames::AUTHORIZATION))
+    def response_headers
+      @input.response_headers
     end
   end
 end; end; end

data/spec/inputs/http_spec.rb

@@ -29,6 +29,7 @@ describe LogStash::Inputs::Http do
 
   describe "request handling" do
     subject { LogStash::Inputs::Http.new("port" => port) }
+
     before :each do
       subject.register
       t = Thread.new { subject.run(logstash_queue) }
@@ -81,6 +82,57 @@ describe LogStash::Inputs::Http do
       end
     end
 
+    describe "remote host" do
+      subject { LogStash::Inputs::Http.new(config.merge("port" => port)) }
+      context "by default" do
+        let(:config) { {} }
+        it "is written to the \"host\" field" do
+          client.post("http://localhost:#{port}/meh.json",
+                      :headers => { "content-type" => "text/plain" },
+                      :body => "hello").call
+          event = logstash_queue.pop
+          expect(event.get("host")).to eq("127.0.0.1")
+        end
+      end
+
+      context "when using remote_host_target_field" do
+        let(:config) { { "remote_host_target_field" => "remote_host" } }
+        it "is written to the value of \"remote_host_target_field\" property" do
+          client.post("http://localhost:#{port}/meh.json",
+                      :headers => { "content-type" => "text/plain" },
+                      :body => "hello").call
+          event = logstash_queue.pop
+          expect(event.get("remote_host")).to eq("127.0.0.1")
+        end
+      end
+    end
+
+    describe "request headers" do
+      subject { LogStash::Inputs::Http.new(config.merge("port" => port)) }
+      context "by default" do
+        let(:config) { {} }
+        it "are written to the \"headers\" field" do
+          client.post("http://localhost:#{port}/meh.json",
+                      :headers => { "content-type" => "text/plain" },
+                      :body => "hello").call
+          event = logstash_queue.pop
+          expect(event.get("headers")).to be_a(Hash)
+          expect(event.get("headers")).to include("request_method" => "POST")
+        end
+      end
+      context "when using request_headers_target_field" do
+        let(:config) { { "request_headers_target_field" => "request_headers" } }
+        it "are written to the field set in \"request_headers_target_field\"" do
+          client.post("http://localhost:#{port}/meh.json",
+                      :headers => { "content-type" => "text/plain" },
+                      :body => "hello").call
+          event = logstash_queue.pop
+          expect(event.get("request_headers")).to be_a(Hash)
+          expect(event.get("request_headers")).to include("request_method" => "POST")
+        end
+      end
+    end
+
     it "should include remote host in \"host\" property" do
       client.post("http://127.0.0.1:#{port}/meh.json",
                   :headers => { "content-type" => "text/plain" },
@@ -190,6 +242,19 @@ describe LogStash::Inputs::Http do
         expect(event.get("message")).to eq(body)
       end
     end
+    
+    context "when receiving a content-type with a charset" do
+      subject { LogStash::Inputs::Http.new("port" => port,
+                                           "additional_codecs" => { "application/json" => "plain" }) }
+      it "should decode the message accordingly" do
+        body = { "message" => "Hello" }.to_json
+        client.post("http://127.0.0.1:#{port}/meh.json",
+                    :headers => { "content-type" => "application/json; charset=utf-8" },
+                      :body => body).call
+        event = logstash_queue.pop
+        expect(event.get("message")).to eq(body)
+      end
+    end
 
     context "when using custom headers" do
       let(:custom_headers) { { 'access-control-allow-origin' => '*' } }
@@ -197,7 +262,7 @@ describe LogStash::Inputs::Http do
 
       describe "the response" do
         it "should include the custom headers" do
-          response = client.post("http://127.0.0.1:#{port}/meh", :body => "hello")
+          response = client.post("http://127.0.0.1:#{port}/meh", :body => "hello").call
           expect(response.headers.to_hash).to include(custom_headers)
         end
       end
@@ -248,6 +313,32 @@ describe LogStash::Inputs::Http do
       end
     end
 
+    describe "HTTP Protocol Handling" do
+      context "when an HTTP1.1 request is made" do
+        let(:protocol_version) do
+          Java::OrgApacheHttp::HttpVersion::HTTP_1_1
+        end
+        it "responds with a HTTP1.1 response" do
+          response = client.post("http://127.0.0.1:#{port}", :body => "hello")
+          response.request.set_protocol_version(protocol_version)
+          response.call
+          response_protocol_version = response.instance_variable_get(:@response).get_protocol_version
+          expect(response_protocol_version).to eq(protocol_version)
+        end
+      end
+      context "when an HTTP1.0 request is made" do
+        let(:protocol_version) do
+          Java::OrgApacheHttp::HttpVersion::HTTP_1_0
+        end
+        it "responds with a HTTP1.0 response" do
+          response = client.post("http://127.0.0.1:#{port}", :body => "hello")
+          response.request.set_protocol_version(protocol_version)
+          response.call
+          response_protocol_version = response.instance_variable_get(:@response).get_protocol_version
+          expect(response_protocol_version).to eq(protocol_version)
+        end
+      end
+    end
   end
 
   context "with :ssl => false" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-http
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-26 00:00:00.000000000 Z
+date: 2018-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -141,7 +141,7 @@ files:
 - vendor/jar-dependencies/io/netty/netty-all/4.1.18.Final/netty-all-4.1.18.Final.jar
 - vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/2.0.7.Final/netty-tcnative-boringssl-static-2.0.7.Final.jar
 - vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
-- vendor/jar-dependencies/org/logstash/plugins/input/http/logstash-input-http/3.1.0/logstash-input-http-3.1.0.jar
+- vendor/jar-dependencies/org/logstash/plugins/input/http/logstash-input-http/3.2.0/logstash-input-http-3.2.0.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -165,7 +165,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Receives events over HTTP or HTTPS