fluentd-hubspot 0.14.14.1

data/lib/fluent/plugin/in_gc_stat.rb

@@ -0,0 +1,56 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+
+module Fluent::Plugin
+  class GCStatInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('gc_stat', self)
+
+    helpers :timer
+
+    def initialize
+      super
+    end
+
+    config_param :emit_interval, :time, default: 60
+    config_param :tag, :string
+
+    def configure(conf)
+      super
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      timer_execute(:in_gc_stat, @emit_interval, &method(:on_timer))
+    end
+
+    def shutdown
+      super
+    end
+
+    def on_timer
+      now = Fluent::EventTime.now
+      record = GC.stat
+      router.emit(@tag, now, record)
+    end
+  end
+end

data/lib/fluent/plugin/in_http.rb

@@ -0,0 +1,433 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+require 'fluent/plugin/parser'
+require 'fluent/event'
+
+require 'http/parser'
+require 'webrick/httputils'
+require 'uri'
+require 'socket'
+require 'json'
+
+module Fluent::Plugin
+  class InHttpParser < Parser
+    Fluent::Plugin.register_parser('in_http', self)
+    def parse(text)
+      # this plugin is dummy implementation not to raise error
+      yield nil, nil
+    end
+  end
+
+  class HttpInput < Input
+    Fluent::Plugin.register_input('http', self)
+
+    # TODO: update this plugin implementation to use server plugin helper, after adding keepalive feature on it
+
+    helpers :parser, :compat_parameters, :event_loop
+
+    EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")
+
+    desc 'The port to listen to.'
+    config_param :port, :integer, default: 9880
+    desc 'The bind address to listen to.'
+    config_param :bind, :string, default: '0.0.0.0'
+    desc 'The size limit of the POSTed element. Default is 32MB.'
+    config_param :body_size_limit, :size, default: 32*1024*1024  # TODO default
+    desc 'The timeout limit for keeping the connection alive.'
+    config_param :keepalive_timeout, :time, default: 10   # TODO default
+    config_param :backlog, :integer, default: nil
+    desc 'Add HTTP_ prefix headers to the record.'
+    config_param :add_http_headers, :bool, default: false
+    desc 'Add REMOTE_ADDR header to the record.'
+    config_param :add_remote_addr, :bool, default: false
+    config_param :blocking_timeout, :time, default: 0.5
+    desc 'Set a white list of domains that can do CORS (Cross-Origin Resource Sharing)'
+    config_param :cors_allow_origins, :array, default: nil
+    desc 'Respond with empty gif image of 1x1 pixel.'
+    config_param :respond_with_empty_img, :bool, default: false
+
+    config_section :parse do
+      config_set_default :@type, 'in_http'
+    end
+
+    EVENT_RECORD_PARAMETER = '_event_record'
+
+    def configure(conf)
+      compat_parameters_convert(conf, :parser)
+
+      super
+
+      m = if @parser_configs.first['@type'] == 'in_http'
+            @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
+            @parser_msgpack.estimate_current_event = false
+            @parser_json = parser_create(usage: 'parser_in_http_json', type: 'json')
+            @parser_json.estimate_current_event = false
+            @format_name = 'default'
+            method(:parse_params_default)
+          else
+            @parser = parser_create
+            @format_name = @parser_configs.first['@type']
+            method(:parse_params_with_parser)
+          end
+      self.singleton_class.module_eval do
+        define_method(:parse_params, m)
+      end
+    end
+
+    class KeepaliveManager < Coolio::TimerWatcher
+      def initialize(timeout)
+        super(1, true)
+        @cons = {}
+        @timeout = timeout.to_i
+      end
+
+      def add(sock)
+        @cons[sock] = sock
+      end
+
+      def delete(sock)
+        @cons.delete(sock)
+      end
+
+      def on_timer
+        @cons.each_pair {|sock,val|
+          if sock.step_idle > @timeout
+            sock.close
+          end
+        }
+      end
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      @_event_loop_run_timeout = @blocking_timeout
+
+      super
+
+      log.debug "listening http", bind: @bind, port: @port
+
+      socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
+      if Fluent.windows?
+        socket_manager_path = socket_manager_path.to_i
+      end
+      client = ServerEngine::SocketManager::Client.new(socket_manager_path)
+      lsock = client.listen_tcp(@bind, @port)
+
+      @km = KeepaliveManager.new(@keepalive_timeout)
+      @lsock = Coolio::TCPServer.new(
+        lsock, nil, Handler, @km, method(:on_request),
+        @body_size_limit, @format_name, log,
+        @cors_allow_origins
+      )
+      @lsock.listen(@backlog) unless @backlog.nil?
+      event_loop_attach(@km)
+      event_loop_attach(@lsock)
+
+      @float_time_parser = Fluent::NumericTimeParser.new(:float)
+    end
+
+    def close
+      @lsock.close
+      super
+    end
+
+    def on_request(path_info, params)
+      begin
+        path = path_info[1..-1]  # remove /
+        tag = path.split('/').join('.')
+        record_time, record = parse_params(params)
+
+        # Skip nil record
+        if record.nil?
+          if @respond_with_empty_img
+            return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+          else
+            return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+          end
+        end
+
+        unless record.is_a?(Array)
+          if @add_http_headers
+            params.each_pair { |k,v|
+              if k.start_with?("HTTP_")
+                record[k] = v
+              end
+            }
+          end
+          if @add_remote_addr
+            record['REMOTE_ADDR'] = params['REMOTE_ADDR']
+          end
+        end
+        time = if param_time = params['time']
+                 param_time = param_time.to_f
+                 param_time.zero? ? Fluent::Engine.now : @float_time_parser.parse(param_time)
+               else
+                 record_time.nil? ? Fluent::Engine.now : record_time
+               end
+      rescue
+        return ["400 Bad Request", {'Content-Type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
+      end
+
+      # TODO server error
+      begin
+        # Support batched requests
+        if record.is_a?(Array)
+          mes = Fluent::MultiEventStream.new
+          record.each do |single_record|
+            if @add_http_headers
+              params.each_pair { |k,v|
+                if k.start_with?("HTTP_")
+                  single_record[k] = v
+                end
+              }
+            end
+            if @add_remote_addr
+              single_record['REMOTE_ADDR'] = params['REMOTE_ADDR']
+            end
+            single_time = single_record.delete("time") || time
+            mes.add(single_time, single_record)
+          end
+          router.emit_stream(tag, mes)
+        else
+          router.emit(tag, time, record)
+        end
+      rescue
+        return ["500 Internal Server Error", {'Content-Type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
+      end
+
+      if @respond_with_empty_img
+        return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+      else
+        return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+      end
+    end
+
+    private
+
+    def parse_params_default(params)
+      if msgpack = params['msgpack']
+        @parser_msgpack.parse(msgpack) do |_time, record|
+          return nil, record
+        end
+      elsif js = params['json']
+        @parser_json.parse(js) do |_time, record|
+          return nil, record
+        end
+      else
+        raise "'json' or 'msgpack' parameter is required"
+      end
+    end
+
+    def parse_params_with_parser(params)
+      if content = params[EVENT_RECORD_PARAMETER]
+        @parser.parse(content) { |time, record|
+          raise "Received event is not #{@format_name}: #{content}" if record.nil?
+          return time, record
+        }
+      else
+        raise "'#{EVENT_RECORD_PARAMETER}' parameter is required"
+      end
+    end
+
+    class Handler < Coolio::Socket
+      attr_reader :content_type
+
+      def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins)
+        super(io)
+        @km = km
+        @callback = callback
+        @body_size_limit = body_size_limit
+        @next_close = false
+        @format_name = format_name
+        @log = log
+        @cors_allow_origins = cors_allow_origins
+        @idle = 0
+        @km.add(self)
+
+        @remote_port, @remote_addr = *Socket.unpack_sockaddr_in(io.getpeername) rescue nil
+      end
+
+      def step_idle
+        @idle += 1
+      end
+
+      def on_close
+        @km.delete(self)
+      end
+
+      def on_connect
+        @parser = Http::Parser.new(self)
+      end
+
+      def on_read(data)
+        @idle = 0
+        @parser << data
+      rescue
+        @log.warn "unexpected error", error: $!.to_s
+        @log.warn_backtrace
+        close
+      end
+
+      def on_message_begin
+        @body = ''
+      end
+
+      def on_headers_complete(headers)
+        expect = nil
+        size = nil
+
+        if @parser.http_version == [1, 1]
+          @keep_alive = true
+        else
+          @keep_alive = false
+        end
+        @env = {}
+        @content_type = ""
+        headers.each_pair {|k,v|
+          @env["HTTP_#{k.gsub('-','_').upcase}"] = v
+          case k
+          when /Expect/i
+            expect = v
+          when /Content-Length/i
+            size = v.to_i
+          when /Content-Type/i
+            @content_type = v
+          when /Connection/i
+            if v =~ /close/i
+              @keep_alive = false
+            elsif v =~ /Keep-alive/i
+              @keep_alive = true
+            end
+          when /Origin/i
+            @origin  = v
+          when /X-Forwarded-For/i
+            @remote_addr = v.split(",").first
+          end
+        }
+        if expect
+          if expect == '100-continue'
+            if !size || size < @body_size_limit
+              send_response_nobody("100 Continue", {})
+            else
+              send_response_and_close("413 Request Entity Too Large", {}, "Too large")
+            end
+          else
+            send_response_and_close("417 Expectation Failed", {}, "")
+          end
+        end
+      end
+
+      def on_body(chunk)
+        if @body.bytesize + chunk.bytesize > @body_size_limit
+          unless closing?
+            send_response_and_close("413 Request Entity Too Large", {}, "Too large")
+          end
+          return
+        end
+        @body << chunk
+      end
+
+      def on_message_complete
+        return if closing?
+
+        # CORS check
+        # ==========
+        # For every incoming request, we check if we have some CORS
+        # restrictions and white listed origins through @cors_allow_origins.
+        unless @cors_allow_origins.nil?
+          unless @cors_allow_origins.include?(@origin)
+            send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
+            return
+          end
+        end
+
+        @env['REMOTE_ADDR'] = @remote_addr if @remote_addr
+
+        uri = URI.parse(@parser.request_url)
+        params = WEBrick::HTTPUtils.parse_query(uri.query)
+
+        if @format_name != 'default'
+          params[EVENT_RECORD_PARAMETER] = @body
+        elsif @content_type =~ /^application\/x-www-form-urlencoded/
+          params.update WEBrick::HTTPUtils.parse_query(@body)
+        elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
+          boundary = WEBrick::HTTPUtils.dequote($1)
+          params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
+        elsif @content_type =~ /^application\/json/
+          params['json'] = @body
+        elsif @content_type =~ /^application\/msgpack/
+          params['msgpack'] = @body
+        end
+        path_info = uri.path
+
+        params.merge!(@env)
+        @env.clear
+
+        code, header, body = *@callback.call(path_info, params)
+        body = body.to_s
+
+        header['Access-Control-Allow-Origin'] = @origin if !@cors_allow_origins.nil? && @cors_allow_origins.include?(@origin)
+        if @keep_alive
+          header['Connection'] = 'Keep-Alive'
+          send_response(code, header, body)
+        else
+          send_response_and_close(code, header, body)
+        end
+      end
+
+      def on_write_complete
+        close if @next_close
+      end
+
+      def send_response_and_close(code, header, body)
+        send_response(code, header, body)
+        @next_close = true
+      end
+
+      def closing?
+        @next_close
+      end
+
+      def send_response(code, header, body)
+        header['Content-Length'] ||= body.bytesize
+        header['Content-Type'] ||= 'text/plain'
+
+        data = %[HTTP/1.1 #{code}\r\n]
+        header.each_pair {|k,v|
+          data << "#{k}: #{v}\r\n"
+        }
+        data << "\r\n"
+        write data
+
+        write body
+      end
+
+      def send_response_nobody(code, header)
+        data = %[HTTP/1.1 #{code}\r\n]
+        header.each_pair {|k,v|
+          data << "#{k}: #{v}\r\n"
+        }
+        data << "\r\n"
+        write data
+      end
+    end
+  end
+end