fluent-plugin-cwm-http 0.3.1

data/fluent-plugin-cwm-http.gemspec

@@ -0,0 +1,30 @@
+# frozen-string-literal: true
+
+lib = File.expand_path('../lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name           = 'fluent-plugin-cwm-http'
+  spec.version        = '0.3.1'
+  spec.authors        = ['Azeem Sajid']
+  spec.email          = ['azeem.sajid@gmail.com']
+
+  spec.summary        = 'fluentd HTTP Input Plugin for CloudWebManage Logging Component'
+  spec.description    = 'fluentd HTTP Input Plugin for CloudWebManage Logging Component with Log Metrics Support'
+  spec.homepage       = 'https://github.com/CloudWebManage/fluent-plugin-http-cwm'
+  spec.license        = 'Apache-2.0'
+
+  test_files, files   = `git ls-files -z`.split("\x0").partition { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.files          = files
+  spec.executables    = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files     = test_files
+  spec.require_paths  = ['lib']
+
+  spec.add_development_dependency 'bundler', '>= 1.14', '< 3'
+  spec.add_development_dependency 'rake', '~> 12.3', '>= 12.3.3'
+  spec.add_development_dependency 'simplecov', '~> 0.12', '<= 0.12.2'
+  spec.add_development_dependency 'test-unit', '~> 3.0'
+  spec.add_runtime_dependency 'fluentd', '>= 0.14.10', '< 2'
+  spec.add_runtime_dependency 'json', '~> 2.3', '>= 2.3.0'
+  spec.add_runtime_dependency 'redis', '~> 4.2', '>= 4.2.5'
+end

data/lib/fluent/plugin/in_http_cwm.rb

@@ -0,0 +1,299 @@
+# frozen-string-literal: true
+
+#
+# Copyright 2020 Azeem Sajid
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/input'
+require 'fluent/config/error'
+require 'fluent/plugin_helper/http_server'
+require 'fluent/plugin_helper/timer'
+require 'webrick/httputils'
+require 'json'
+require 'redis'
+
+FMT_DATETIME = '%Y-%m-%dT%H:%M:%S.%8NZ'
+
+module Fluent
+  module Plugin
+    # Custom HTTP Input Plugin class for CWM
+    class CwmHttpInput < Fluent::Plugin::Input
+      Fluent::Plugin.register_input('http_cwm', self)
+
+      helpers :http_server, :event_emitter, :timer
+
+      desc 'The address to bind to.'
+      config_param :host, :string, default: 'localhost'
+
+      desc 'The port to listen to.'
+      config_param :port, :integer, default: 8080
+
+      desc 'The tag for the event.'
+      config_param :tag, :string
+
+      desc 'The Redis configuration section for flushing metrics.'
+      config_section :redis, required: false, multi: false, init: true, param_name: :redis_config do
+        desc 'The address of Redis server.'
+        config_param :host, :string, default: 'localhost'
+
+        desc 'The port of Redis server.'
+        config_param :port, :integer, default: 6379
+
+        desc 'The db to use.'
+        config_param :db, :integer, default: 0
+
+        desc 'The grace period for last action update.'
+        config_param :grace_period, :time, default: '300s'
+
+        desc 'The flush interval to send metrics.'
+        config_param :flush_interval, :time, default: '300s'
+
+        desc 'The prefix for last update key.'
+        config_param :last_update_prefix, :string, default: 'deploymentid:last_action'
+
+        desc 'The prefix for metrics key.'
+        config_param :metrics_prefix, :string, default: 'deploymentid:minio-metrics'
+      end
+
+      def initialize
+        super
+
+        @redis = nil
+        @deployment_api_metrics = default_api_metrics_hash
+
+        @last_action_queue = Queue.new
+        @last_action_entry = []
+      end
+
+      def configure(conf)
+        super
+
+        set_up_redis
+      end
+
+      def start
+        super
+
+        # start interval timer to flush api metrics
+        timer_execute(:api_metrics_flush_timer, @redis_config.flush_interval) do
+          flush_api_metrics
+        end
+
+        # start interval timer to flush last action entry
+        timer_execute(:last_action_flush_timer, '1s') do
+          flush_last_action
+        end
+
+        log.info("Starting HTTP server [#{@host}:#{@port}]...")
+        http_server_create_http_server(:http_server, addr: @host, port: @port, logger: log) do |server|
+          server.post("/#{tag}") do |req|
+            data = parse_data(req.body)
+            route(data) if update_deployment_metrics(data)
+
+            # return HTTP 200 OK response with emtpy body
+            [200, { 'Content-Type' => 'text/plain' }, nil]
+          end
+          server.get("/health") do |req|
+            [200, { 'Content-Type' => 'application/json' }, {}.to_json]
+          end
+        end
+      end
+
+      private
+
+      def default_api_metrics_hash
+        Hash.new do |h, k|
+          h[k] = {
+            'bytes_in' => 0, 'bytes_out' => 0,
+            'num_requests_in' => 0, 'num_requests_out' => 0, 'num_requests_misc' => 0
+          }
+        end
+      end
+
+      def set_up_redis
+        host = @redis_config.host
+        port = @redis_config.port
+        db = @redis_config.db
+        log.info("Connecting with Redis [address: #{host}:#{port}, db: #{db}]")
+        @redis = Redis.new(host: host, port: port, db: db)
+        ready = false
+        until ready
+          sleep(1)
+          begin
+            @redis.ping
+            ready = true
+          rescue StandardError => e
+            log.error("Unable to connect to Redis server! ERROR: '#{e}'. Retrying...")
+          end
+        end
+      end
+
+      def parse_data(data)
+        JSON.parse(data)
+      rescue StandardError => e
+        log.debug("ERROR: #{e}")
+        nil
+      end
+
+      def route(data)
+        time = Fluent::Engine.now
+        record = { 'message' => data }
+        router.emit(@tag, time, record)
+      end
+
+      def flush_last_action
+        if @last_action_entry.empty?
+          @last_action_entry = @last_action_queue.deq.split('|')
+          log.debug("Dequed last action entry. #{@last_action_entry}")
+        else
+          deploymentid, last_action = @last_action_entry
+          @last_action_entry = [] if update_deployment_last_action(deploymentid, last_action)
+        end
+      end
+
+      def datetime_diff_in_secs(dt_begin, dt_end)
+        seconds = ((dt_end - dt_begin) * 24 * 60 * 60)
+        seconds.to_i
+      end
+
+      def update_deployment_last_action(deploymentid, last_action)
+        key = "#{@redis_config.last_update_prefix}:#{deploymentid}"
+        log.debug("Checking existing last action entry [key: #{key}]")
+        lastval = @redis.get(key)
+
+        is_grace_period_expired = false
+        if lastval
+          curdt = DateTime.now
+          lastdt = DateTime.parse(lastval, FMT_DATETIME)
+          dt_diff_secs = datetime_diff_in_secs(lastdt, curdt)
+          log.debug("Current  Data/Time: #{curdt}")
+          log.debug("Previous Date/Time: #{lastdt}")
+          log.debug("Date/Time diff (s): #{dt_diff_secs}")
+
+          if dt_diff_secs >= @redis_config.grace_period
+            is_grace_period_expired = true
+            log.debug("Grace period expired for last action update. [#{@redis_config.grace_period}]")
+          end
+        else
+          log.debug('Last action entry does not exist. It will be set for the first time.')
+        end
+
+        if lastdt.nil? || is_grace_period_expired
+          log.debug('Updating deployment last action')
+          last_action = DateTime.parse(last_action, FMT_DATETIME)
+          @redis.set(key, last_action)
+          log.debug("Updated last action entry [#{key} => #{last_action}]")
+          true
+        else
+          false
+        end
+      rescue StandardError => e
+        log.error("Unable to update last action! ERROR: '#{e}'.")
+        false
+      end
+
+      def enque_last_action_entry(deploymentid)
+        last_action = DateTime.now
+        entry = "#{deploymentid}|#{last_action}"
+        @last_action_queue.enq(entry)
+        log.debug("Enqued last action entry. [#{entry}]")
+      end
+
+      def validate_and_get_value(data_hash, key)
+        value = data_hash[key]
+        log.debug("missing '#{key}': #{data_hash.to_json}") unless value
+        value
+      end
+
+      def update_deployment_metrics(data)
+        return false unless data
+
+        log.debug('Updating deployment metrics')
+
+        deploymentid = validate_and_get_value(data, 'deploymentid')
+        return false unless deploymentid
+
+        enque_last_action_entry(deploymentid)
+
+        api_data = validate_and_get_value(data, 'api')
+        return false unless api_data
+
+        api_name = validate_and_get_value(api_data, 'name')
+        return false unless api_name
+
+        response_header_data = validate_and_get_value(data, 'responseHeader')
+        return false unless response_header_data
+
+        request_header_data = validate_and_get_value(data, 'requestHeader')
+        return false unless request_header_data
+
+        response_content_length = response_header_data['Content-Length'].to_i
+        response_content_length += response_header_data.to_s.length
+
+        response_is_cached = (response_header_data['X-Cache'] == 'HIT')
+
+        request_content_length = request_header_data['Content-Length'].to_i
+        request_content_length += request_header_data.to_s.length
+
+        update_deployment_api_metrics(deploymentid, api_name, request_content_length, response_content_length, response_is_cached)
+
+        true
+      end
+
+      def get_request_type(api_name)
+        in_apis = %w[WebUpload PutObject DeleteObject].freeze
+        out_apis = %w[WebDownload GetObject].freeze
+        return 'in' if in_apis.include?(api_name)
+        return 'out' if out_apis.include?(api_name)
+
+        'misc'
+      end
+
+      def update_deployment_api_metrics(deploymentid, api_name, request_content_length, response_content_length, response_is_cached)
+        log.debug('Updating deployment API metrics')
+
+        request_type = get_request_type(api_name)
+        log.debug("#{deploymentid}.#{api_name}: (type=#{request_type}, req_size=#{request_content_length}, res_size=#{response_content_length}, res_cache=#{response_is_cached})")
+
+        metrics = @deployment_api_metrics[deploymentid]
+        metrics['bytes_in'] += request_content_length
+        metrics['bytes_out'] += response_content_length
+        metrics["num_requests_#{request_type}"] += 1
+        @deployment_api_metrics[deploymentid] = metrics
+      end
+
+      def flush_api_metrics
+        return if @deployment_api_metrics.empty?
+
+        log.debug("Flushing metrics: #{@deployment_api_metrics}")
+
+        begin
+          @redis.pipelined do
+            @deployment_api_metrics.each do |deploymentid, metrics|
+              metrics.each do |metric, value|
+                @redis.incrby("#{@redis_config.metrics_prefix}:#{deploymentid}:#{metric}", value) if value.positive?
+              end
+            end
+          end
+
+          @deployment_api_metrics = default_api_metrics_hash
+
+          log.debug('Flushing complete!')
+        rescue StandardError => e
+          log.error("Unable to flush metrics! ERROR: '#{e}'.")
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,13 @@
+# frozen-string-literal: true
+
+require 'simplecov'
+SimpleCov.start
+
+$LOAD_PATH.unshift(File.expand_path('..', __dir__))
+require 'test-unit'
+require 'fluent/test'
+require 'fluent/test/driver/input'
+require 'fluent/test/helpers'
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/logs.txt

@@ -0,0 +1,29 @@
+
+{}
+{"version":"1","deploymentid":"docker-compose-https","time":"2020-12-13T09:59:53.452955009Z","api":{"name":"WebUpload","bucket":"test","object":"test2/pray.svg","status":"OK","statusCode":200,"timeToResponse":"2245254ns"},"remotehost":"172.20.0.1","requestID":"16503E531076BE1D","userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36","requestClaims":{"accessKey":"12345678","exp":1607939949,"sub":"12345678"},"requestHeader":{"Accept":"*/*","Accept-Encoding":"gzip, deflate, br","Accept-Language":"en-US,en;q=0.9,he;q=0.8,lb;q=0.7,de;q=0.6","Authorization":"Bearer eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiIxMjM0NTY3OCIsImV4cCI6MTYwNzkzOTk0OSwic3ViIjoiMTIzNDU2NzgifQ.zXmWWBSu6PCh1UP-aPALSo4BrVFaHtuH6Gk33927fdc-VvDxIa6TYc6gKVdB6UOlmPMYAOenapyJ_36W5iXIsw","Content-Length":"5982","Content-Type":"image/svg+xml","Origin":"https://local.cwm-worker-minio:8443","Referer":"https://local.cwm-worker-minio:8443/minio/test/test2/","Sec-Fetch-Dest":"empty","Sec-Fetch-Mode":"cors","Sec-Fetch-Site":"same-origin","User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36","X-Amz-Date":"20201213T095953Z"},"responseHeader":{"Access-Control-Allow-Credentials":"true","Access-Control-Allow-Origin":"https://local.cwm-worker-minio:8443","Access-Control-Expose-Headers":"Date, Etag, Server, Connection, Accept-Ranges, Content-Range, Content-Encoding, Content-Length, Content-Type, Content-Disposition, Last-Modified, Content-Language, Cache-Control, Retry-After, X-Amz-Bucket-Region, Expires, X-Amz*, X-Amz*, *","Content-Security-Policy":"block-all-mixed-content","ETag":"","Vary":"Origin","X-Amz-Request-Id":"16503E531076BE1D","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-https","time":"2020-12-13T10:00:11.162262185Z","api":{"name":"WebUpload","bucket":"test2","object":"Misc.csv","status":"OK","statusCode":200,"timeToResponse":"2414404ns"},"remotehost":"172.20.0.1","requestID":"16503E573002726A","userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36","requestClaims":{"accessKey":"12345678","exp":1607939949,"sub":"12345678"},"requestHeader":{"Accept":"*/*","Accept-Encoding":"gzip, deflate, br","Accept-Language":"en-US,en;q=0.9,he;q=0.8,lb;q=0.7,de;q=0.6","Authorization":"Bearer eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiIxMjM0NTY3OCIsImV4cCI6MTYwNzkzOTk0OSwic3ViIjoiMTIzNDU2NzgifQ.zXmWWBSu6PCh1UP-aPALSo4BrVFaHtuH6Gk33927fdc-VvDxIa6TYc6gKVdB6UOlmPMYAOenapyJ_36W5iXIsw","Content-Length":"867","Content-Type":"text/csv","Origin":"https://local.cwm-worker-minio:8443","Referer":"https://local.cwm-worker-minio:8443/minio/test2/","Sec-Fetch-Dest":"empty","Sec-Fetch-Mode":"cors","Sec-Fetch-Site":"same-origin","User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36","X-Amz-Date":"20201213T100011Z"},"responseHeader":{"Access-Control-Allow-Credentials":"true","Access-Control-Allow-Origin":"https://local.cwm-worker-minio:8443","Access-Control-Expose-Headers":"Date, Etag, Server, Connection, Accept-Ranges, Content-Range, Content-Encoding, Content-Length, Content-Type, Content-Disposition, Last-Modified, Content-Language, Cache-Control, Retry-After, X-Amz-Bucket-Region, Expires, X-Amz*, X-Amz*, *","Content-Security-Policy":"block-all-mixed-content","ETag":"","Vary":"Origin","X-Amz-Request-Id":"16503E573002726A","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-https","time":"2020-12-13T10:00:11.163519081Z","api":{"name":"WebUpload","bucket":"test2","object":"Constru.csv","status":"OK","statusCode":200,"timeToResponse":"2857463ns"},"remotehost":"172.20.0.1","requestID":"16503E57300F4542","userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36","requestClaims":{"accessKey":"12345678","exp":1607939949,"sub":"12345678"},"requestHeader":{"Accept":"*/*","Accept-Encoding":"gzip, deflate, br","Accept-Language":"en-US,en;q=0.9,he;q=0.8,lb;q=0.7,de;q=0.6","Authorization":"Bearer eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiIxMjM0NTY3OCIsImV4cCI6MTYwNzkzOTk0OSwic3ViIjoiMTIzNDU2NzgifQ.zXmWWBSu6PCh1UP-aPALSo4BrVFaHtuH6Gk33927fdc-VvDxIa6TYc6gKVdB6UOlmPMYAOenapyJ_36W5iXIsw","Content-Length":"844","Content-Type":"text/csv","Origin":"https://local.cwm-worker-minio:8443","Referer":"https://local.cwm-worker-minio:8443/minio/test2/","Sec-Fetch-Dest":"empty","Sec-Fetch-Mode":"cors","Sec-Fetch-Site":"same-origin","User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36","X-Amz-Date":"20201213T100011Z"},"responseHeader":{"Access-Control-Allow-Credentials":"true","Access-Control-Allow-Origin":"https://local.cwm-worker-minio:8443","Access-Control-Expose-Headers":"Date, Etag, Server, Connection, Accept-Ranges, Content-Range, Content-Encoding, Content-Length, Content-Type, Content-Disposition, Last-Modified, Content-Language, Cache-Control, Retry-After, X-Amz-Bucket-Region, Expires, X-Amz*, X-Amz*, *","Content-Security-Policy":"block-all-mixed-content","ETag":"","Vary":"Origin","X-Amz-Request-Id":"16503E57300F4542","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-https","time":"2020-12-13T10:00:11.164228465Z","api":{"name":"WebUpload","bucket":"test2","object":"Hasadna.csv","status":"OK","statusCode":200,"timeToResponse":"3569188ns"},"remotehost":"172.20.0.1","requestID":"16503E57300F3E83","userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36","requestClaims":{"accessKey":"12345678","exp":1607939949,"sub":"12345678"},"requestHeader":{"Accept":"*/*","Accept-Encoding":"gzip, deflate, br","Accept-Language":"en-US,en;q=0.9,he;q=0.8,lb;q=0.7,de;q=0.6","Authorization":"Bearer eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiIxMjM0NTY3OCIsImV4cCI6MTYwNzkzOTk0OSwic3ViIjoiMTIzNDU2NzgifQ.zXmWWBSu6PCh1UP-aPALSo4BrVFaHtuH6Gk33927fdc-VvDxIa6TYc6gKVdB6UOlmPMYAOenapyJ_36W5iXIsw","Content-Length":"2074","Content-Type":"text/csv","Origin":"https://local.cwm-worker-minio:8443","Referer":"https://local.cwm-worker-minio:8443/minio/test2/","Sec-Fetch-Dest":"empty","Sec-Fetch-Mode":"cors","Sec-Fetch-Site":"same-origin","User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36","X-Amz-Date":"20201213T100011Z"},"responseHeader":{"Access-Control-Allow-Credentials":"true","Access-Control-Allow-Origin":"https://local.cwm-worker-minio:8443","Access-Control-Expose-Headers":"Date, Etag, Server, Connection, Accept-Ranges, Content-Range, Content-Encoding, Content-Length, Content-Type, Content-Disposition, Last-Modified, Content-Language, Cache-Control, Retry-After, X-Amz-Bucket-Region, Expires, X-Amz*, X-Amz*, *","Content-Security-Policy":"block-all-mixed-content","ETag":"","Vary":"Origin","X-Amz-Request-Id":"16503E57300F3E83","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-https","time":"2020-12-13T10:00:11.166982552Z","api":{"name":"WebUpload","bucket":"test2","object":"personal.csv","status":"OK","statusCode":200,"timeToResponse":"3350434ns"},"remotehost":"172.20.0.1","requestID":"16503E57303C9CA9","userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36","requestClaims":{"accessKey":"12345678","exp":1607939949,"sub":"12345678"},"requestHeader":{"Accept":"*/*","Accept-Encoding":"gzip, deflate, br","Accept-Language":"en-US,en;q=0.9,he;q=0.8,lb;q=0.7,de;q=0.6","Authorization":"Bearer eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiIxMjM0NTY3OCIsImV4cCI6MTYwNzkzOTk0OSwic3ViIjoiMTIzNDU2NzgifQ.zXmWWBSu6PCh1UP-aPALSo4BrVFaHtuH6Gk33927fdc-VvDxIa6TYc6gKVdB6UOlmPMYAOenapyJ_36W5iXIsw","Content-Length":"1300","Content-Type":"text/csv","Origin":"https://local.cwm-worker-minio:8443","Referer":"https://local.cwm-worker-minio:8443/minio/test2/","Sec-Fetch-Dest":"empty","Sec-Fetch-Mode":"cors","Sec-Fetch-Site":"same-origin","User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36","X-Amz-Date":"20201213T100011Z"},"responseHeader":{"Access-Control-Allow-Credentials":"true","Access-Control-Allow-Origin":"https://local.cwm-worker-minio:8443","Access-Control-Expose-Headers":"Date, Etag, Server, Connection, Accept-Ranges, Content-Range, Content-Encoding, Content-Length, Content-Type, Content-Disposition, Last-Modified, Content-Language, Cache-Control, Retry-After, X-Amz-Bucket-Region, Expires, X-Amz*, X-Amz*, *","Content-Security-Policy":"block-all-mixed-content","ETag":"","Vary":"Origin","X-Amz-Request-Id":"16503E57303C9CA9","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:28.146379165Z","api":{"name":"PutObject","bucket":"warp-benchmark-bucket","object":"XTSAfFmC/wmAgfs(Y5gfvrUvH.rnd","status":"OK","statusCode":200,"timeToResponse":"5600028ns"},"remotehost":"172.20.0.1","requestID":"16503E7714B596E7","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=f9617e6380d377f96574e8789b369de3e3c1873a5142ab43a94da2df845195db","Content-Length":"10415","Content-Type":"application/octet-stream","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"STREAMING-AWS4-HMAC-SHA256-PAYLOAD","X-Amz-Date":"20201213T100228Z","X-Amz-Decoded-Content-Length":"10240"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"0","Content-Security-Policy":"block-all-mixed-content","ETag":"7567b673c6b45518b054f71053745bbb","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E7714B596E7","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:28.148376209Z","api":{"name":"PutObject","bucket":"warp-benchmark-bucket","object":"MpIBxAbZ/19VPLXm3I2ucVYIa.rnd","status":"OK","statusCode":200,"timeToResponse":"4019987ns"},"remotehost":"172.20.0.1","requestID":"16503E7714EC5781","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=6b9fe765f9d66be6e4f31ef045919d95084d6eed5c3b4100689599d8eacba174","Content-Length":"10415","Content-Type":"application/octet-stream","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"STREAMING-AWS4-HMAC-SHA256-PAYLOAD","X-Amz-Date":"20201213T100228Z","X-Amz-Decoded-Content-Length":"10240"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"0","Content-Security-Policy":"block-all-mixed-content","ETag":"1e36a820bbd64d49a99805bb704ce524","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E7714EC5781","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:28.150866606Z","api":{"name":"PutObject","bucket":"warp-benchmark-bucket","object":"uPe9zwVp/TEV(fS)CtKQj9)Kk.rnd","status":"OK","statusCode":200,"timeToResponse":"6407612ns"},"remotehost":"172.20.0.1","requestID":"16503E7714EDEC3C","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=4905027a735dbff6cee19490fe93a560bad6589255f18d347ee876229ab19331","Content-Length":"10415","Content-Type":"application/octet-stream","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"STREAMING-AWS4-HMAC-SHA256-PAYLOAD","X-Amz-Date":"20201213T100228Z","X-Amz-Decoded-Content-Length":"10240"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"0","Content-Security-Policy":"block-all-mixed-content","ETag":"103806e3c82de16ed70fc943573c4248","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E7714EDEC3C","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:28.150997904Z","api":{"name":"PutObject","bucket":"warp-benchmark-bucket","object":"SACTQyFh/6Khq9rqx5iR9S313.rnd","status":"OK","statusCode":200,"timeToResponse":"6598556ns"},"remotehost":"172.20.0.1","requestID":"16503E7714ED097A","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=888587f0d7a475dbc31654d9bbfb5090973057601bc4981ebe9bb3d458cb819d","Content-Length":"10415","Content-Type":"application/octet-stream","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"STREAMING-AWS4-HMAC-SHA256-PAYLOAD","X-Amz-Date":"20201213T100228Z","X-Amz-Decoded-Content-Length":"10240"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"0","Content-Security-Policy":"block-all-mixed-content","ETag":"3c226393e1fda01fba77eeee8cdfba94","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E7714ED097A","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:28.152358033Z","api":{"name":"PutObject","bucket":"warp-benchmark-bucket","object":"U9dAIeMw/xFlwgtYWHmWDU92i.rnd","status":"OK","statusCode":200,"timeToResponse":"6643027ns"},"remotehost":"172.20.0.1","requestID":"16503E7715012723","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=ecca35dd261dae1875de09ce62b2ffc9aa98d1734fcd1f3d3e0ca9020937d895","Content-Length":"10415","Content-Type":"application/octet-stream","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"STREAMING-AWS4-HMAC-SHA256-PAYLOAD","X-Amz-Date":"20201213T100228Z","X-Amz-Decoded-Content-Length":"10240"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"0","Content-Security-Policy":"block-all-mixed-content","ETag":"122d13e787596645b9310a6e56a51a17","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E7715012723","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.307623369Z","api":{"name":"DeleteObject","bucket":"warp-benchmark-bucket","object":"DsrjeNDy/oeGegUiJWIDTWOOq.rnd","status":"No Content","statusCode":204,"timeToResponse":"1819254ns"},"remotehost":"172.20.0.1","requestID":"16503E77D15C32DB","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=25804b0b3ed5ba119fd6504f8492e2ba47a0fc2ca4f22d865592022c5728b332","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"0","Content-Security-Policy":"block-all-mixed-content","ETag":"","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D15C32DB","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.311626518Z","api":{"name":"DeleteObject","bucket":"warp-benchmark-bucket","object":"DsrjeNDy/(60XuwNT(uVnBWFc.rnd","status":"No Content","statusCode":204,"timeToResponse":"1070737ns"},"remotehost":"172.20.0.1","requestID":"16503E77D1A4F9F1","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a3a705dbe6a5b8d74ec26b1961b19553571753d745c8687e08d362490dca40de","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"0","Content-Security-Policy":"block-all-mixed-content","ETag":"","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D1A4F9F1","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.312482136Z","api":{"name":"DeleteObject","bucket":"warp-benchmark-bucket","object":"uPe9zwVp/kSu7mJneCaD)Fuit.rnd","status":"No Content","statusCode":204,"timeToResponse":"296524ns"},"remotehost":"172.20.0.1","requestID":"16503E77D1BDD9DE","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=817281c9cb6ecc0d6f6f46949b71f7d62cf8d845c51b38c2e3a25b375ed248fd","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"0","Content-Security-Policy":"block-all-mixed-content","ETag":"","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D1BDD9DE","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.31500656Z","api":{"name":"DeleteObject","bucket":"warp-benchmark-bucket","object":"uPe9zwVp/CG2EVQTBBoZS(A2Q.rnd","status":"No Content","statusCode":204,"timeToResponse":"1104650ns"},"remotehost":"172.20.0.1","requestID":"16503E77D1D80933","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=50b12d8a0ea7b9aa1a6616ecbf49c49b5e96a17c39fc14b81564fc6f147f332a","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"0","Content-Security-Policy":"block-all-mixed-content","ETag":"","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D1D80933","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.315527682Z","api":{"name":"DeleteObject","bucket":"warp-benchmark-bucket","object":"MpIBxAbZ/BsNwtds2hjH06r2T.rnd","status":"No Content","statusCode":204,"timeToResponse":"322966ns"},"remotehost":"172.20.0.1","requestID":"16503E77D1EBE9B1","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=1fac2438b2611e1efe51cc70891a62a23534f6559dfcbf5084200c400a57cc65","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"0","Content-Security-Policy":"block-all-mixed-content","ETag":"","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D1EBE9B1","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.315755898Z","api":{"name":"GetObject","bucket":"warp-benchmark-bucket","object":"uPe9zwVp/xaS)33LTUGT1SqAk.rnd","status":"OK","statusCode":200,"timeToFirstByte":"547107ns","timeToResponse":"560938ns"},"remotehost":"172.20.0.1","requestID":"16503E77D1EBBD86","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=f9acf5cad5bbde784b8f15680ed50e3987a5b63231a84246ff871e7a9856b613","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"10240","Content-Security-Policy":"block-all-mixed-content","Content-Type":"application/octet-stream","ETag":"28f85d9558fa9c2f2eca1b70dae20832","Last-Modified":"Sun, 13 Dec 2020 10:02:28 GMT","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D1EBBD86","X-Cache":"MISS","X-Cache-Lookup":"MISS","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.316316176Z","api":{"name":"GetObject","bucket":"warp-benchmark-bucket","object":"SACTQyFh/r3VI4ADkZo2KA8MT.rnd","status":"OK","statusCode":200,"timeToFirstByte":"734869ns","timeToResponse":"750395ns"},"remotehost":"172.20.0.1","requestID":"16503E77D1F15E91","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=bba0dd4a5b46361a146631bee00a77a5c2a541979dc4ab7d0d8e3156bb2be69b","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"10240","Content-Security-Policy":"block-all-mixed-content","Content-Type":"application/octet-stream","ETag":"adab22732f9746f00ff7c3cbe93d5a75","Last-Modified":"Sun, 13 Dec 2020 10:02:28 GMT","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D1F15E91","X-Cache":"MISS","X-Cache-Lookup":"MISS","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.316583584Z","api":{"name":"GetObject","bucket":"warp-benchmark-bucket","object":"SACTQyFh/33lecLH841eGQqij.rnd","status":"OK","statusCode":200,"timeToFirstByte":"552973ns","timeToResponse":"1377348ns"},"remotehost":"172.20.0.1","requestID":"16503E77D1EBEB39","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6b616c836152a0d3e342bc0c0ddcdb037185066d942e87cdac0f03de9110f2e4","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"10240","Content-Security-Policy":"block-all-mixed-content","Content-Type":"application/octet-stream","ETag":"a074a4c7facee1c0dbb440545f7d6579","Last-Modified":"Sun, 13 Dec 2020 10:02:28 GMT","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D1EBEB39","X-Cache":"MISS","X-Cache-Lookup":"MISS","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.317006173Z","api":{"name":"GetObject","bucket":"warp-benchmark-bucket","object":"U9dAIeMw/ZdueXA8roNczk4pp.rnd","status":"OK","statusCode":200,"timeToFirstByte":"625551ns","timeToResponse":"641906ns"},"remotehost":"172.20.0.1","requestID":"16503E77D1FD94CA","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6695dd830a24e33807d442ea0d3dccf7b326fcb4e2fda06ee043f448743e7430","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"10240","Content-Security-Policy":"block-all-mixed-content","Content-Type":"application/octet-stream","ETag":"8e331b8121dba9e2b449c6d2732b49f6","Last-Modified":"Sun, 13 Dec 2020 10:02:28 GMT","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D1FD94CA","X-Cache":"MISS","X-Cache-Lookup":"MISS","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.318899032Z","api":{"name":"GetObject","bucket":"warp-benchmark-bucket","object":"U9dAIeMw/bOi8mDhPI7xkpLc(.rnd","status":"OK","statusCode":200,"timeToFirstByte":"659834ns","timeToResponse":"675772ns"},"remotehost":"172.20.0.1","requestID":"16503E77D219E810","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=70d55e185056eeb8bda0aa99a0d019c8af35d9d23cdaa1e4efb382e57f43b247","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"10240","Content-Security-Policy":"block-all-mixed-content","Content-Type":"application/octet-stream","ETag":"12a5cae0b22eb8ddede896d51706509c","Last-Modified":"Sun, 13 Dec 2020 10:02:28 GMT","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D219E810","X-Cache":"MISS","X-Cache-Lookup":"MISS","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.319875016Z","api":{"name":"HeadObject","bucket":"warp-benchmark-bucket","object":"uPe9zwVp/I55Vvgsvz6(s5dx(.rnd","status":"OK","statusCode":200,"timeToResponse":"375661ns"},"remotehost":"172.20.0.1","requestID":"16503E77D22D666F","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6d01aa21d6791f12da408b9fedbe3b222b95e0b874b7899400f920e157cf371e","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"10240","Content-Security-Policy":"block-all-mixed-content","Content-Type":"application/octet-stream","ETag":"4d851e2d1b56eda8c047c498e557b568","Last-Modified":"Sun, 13 Dec 2020 10:02:28 GMT","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D22D666F","X-Cache":"MISS","X-Cache-Lookup":"MISS","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.322382865Z","api":{"name":"HeadObject","bucket":"warp-benchmark-bucket","object":"MpIBxAbZ/Xomp81ekJZPqDI0Y.rnd","status":"OK","statusCode":200,"timeToResponse":"298541ns"},"remotehost":"172.20.0.1","requestID":"16503E77D254DF29","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0264ac1e81f4ae83419ea62a0c3bb238c6fffecf853d65fce16c8a43794ba60a","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"10240","Content-Security-Policy":"block-all-mixed-content","Content-Type":"application/octet-stream","ETag":"07817607c15b8d6a52e184cc0a693680","Last-Modified":"Sun, 13 Dec 2020 10:02:28 GMT","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D254DF29","X-Cache":"MISS","X-Cache-Lookup":"MISS","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.325077259Z","api":{"name":"HeadObject","bucket":"warp-benchmark-bucket","object":"SACTQyFh/7x7JgnxP9x3JpdAN.rnd","status":"OK","statusCode":200,"timeToResponse":"304259ns"},"remotehost":"172.20.0.1","requestID":"16503E77D27DE12B","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=3b53df8afe888f9d786c4847d6c2186e77c70ff8cb4e222dcacb65cb11ae7c88","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"10240","Content-Security-Policy":"block-all-mixed-content","Content-Type":"application/octet-stream","ETag":"fc1d79a40d198f25c4a5448cfa82071d","Last-Modified":"Sun, 13 Dec 2020 10:02:28 GMT","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D27DE12B","X-Cache":"MISS","X-Cache-Lookup":"MISS","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.330091282Z","api":{"name":"HeadObject","bucket":"warp-benchmark-bucket","object":"SACTQyFh/33lecLH841eGQqij.rnd","status":"OK","statusCode":200,"timeToResponse":"417243ns"},"remotehost":"172.20.0.1","requestID":"16503E77D2C8A79D","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=7a99e1b7d3eaa3f466fb5c4352f84657ab22422c2be62cc8694eccf6fbd833fc","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"10240","Content-Security-Policy":"block-all-mixed-content","Content-Type":"application/octet-stream","ETag":"a074a4c7facee1c0dbb440545f7d6579","Last-Modified":"Sun, 13 Dec 2020 10:02:28 GMT","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D2C8A79D","X-Cache":"MISS","X-Cache-Lookup":"MISS","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:02:31.330171641Z","api":{"name":"HeadObject","bucket":"warp-benchmark-bucket","object":"XTSAfFmC/6jVNqbTHjY)TMUsf.rnd","status":"OK","statusCode":200,"timeToResponse":"417005ns"},"remotehost":"172.20.0.1","requestID":"16503E77D2C9E103","userAgent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=12345678/20201213/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=686a542242d464fd2787765209b2ea1d6bf711ee9d12fc9bcf6cabb4b3c936b0","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.5 warp/0.3.17","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20201213T100231Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"10240","Content-Security-Policy":"block-all-mixed-content","Content-Type":"application/octet-stream","ETag":"68a0d2b0c2733dfca0b6f5d37ea01b99","Last-Modified":"Sun, 13 Dec 2020 10:02:28 GMT","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503E77D2C9E103","X-Cache":"MISS","X-Cache-Lookup":"MISS","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:27:55.410575308Z","api":{"name":"WebDownload","bucket":"test2","object":"personal.csv","status":"OK","statusCode":200,"timeToFirstByte":"539568ns","timeToResponse":"554854ns"},"remotehost":"172.20.0.1","requestID":"16503FDAAD0D3460","userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36","requestClaims":{"accessKey":"12345678","exp":1607855335,"sub":"12345678"},"requestQuery":{"token":"eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiIxMjM0NTY3OCIsImV4cCI6MTYwNzg1NTMzNSwic3ViIjoiMTIzNDU2NzgifQ.Lr6b1NB4cydukrBF-_1StDLwR6UGsRJZlVlfRIeVlWJZVp2AAS9DMXpVueUzSOmqiNvlyCPfjsIzHC5nIjiyzw"},"requestHeader":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9","Accept-Encoding":"gzip, deflate","Accept-Language":"en-US,en;q=0.9,he;q=0.8,lb;q=0.7,de;q=0.6","Connection":"keep-alive","Referer":"http://local.cwm-worker-minio:8080/minio/test2/","Upgrade-Insecure-Requests":"1","User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36"},"responseHeader":{"Accept-Ranges":"bytes","Cache-Control":"no-store","Content-Disposition":"attachment; filename=\"personal.csv\"","Content-Length":"1300","Content-Security-Policy":"block-all-mixed-content","Content-Type":"text/csv","ETag":"0ea1fa5fd9afe7f0f9519c5825f57310","Last-Modified":"Sun, 13 Dec 2020 10:00:11 GMT","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503FDAAD0D3460","X-Cache":"MISS","X-Cache-Lookup":"MISS","X-Xss-Protection":"1; mode=block"}}
+{"version":"1","deploymentid":"docker-compose-http","time":"2020-12-13T10:27:57.202896957Z","api":{"name":"WebDownload","bucket":"test2","object":"Misc.csv","status":"OK","statusCode":200,"timeToFirstByte":"647394ns","timeToResponse":"667159ns"},"remotehost":"172.20.0.1","requestID":"16503FDB17E01EDE","userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36","requestClaims":{"accessKey":"12345678","exp":1607855337,"sub":"12345678"},"requestQuery":{"token":"eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiIxMjM0NTY3OCIsImV4cCI6MTYwNzg1NTMzNywic3ViIjoiMTIzNDU2NzgifQ.8jzmcdPBPKjwCuKJLES2qtfhf7iYN-LXZz_zXU0rlCjMzbFHJI0hFF0naMtFQEA1R9Ad1WhZY8mYErqdioMonw"},"requestHeader":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9","Accept-Encoding":"gzip, deflate","Accept-Language":"en-US,en;q=0.9,he;q=0.8,lb;q=0.7,de;q=0.6","Connection":"keep-alive","Referer":"http://local.cwm-worker-minio:8080/minio/test2/","Upgrade-Insecure-Requests":"1","User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36"},"responseHeader":{"Accept-Ranges":"bytes","Cache-Control":"no-store","Content-Disposition":"attachment; filename=\"Misc.csv\"","Content-Length":"867","Content-Security-Policy":"block-all-mixed-content","Content-Type":"text/csv","ETag":"b58887295950f4a7edceccab141d8c92","Last-Modified":"Sun, 13 Dec 2020 10:00:11 GMT","Server":"MinIO","Vary":"Origin","X-Amz-Request-Id":"16503FDB17E01EDE","X-Cache":"MISS","X-Cache-Lookup":"MISS","X-Xss-Protection":"1; mode=block"}}

data/test/plugin/test_in_http_cwm.rb

@@ -0,0 +1,125 @@
+# frozen-string-literal: true
+
+require 'helper'
+require 'fluent/plugin/in_http_cwm'
+require 'net/http'
+require 'json'
+
+class CwmHttpInputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+
+    @default_conf = config_element('ROOT', '', { 'tag' => 'test' })
+    @custom_conf = config_element('ROOT', '', { 'tag' => 'test' }, [
+                                    config_element('redis', '', {
+                                                     'grace_period' => '1s',
+                                                     'flush_interval' => '1s'
+                                                   })
+                                  ])
+
+    @expected_redis_output = {
+      'deploymentid:last_action:docker-compose-http' => '',
+      'deploymentid:last_action:docker-compose-https' => '',
+      'deploymentid:minio-metrics:docker-compose-http:bytes_in' => '61852',
+      'deploymentid:minio-metrics:docker-compose-https:bytes_in' => '14875',
+      'deploymentid:minio-metrics:docker-compose-http:bytes_out' => '112012',
+      'deploymentid:minio-metrics:docker-compose-https:bytes_out' => '2755',
+      'deploymentid:minio-metrics:docker-compose-http:num_requests_out' => '7',
+      'deploymentid:minio-metrics:docker-compose-http:num_requests_in' => '10',
+      'deploymentid:minio-metrics:docker-compose-https:num_requests_in' => '5',
+      'deploymentid:minio-metrics:docker-compose-http:num_requests_misc' => '5'
+    }.freeze
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::CwmHttpInput).configure(conf)
+  end
+
+  sub_test_case 'configuration' do
+    test 'default configuration test' do
+      driver = create_driver(@default_conf)
+      plugin = driver.instance
+      assert_equal Fluent::Plugin::CwmHttpInput, plugin.class
+      assert_equal 'localhost', plugin.host
+      assert_equal 8080, plugin.port
+    end
+
+    test 'redis default configuration test' do
+      driver = create_driver(@default_conf)
+      plugin = driver.instance
+      redis = plugin.redis_config
+      assert_equal 'localhost', redis.host
+      assert_equal 6379, redis.port
+      assert_equal '300s', redis.grace_period
+      assert_equal '300s', redis.flush_interval
+      assert_equal 'deploymentid:last_action', redis.last_update_prefix
+      assert_equal 'deploymentid:minio-metrics', redis.metrics_prefix
+    end
+  end
+
+  sub_test_case 'route#emit' do
+    test 'emit test' do
+      driver = create_driver(@custom_conf)
+      plugin = driver.instance
+      redis = plugin.redis_config
+
+      res_codes = []
+      lines = 0
+      health_res_code = ''
+      health_res_body = ''
+
+      driver.run do
+        health_res = get('/health')
+        health_res_code = health_res.code
+        health_res_body = health_res.body
+        File.readlines('./test/logs.txt').each do |line|
+          res = post('/test', line.chomp)
+          res_codes << res.code
+          lines += 1
+        end
+      end
+
+      assert_equal lines, res_codes.size
+      assert_equal '200', res_codes[0]
+      assert_equal 1, res_codes.uniq.size
+
+      assert_equal '200', health_res_code
+      assert_equal ({}), JSON.parse(health_res_body)
+
+      # run and test private flushing methods
+      `redis-cli FLUSHALL`
+      sleep(redis.grace_period)
+      driver.events.each do
+        plugin.send(:flush_api_metrics)
+        plugin.send(:flush_last_action)
+      end
+
+      # verify from Redis server
+      @expected_redis_output.each do |key, expected_value|
+        if key.include? 'last_action'
+          exists = `redis-cli EXISTS #{key}`.chomp
+          assert_equal '1', exists
+        else
+          actual_value = `redis-cli GET #{key}`.chomp
+          assert_equal expected_value, actual_value
+        end
+      end
+    end
+  end
+
+  private
+
+  def post(path, body)
+    http = Net::HTTP.new('127.0.0.1', 8080)
+    header = { 'Content-Type' => 'application/json' }
+    req = Net::HTTP::Post.new(path, header)
+    req.body = body
+    http.request(req)
+  end
+
+  def get(path)
+    http = Net::HTTP.new('127.0.0.1', 8080)
+    req = Net::HTTP::Get.new(path)
+    http.request(req)
+  end
+end