fluent-plugin-vmware-log-intelligence 2.0.5 → 2.0.6

data/lib/fluent/plugin/out_vmware_log_intelligence.rb

@@ -1,179 +1,201 @@
-# Copyright (c) 2013 ablagoev
-# Copyright 2018 VMware, Inc.
-# SPDX-License-Identifier: MIT
-
-require "fluent/plugin/output"
-require "fluent/plugin/http_client"
-
-module Fluent::Plugin
-  class LogIntelligenceOutput < Output
-    Fluent::Plugin.register_output('vmware_log_intelligence', self)
-
-    config_param :endpoint_url, :string
-    config_param :http_retry_statuses, :string, default: ''
-    config_param :read_timeout, :integer, default: 60
-    config_param :open_timeout, :integer, default: 60
-    # config_param :use_ssl, :bool, :default => false
-    config_param :verify_ssl, :bool, :default => true
-    config_param :rate_limit_msec, :integer, :default => 0
-    # Keys from log event whose values should be added as log message/text
-    # to log-intelligence. Note these key/value pairs  won't be added as metadata/fields
-    config_param :log_text_keys, :array, default: ["log", "message", "msg"], value_type: :string
-    # Flatten hashes to create one key/val pair w/o losing log data
-    config_param :flatten_hashes, :bool, :default => true
-    # Seperator to use for joining flattened keys
-    config_param :flatten_hashes_separator, :string, :default => "_"
-
-    config_section :buffer do
-      config_set_default :@type, "memory"
-      config_set_default :chunk_keys, []
-      config_set_default :timekey_use_utc, true
-    end
-
-    def initialize
-      super
-      require 'http'
-      require 'uri'
-    end
-
-    def validate_uri(uri_string)
-      unless uri_string =~ /^#{URI.regexp}$/
-        fail Fluent::ConfigError, 'endpoint_url invalid'
-      end
-
-      begin
-        @uri = URI.parse(uri_string)
-      rescue URI::InvalidURIError
-        raise Fluent::ConfigError, 'endpoint_url invalid'
-      end
-    end
-
-    def retrieve_headers(conf)
-      headers = {}
-      conf.elements.each do |element|
-        if element.name == 'headers'
-          headers = element.to_hash
-        end
-      end
-      headers
-    end
-
-    def shorten_key(key)
-      # LINT doesn't allow some characters in field 'name'
-      # like '/', '-', '\', '.', etc. so replace them with @flatten_hashes_separator
-      key = key.gsub(/[\/\.\-\\]/,@flatten_hashes_separator).downcase
-      key
-    end
-
-    def create_lint_event(record)
-      flattened_records = {}
-      merged_records = {}
-      if @flatten_hashes
-        flattened_records = flatten_record(record, [])
-      else
-        flattened_records = record
-      end
-
-      keys = []
-      log = ''
-      flattened_records.each do |key, value|
-        begin
-          next if value.nil?
-          # LINT doesn't support duplicate fields, make unique names by appending underscore
-          key = shorten_key(key)
-          if keys.include?(key)
-            value = merged_records[key] + " " + value
-          end
-          keys.push(key)
-          key.force_encoding("utf-8")
-
-          if value.is_a?(String) 
-            value.force_encoding("utf-8")
-          end
-        end
-
-        if @log_text_keys.include?(key)
-          if log != "#{value}"
-            if log.empty?
-              log = "#{value}"
-            else
-              log += " #{value}"
-            end
-          end
-        else
-          merged_records[key] = value
-        end
-      end
-      merged_records["text"] = log
-
-      if log == "\\n"
-        {}
-      else
-        merged_records
-      end
-    end
-
-    def flatten_record(record, prefix=[])
-      ret = {}
-
-      case record
-      when Hash
-        record.each do |key, value|
-          if @log_text_keys.include?(key)
-            ret.merge!({key.to_s => value})
-          else
-            ret.merge! flatten_record(value, prefix + [key.to_s])
-          end
-        end
-      when Array
-        record.each do |value|
-          ret.merge! flatten_record(value, prefix)
-        end
-      else
-        return {prefix.join(@flatten_hashes_separator) => record}
-      end
-      ret
-    end
-    def configure(conf)
-      super
-      validate_uri(@endpoint_url)
-
-      @statuses = @http_retry_statuses.split(',').map { |status| status.to_i }
-      @statuses = [] if @statuses.nil?
-
-      @headers = retrieve_headers(conf)
-
-      @http_client = Fluent::Plugin::HttpClient.new(
-        @endpoint_url, @verify_ssl, @headers, @statuses,
-        @open_timeout, @read_timeout, @log)
-    end
-
-    def start
-      super
-    end
-
-    def shutdown
-      super
-      begin
-        @http_client.close if @http_client
-      rescue
-      end
-    end
-
-    def write(chunk)
-      is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
-      if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
-        @log.info('Dropped request due to rate limiting')
-        return
-      end
-
-      data = []
-      chunk.each do |time, record|
-        data << create_lint_event(record)
-      end
-
-      @last_request_time = Time.now.to_f
-      @http_client.post(JSON.dump(data))
-    end
-  end
-end
+# Copyright (c) 2013 ablagoev
+# Copyright 2018 VMware, Inc.
+# SPDX-License-Identifier: MIT
+
+	
+require 'zlib'
+require "fluent/plugin/output"
+require "fluent/plugin/http_client"
+
+module Fluent::Plugin
+  class LogIntelligenceOutput < Output
+    Fluent::Plugin.register_output('vmware_log_intelligence', self)
+
+    config_param :http_compress, :bool, :default => false
+    config_param :endpoint_url, :string
+    config_param :bearer_token, :string, default: ''
+    config_param :http_retry_statuses, :string, default: ''
+    config_param :read_timeout, :integer, default: 60
+    config_param :open_timeout, :integer, default: 60
+    # config_param :use_ssl, :bool, :default => false
+    config_param :verify_ssl, :bool, :default => true
+    config_param :rate_limit_msec, :integer, :default => 0
+    # Keys from log event whose values should be added as log message/text
+    # to log-intelligence. Note these key/value pairs  won't be added as metadata/fields
+    config_param :log_text_keys, :array, default: ["log", "message", "msg"], value_type: :string
+    # Flatten hashes to create one key/val pair w/o losing log data
+    config_param :flatten_hashes, :bool, :default => true
+    # Seperator to use for joining flattened keys
+    config_param :flatten_hashes_separator, :string, :default => "_"
+
+    config_section :buffer do
+      config_set_default :@type, "memory"
+      config_set_default :chunk_keys, []
+      config_set_default :timekey_use_utc, true
+    end
+
+    def initialize
+      super
+      require 'http'
+      require 'uri'
+    end
+
+    def validate_uri(uri_string)
+      unless uri_string =~ /^#{URI.regexp}$/
+        fail Fluent::ConfigError, 'endpoint_url invalid'
+      end
+
+      begin
+        @uri = URI.parse(uri_string)
+      rescue URI::InvalidURIError
+        raise Fluent::ConfigError, 'endpoint_url invalid'
+      end
+    end
+
+    def retrieve_headers(conf)
+      headers = {}
+      conf.elements.each do |element|
+        if @http_compress
+          set_gzip_header(element)
+        end
+        if element.name == 'headers'
+          if @bearer_token != ''
+            element['Authorization'] = 'Bearer ' + @bearer_token
+          end
+          headers = element.to_hash
+        end
+      end
+      headers
+    end
+
+    def set_gzip_header(element)
+      element['Content-Encoding'] = 'gzip'
+      element
+    end
+
+    def shorten_key(key)
+      # LINT doesn't allow some characters in field 'name'
+      # like '/', '-', '\', '.', etc. so replace them with @flatten_hashes_separator
+      key = key.gsub(/[\/\.\-\\]/,@flatten_hashes_separator).downcase
+      key
+    end
+
+    def create_lint_event(record)
+      flattened_records = {}
+      merged_records = {}
+      if @flatten_hashes
+        flattened_records = flatten_record(record, [])
+      else
+        flattened_records = record
+      end
+
+      keys = []
+      log = ''
+      flattened_records.each do |key, value|
+        begin
+          next if value.nil?
+          # LINT doesn't support duplicate fields, make unique names by appending underscore
+          key = shorten_key(key)
+          if keys.include?(key)
+            value = merged_records[key] + " " + value
+          end
+          keys.push(key)
+          key.force_encoding("utf-8")
+
+          if value.is_a?(String) 
+            value.force_encoding("utf-8")
+          end
+        end
+
+        if @log_text_keys.include?(key)
+          if log != "#{value}"
+            if log.empty?
+              log = "#{value}"
+            else
+              log += " #{value}"
+            end
+          end
+        else
+          merged_records[key] = value
+        end
+      end
+      merged_records["text"] = log
+
+      if log == "\\n"
+        {}
+      else
+        merged_records
+      end
+    end
+
+    def flatten_record(record, prefix=[])
+      ret = {}
+
+      case record
+      when Hash
+        record.each do |key, value|
+          if @log_text_keys.include?(key)
+            ret.merge!({key.to_s => value})
+          else
+            ret.merge! flatten_record(value, prefix + [key.to_s])
+          end
+        end
+      when Array
+        record.each do |value|
+          ret.merge! flatten_record(value, prefix)
+        end
+      else
+        return {prefix.join(@flatten_hashes_separator) => record}
+      end
+      ret
+    end
+
+    def configure(conf)
+      super
+      validate_uri(@endpoint_url)
+
+      @statuses = @http_retry_statuses.split(',').map { |status| status.to_i }
+      @statuses = [] if @statuses.nil?
+
+      @headers = retrieve_headers(conf)
+
+      @http_client = Fluent::Plugin::HttpClient.new(
+        @endpoint_url, @verify_ssl, @headers, @statuses,
+        @open_timeout, @read_timeout, @log)
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+      begin
+        @http_client.close if @http_client
+      rescue
+      end
+    end
+
+    def write(chunk)
+      is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
+      if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
+        @log.info('Dropped request due to rate limiting')
+        return
+      end
+
+      data = []
+      chunk.each do |time, record|
+        data << create_lint_event(record)
+      end
+    
+      if @http_compress
+        gzip_body = Zlib::GzipWriter.new(StringIO.new)
+        gzip_body << data.to_json
+        @http_client.post(gzip_body.close.string)
+      else  
+        @last_request_time = Time.now.to_f
+        @http_client.post(JSON.dump(data))
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -1,31 +1,31 @@
-# Copyright (c) 2013 ablagoev
-# Copyright 2018 VMware, Inc.
-# SPDX-License-Identifier: MIT
-
-require 'coveralls'
-Coveralls.wear!
-
-require 'rubygems'
-require 'bundler'
-
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts 'Run `bundle install` to install missing gems'
-  exit e.status_code
-end
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-
-require 'test-unit'
-require 'fluent/test'
-require "fluent/test/driver/output"
-require "fluent/test/helpers"
-Test::Unit::TestCase.include(Fluent::Test::Helpers)
-Test::Unit::TestCase.extend(Fluent::Test::Helpers)
-
-require 'fluent/plugin/out_vmware_log_intelligence'
-require 'webmock/test_unit'
-WebMock.disable_net_connect!
+# Copyright (c) 2013 ablagoev
+# Copyright 2018 VMware, Inc.
+# SPDX-License-Identifier: MIT
+
+require 'coveralls'
+Coveralls.wear!
+
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts 'Run `bundle install` to install missing gems'
+  exit e.status_code
+end
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'test-unit'
+require 'fluent/test'
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)
+
+require 'fluent/plugin/out_vmware_log_intelligence'
+require 'webmock/test_unit'
+WebMock.disable_net_connect!

data/test/plugin/test_http_client.rb

@@ -1,95 +1,95 @@
-# Copyright 2018 VMware, Inc.
-# SPDX-License-Identifier: MIT
-
-require 'helper'
-
-class HttpClientTest < Test::Unit::TestCase
-  DEFAULT_URL = "https://local.endpoint:3000/dummy/xyz"
-
-  def stub_server_out_of_quota(url=DEFAULT_URL)
-    stub_request(:post, url).to_return(:status => [429, "User out of ingestion quota."])
-  end
-
-  def test_check_quota
-    http_client = create_http_client()
-    assert_equal http_client.check_quota, true
-
-    stub_server_out_of_quota
-    http_client.post(JSON.dump(sample_record()))
-    assert_equal http_client.check_quota, false
-  end
-
-  def stub_server_unavailable(url=DEFAULT_URL)
-    stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
-  end
-
-  def stub_server_returns_500(url=DEFAULT_URL)
-    stub_request(:post, url).to_return(:status => [500, "Internal service error"])
-  end
-
-  def stub_server_raise_error(url=DEFAULT_URL)
-    stub_request(:post, url).with do |req|
-      raise IOError
-    end
-  end
-
-  def stub_post_logs(url=DEFAULT_URL)
-    stub_request(:post, url)
-      .with(
-        body: "[{\"field1\":26,\"field2\":\"value26\"},{\"field1\":27,\"field2\":\"value27\"},{\"field1\":28,\"field2\":\"value28\"}]",
-        headers: {
-          'Authorization'=>'Bearer EdaNNN68y',
-  	      'Connection'=>'Keep-Alive',
-          'Content-Type'=>'application/json',
-          'Format'=>'syslog',
-          'Structure'=>'simple'
-        })
-      .to_return(:status => 200, :body => "ok")
-  end
-
-  def sample_record()
-    [
-      {'field1' => 26, 'field2' => 'value26'},
-      {'field1' => 27, 'field2' => 'value27'},
-      {'field1' => 28, 'field2' => 'value28'}
-    ]
-  end
-
-  def create_http_client
-    Fluent::Plugin::HttpClient.new(
-      DEFAULT_URL, true, 
-        {
-            'Authorization'=>'Bearer EdaNNN68y',
-            'Connection'=>'Keep-Alive',
-            'Content-Type'=>'application/json',
-            'Format'=>'syslog',
-            'Host'=>'local.endpoint:3000',
-            'Structure'=>'simple'
-        }, [500, 510], 60, 60, Logger.new(STDOUT))
-  end
-
-  def test_retry_on_response_status_code
-    http_client = create_http_client()
-    stub_server_returns_500
-    assert_raise RuntimeError do
-      http_client.post(JSON.dump(sample_record()))
-    end
-  end
-
-  def test_server_raise_error
-    http_client = create_http_client()
-    stub_server_raise_error
-    assert_raise IOError do
-      http_client.post(JSON.dump(sample_record()))
-    end
-  end
-
-  def test_post_logs
-    stub_post_logs
-    http_client = create_http_client()
-    http_client.post(JSON.dump(sample_record()))
-
-    stub_post_logs
-    http_client.post(JSON.dump(sample_record()))
-  end
-end
+# Copyright 2018 VMware, Inc.
+# SPDX-License-Identifier: MIT
+
+require 'helper'
+
+class HttpClientTest < Test::Unit::TestCase
+  DEFAULT_URL = "https://local.endpoint:3000/dummy/xyz"
+
+  def stub_server_out_of_quota(url=DEFAULT_URL)
+    stub_request(:post, url).to_return(:status => [429, "User out of ingestion quota."])
+  end
+
+  def test_check_quota
+    http_client = create_http_client()
+    assert_equal http_client.check_quota, true
+
+    stub_server_out_of_quota
+    http_client.post(JSON.dump(sample_record()))
+    assert_equal http_client.check_quota, false
+  end
+
+  def stub_server_unavailable(url=DEFAULT_URL)
+    stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
+  end
+
+  def stub_server_returns_500(url=DEFAULT_URL)
+    stub_request(:post, url).to_return(:status => [500, "Internal service error"])
+  end
+
+  def stub_server_raise_error(url=DEFAULT_URL)
+    stub_request(:post, url).with do |req|
+      raise IOError
+    end
+  end
+
+  def stub_post_logs(url=DEFAULT_URL)
+    stub_request(:post, url)
+      .with(
+        body: "[{\"field1\":26,\"field2\":\"value26\"},{\"field1\":27,\"field2\":\"value27\"},{\"field1\":28,\"field2\":\"value28\"}]",
+        headers: {
+          'Authorization'=>'Bearer EdaNNN68y',
+  	      'Connection'=>'Keep-Alive',
+          'Content-Type'=>'application/json',
+          'Format'=>'syslog',
+          'Structure'=>'simple'
+        })
+      .to_return(:status => 200, :body => "ok")
+  end
+
+  def sample_record()
+    [
+      {'field1' => 26, 'field2' => 'value26'},
+      {'field1' => 27, 'field2' => 'value27'},
+      {'field1' => 28, 'field2' => 'value28'}
+    ]
+  end
+
+  def create_http_client
+    Fluent::Plugin::HttpClient.new(
+      DEFAULT_URL, true, 
+        {
+            'Authorization'=>'Bearer EdaNNN68y',
+            'Connection'=>'Keep-Alive',
+            'Content-Type'=>'application/json',
+            'Format'=>'syslog',
+            'Host'=>'local.endpoint:3000',
+            'Structure'=>'simple'
+        }, [500, 510], 60, 60, Logger.new(STDOUT))
+  end
+
+  def test_retry_on_response_status_code
+    http_client = create_http_client()
+    stub_server_returns_500
+    assert_raise RuntimeError do
+      http_client.post(JSON.dump(sample_record()))
+    end
+  end
+
+  def test_server_raise_error
+    http_client = create_http_client()
+    stub_server_raise_error
+    assert_raise IOError do
+      http_client.post(JSON.dump(sample_record()))
+    end
+  end
+
+  def test_post_logs
+    stub_post_logs
+    http_client = create_http_client()
+    http_client.post(JSON.dump(sample_record()))
+
+    stub_post_logs
+    http_client.post(JSON.dump(sample_record()))
+  end
+end