fluent-plugin-vmware-aria-log-intelligence 2.0.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 46225b14815eaaa0bbbf25007488eb19c4ab58f7fba5adeb2256888a9bec1371
+  data.tar.gz: 93123202b248e40de0b0d199d875a2291731d478ca6658c3e0bfeb9b18cb6f27
+SHA512:
+  metadata.gz: 653bb8fe0202fb5557f569fd356df5ae8e0d6a5eb5cd7afe37ba0cb35cffb98f9acc86275b448b6d760b2c215c48e3492a47a351da27e518a9da1af712a9ecdc
+  data.tar.gz: 9c321a8e781a8cc1fa29ae26cc96e5866bc7a8f67530366501e46f8af4c9a08701ad8fb5633211aa8b08525b030a71047f115f100ca19d80c7ae8cc902e4a9c5

data/Gemfile

@@ -0,0 +1,11 @@
+# Copyright (c) 2013 ablagoev
+# Copyright 2018 VMware, Inc.
+# SPDX-License-Identifier: MIT
+
+source "http://rubygems.org"
+
+gemspec
+
+group :test do
+  gem 'coveralls', require: false
+end

data/LICENSE.txt

@@ -0,0 +1,15 @@
+fluent-plugin-vmware-log-intelligence
+
+Copyright (c) 2013 ablagoev
+Copyright (c) 2018 VMware, Inc.				
+
+The MIT license (the "License") set forth below applies to all parts of the fluent-plugin-vmware-log-intelligence project.  You may not use this file except in compliance with the License.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,113 @@
+= Overview
+
+vmware_log_intelligence is a Fluentd output plugin that buffers data and sends the data to VMware Log Intelligence.
+
+== Requirements
+
+fluentd version: 0.14.0 or above
+
+Ruby version: 2.2 or above
+
+== Build and install the plugin
+
+`gem build fluent-plugin-vmware-log-intelligence.gemspec`
+
+`gem install fluent-plugin-vmware-log-intelligence-1.0.0.gem`
+
+== Usage
+vmware_log_intelligence does not support Fluentd v0.12 style configurations.
+Please use Fluentd v1.0 style configurations when working with the plugin.
+
+Here is an example of the vmware_log_intelligence plugin configurations:
+
+  <match pattern>
+    @type vmware_log_intelligence
+    endpoint_url https://data.upgrade.symphony-dev.com/le-mans/v1/streams/ingestion-pipeline-stream
+    verify_ssl false
+    http_compress false
+    <headers>
+      Content-Type application/json
+      Authorization Bearer Your-VMware-Log-Intelligence-API-Token
+      structure simple
+    </headers>
+    <buffer>
+        @type file
+        path /path/to/buffer/lint.buf
+        chunk_limit_records 300
+        flush_interval 20s
+        delayed_commit_timeout 120s
+        retry_max_times 3
+      </buffer>
+    <format>
+      @type json
+      tag_key text
+    </format>
+  </match>
+
+You can find more configuration examples in the files under the examples folder.
+
+Besides Fluentd, you can also use the plugin with td-agent.
+
+== Payload format
+VMware Log Intelligence expects the body of the HTTP requests you send to be in JSON format. It also expects the JSON payload to have a 'text' field that contains your raw log text.
+
+The fluentd way of turning raw log text into a JSON is to use the JSON formatter plugin (https://docs.fluentd.org/v0.12/articles/formatter_json). The plugin uses a 'tag' field to store the text of a log. One configuration option of the JSON formatter plugin is called 'tag_key'. You need to set 'tag_key' to 'text' so that the JSON formatter will generate JSON payloads that conform to what VMware Log Intelligence expects.
+
+== Configuration
+
+=== endpoint_url
+This is the URL to which you want to send the logs to. You should set it to the URL of VMware Log Intelligence:
+https://data.upgrade.symphony-dev.com/le-mans/v1/streams/ingestion-pipeline-stream
+
+=== verify_ssl
+You can use this setting to control whether the vmware_log_intelligence plugin should verify
+the SSL certificate of the endpoint_url. You should always set this to `true` except in testing
+or troubleshooting.
+
+=== http_compress
+You can use this setting to compress the data while sending it to vRealize Log Insight Cloud. The default value is set 'false' here, If needed you can set it 'true'.   
+
+=== <headers>
+In order to send logs to VMware Log Intelligence, you need to have the `<headers>`
+section in your configuration. The `<headers>` section should look like the following:
+
+  <headers>
+    Content-Type application/json
+    Authorization Bearer Your-VMware-Log-Intelligence-API-Token
+    structure simple
+  </headers>
+
+=== <buffer>
+The vmware_log_intelligence plugin supports buffering. All the buffering configurations
+of Fluentd are supported. Detailed information of the buffering configurations can be
+found here https://docs.fluentd.org/v1.0/articles/buffer-section.
+
+== Run tests
+If you would like to contribute to this project, you might want to be able to run the tests
+under the `test` folder. You can do that in a Ruby environment you have set up on your dev
+machine. You can also run the tests in a Docker container. To run the tests in a Docker container,
+you can follow these steps:
+
+Go to the root folder of this project and build the Docker image like this:
+`docker build -t fluent-plugin-dev .`
+
+Run the container like this:
+`docker run --rm -i -t --name fluent-plugin-dev -v $(pwd):/app fluent-plugin-dev /bin/bash`
+
+Once inside the container, you can run the following to make sure that all dependecies are installed:
+`bundle install`
+
+Then you can run tests like this:
+`bundle exec rake`
+
+== Contributing
+The vmware_log_intelligence plugin project team welcomes contributions from the community. Before you start working with fluent-plugin-vmware-log-intelligence, please read our [Developer Certificate of Origin](https://cla.vmware.com/dco). All contributions to this repository must be signed as described on that page. Your signature certifies that you wrote the patch or have the right to pass it on as an open-source patch. For more detailed information, refer to [CONTRIBUTING.md](CONTRIBUTING.md).
+
+== License
+Please see LICENSE.txt[LICENSE.txt].
+
+== Copyright
+
+Copyright (c) 2013 ablagoev.
+
+Copyright 2018 VMware, Inc.

data/Rakefile

@@ -0,0 +1,16 @@
+#!/usr/bin/env rake
+
+# Copyright (c) 2013 ablagoev
+# Copyright 2018 VMware, Inc.
+# SPDX-License-Identifier: MIT
+
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/VERSION

@@ -0,0 +1 @@
+2.0.7

data/fluent-plugin-vmware-log-intelligence.gemspec

@@ -0,0 +1,53 @@
+# Copyright (c) 2013 ablagoev
+# Copyright 2018 VMware, Inc.
+# SPDX-License-Identifier: MIT
+
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = 'fluent-plugin-vmware-aria-log-intelligence'
+  s.version     = File.read("VERSION").strip
+  s.date        = '2018-08-12'
+  s.summary     = "Fluentd buffered output plugin for VMware Log Intelligence"
+  s.description = "Send Fluentd buffered logs to VMware Log Intelligence"
+  s.authors     = ["Alexander Blagoev", "Chaur Wu"]
+  s.email       = 'gwu@vmware.com'
+  s.homepage    =
+    'http://github.com/vmware/fluent-plugin-vmware-log-intelligence'
+
+  s.files       = [
+    "lib/fluent/plugin/out_vmware_log_intelligence.rb",
+    "lib/fluent/plugin/http_client.rb",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "fluent-plugin-vmware-log-intelligence.gemspec",
+    "test/helper.rb",
+    "test/plugin/test_out_vmware_log_intelligence.rb",
+    "test/plugin/test_http_client.rb",
+  ]
+
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.licenses = ["MIT"]
+
+  s.require_paths = ['lib']
+
+  s.add_dependency "fluentd", ">= 0.14.20"
+  s.add_dependency "http", ">= 0.9.8"
+  s.add_dependency "myslog", "~> 0.0"
+  s.add_dependency "fluent-plugin-mysqlslowquery", ">= 0.0.9"
+  s.add_development_dependency "rake", ">= 0.9.2"
+  s.add_development_dependency "bundler", ">= 1.3.4"
+  s.add_development_dependency 'test-unit', '~> 3.1.0'
+  s.add_development_dependency 'webmock', '~> 3.4.0'
+  s.add_development_dependency 'fluent-plugin-detect-exceptions', '>= 0.0.12'
+  s.add_development_dependency 'fluent-plugin-concat', '>= 2.0.0'
+  s.add_development_dependency 'fluent-plugin-kubernetes_metadata_filter', '>= 2.0.0'
+  s.add_development_dependency 'fluent-plugin-multi-format-parser', '>= 1.0.0'
+  s.add_development_dependency 'fluent-plugin-postgres', '>= 0.0.1'
+end

data/lib/fluent/plugin/http_client.rb

@@ -0,0 +1,72 @@
+# Copyright 2018 VMware, Inc.
+# SPDX-License-Identifier: MIT
+
+module Fluent::Plugin
+  class HttpClient
+    def initialize(endpoint_url, verify_ssl, 
+      headers, statuses, open_timeout, read_timeout, log)
+      @log = log
+      @statuses = statuses
+      @options = {}
+
+      if !verify_ssl
+        @log.warn('SSL verification of the remote VMware Log Intelligence service is turned off. This is serious security risk. Please turn on SSL verification and restart the Fluentd/td-agent process.')
+        ctx = OpenSSL::SSL::SSLContext.new
+        ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        @options = {:ssl_context => ctx}
+      end
+
+      timeout_options = {
+        :connect_timeout => open_timeout,
+        :read_timeout => read_timeout
+      }
+        
+      @conn = HTTP.persistent(endpoint_url)
+        .headers(headers)
+        .timeout(timeout_options)
+
+      @endpoint_path = HTTP::URI.parse(endpoint_url).path
+      @last_429_time = nil
+    end
+
+    def check_quota
+      if @last_429_time
+        if (Time.new - @last_429_time) < 600
+          return false
+        end
+
+        @last_429_time = nil
+      end
+      return true
+    end
+
+    def post(data)
+      if !check_quota
+        return
+      end
+
+      begin
+        response = @conn.post(@endpoint_path, @options.merge(:body => data))
+        response.body.to_s
+        if (response.code == 429)
+          @log.warn('1GB quota of free account has been reached. Will stop sending data for 1 hour.')
+          @last_429_time = Time.new
+        else
+          @last_429_time = nil
+        end
+        
+        if @statuses.include? response.code.to_i
+          # Raise an exception so that fluent will retry based on the configurations.
+          fail "Server returned bad status: #{response.code}. #{response.to_s}"
+        end
+
+      rescue EOFError, SystemCallError, OpenSSL::SSL::SSLError => e
+        @log.warn "http post raises exception: #{e.class}, '#{e.message}'"
+      end
+    end
+
+    def close
+      @conn.close
+    end
+  end
+end

data/lib/fluent/plugin/out_vmware_log_intelligence.rb

@@ -0,0 +1,205 @@
+# Copyright (c) 2013 ablagoev
+# Copyright 2018 VMware, Inc.
+# SPDX-License-Identifier: MIT
+
+	
+require 'zlib'
+require "fluent/plugin/output"
+require "fluent/plugin/http_client"
+
+module Fluent::Plugin
+  class LogIntelligenceOutput < Output
+    Fluent::Plugin.register_output('vmware_log_intelligence', self)
+
+    config_param :http_compress, :bool, :default => false
+    config_param :endpoint_url, :string
+    config_param :bearer_token, :string, default: ''
+    config_param :http_retry_statuses, :string, default: ''
+    config_param :read_timeout, :integer, default: 60
+    config_param :open_timeout, :integer, default: 60
+    # config_param :use_ssl, :bool, :default => false
+    config_param :verify_ssl, :bool, :default => true
+    config_param :rate_limit_msec, :integer, :default => 0
+    # Keys from log event whose values should be added as log message/text
+    # to log-intelligence. Note these key/value pairs  won't be added as metadata/fields
+    config_param :log_text_keys, :array, default: ["log", "message", "msg"], value_type: :string
+    # Flatten hashes to create one key/val pair w/o losing log data
+    config_param :flatten_hashes, :bool, :default => true
+    # Seperator to use for joining flattened keys
+    config_param :flatten_hashes_separator, :string, :default => "_"
+
+    config_section :buffer do
+      config_set_default :@type, "memory"
+      config_set_default :chunk_keys, []
+      config_set_default :timekey_use_utc, true
+    end
+
+    def initialize
+      super
+      require 'http'
+      require 'uri'
+    end
+
+    def validate_uri(uri_string)
+      unless uri_string =~ /^#{URI.regexp}$/
+        fail Fluent::ConfigError, 'endpoint_url invalid'
+      end
+
+      begin
+        @uri = URI.parse(uri_string)
+      rescue URI::InvalidURIError
+        raise Fluent::ConfigError, 'endpoint_url invalid'
+      end
+    end
+
+    def retrieve_headers(conf)
+      headers = {}
+      conf.elements.each do |element|
+        if @http_compress
+          set_gzip_header(element)
+        end
+        if element.name == 'headers'
+          if @bearer_token != ''
+            element['Authorization'] = 'Bearer ' + @bearer_token
+          end
+          headers = element.to_hash
+        end
+      end
+      headers
+    end
+
+    def set_gzip_header(element)
+      element['Content-Encoding'] = 'gzip'
+      element
+    end
+
+    def shorten_key(key)
+      # LINT doesn't allow some characters in field 'name'
+      # like '/', '-', '\', '.', etc. so replace them with @flatten_hashes_separator
+      key = key.gsub(/[\/\.\-\\]/,@flatten_hashes_separator).downcase
+      key
+    end
+
+    def create_lint_event(record)
+      flattened_records = {}
+      merged_records = {}
+      if @flatten_hashes
+        flattened_records = flatten_record(record, [])
+      else
+        flattened_records = record
+      end
+
+      keys = []
+      log = ''
+      flattened_records.each do |key, value|
+        begin
+          next if value.nil?
+          # LINT doesn't support duplicate fields, make unique names by appending underscore
+          key = shorten_key(key)
+          if keys.include?(key)
+            value = merged_records[key] + " " + value
+          end
+          keys.push(key)
+          key.force_encoding("utf-8")
+
+          if value.is_a?(String) 
+            value.force_encoding("utf-8")
+          end
+        end
+
+        if @log_text_keys.include?(key)
+          if log != "#{value}"
+            if log.empty?
+              log = "#{value}"
+            else
+              log += " #{value}"
+            end
+          end
+        else
+          merged_records[key] = value
+        end
+      end
+      merged_records["text"] = log
+
+      if log == "\\n"
+        {}
+      else
+        merged_records
+      end
+    end
+
+    def flatten_record(record, prefix=[])
+      ret = {}
+
+      case record
+      when Hash
+        record.each do |key, value|
+          if @log_text_keys.include?(key)
+            ret.merge!({key.to_s => value})
+          else
+            ret.merge! flatten_record(value, prefix + [key.to_s])
+          end
+        end
+      when Array
+        record.each do |value|
+          ret.merge! flatten_record(value, prefix)
+        end
+      else
+        return {prefix.join(@flatten_hashes_separator) => record}
+      end
+      ret
+    end
+
+    def configure(conf)
+      super
+      validate_uri(@endpoint_url)
+
+      @statuses = @http_retry_statuses.split(',').map { |status| status.to_i }
+      @statuses = [] if @statuses.nil?
+
+      @headers = retrieve_headers(conf)
+
+      @http_client = Fluent::Plugin::HttpClient.new(
+        @endpoint_url, @verify_ssl, @headers, @statuses,
+        @open_timeout, @read_timeout, @log)
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+      begin
+        @http_client.close if @http_client
+      rescue
+      end
+    end
+
+    def write(chunk)
+      is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
+      if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
+        @log.info('Dropped request due to rate limiting')
+        return
+      end
+
+      data = []
+      chunk.each do |time, record|
+        data << create_lint_event(record)
+      end
+    
+      if @http_compress
+        gzip_body = Zlib::GzipWriter.new(StringIO.new)
+        gzip_body << data.to_json
+        @http_client.post(gzip_body.close.string)
+      else  
+        @last_request_time = Time.now.to_f
+        @http_client.post(JSON.dump(data))
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,31 @@
+# Copyright (c) 2013 ablagoev
+# Copyright 2018 VMware, Inc.
+# SPDX-License-Identifier: MIT
+
+require 'coveralls'
+Coveralls.wear!
+
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts 'Run `bundle install` to install missing gems'
+  exit e.status_code
+end
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'test-unit'
+require 'fluent/test'
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)
+
+require 'fluent/plugin/out_vmware_log_intelligence'
+require 'webmock/test_unit'
+WebMock.disable_net_connect!

data/test/plugin/test_http_client.rb

@@ -0,0 +1,95 @@
+# Copyright 2018 VMware, Inc.
+# SPDX-License-Identifier: MIT
+
+require 'helper'
+
+class HttpClientTest < Test::Unit::TestCase
+  DEFAULT_URL = "https://local.endpoint:3000/dummy/xyz"
+
+  def stub_server_out_of_quota(url=DEFAULT_URL)
+    stub_request(:post, url).to_return(:status => [429, "User out of ingestion quota."])
+  end
+
+  def test_check_quota
+    http_client = create_http_client()
+    assert_equal http_client.check_quota, true
+
+    stub_server_out_of_quota
+    http_client.post(JSON.dump(sample_record()))
+    assert_equal http_client.check_quota, false
+  end
+
+  def stub_server_unavailable(url=DEFAULT_URL)
+    stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
+  end
+
+  def stub_server_returns_500(url=DEFAULT_URL)
+    stub_request(:post, url).to_return(:status => [500, "Internal service error"])
+  end
+
+  def stub_server_raise_error(url=DEFAULT_URL)
+    stub_request(:post, url).with do |req|
+      raise IOError
+    end
+  end
+
+  def stub_post_logs(url=DEFAULT_URL)
+    stub_request(:post, url)
+      .with(
+        body: "[{\"field1\":26,\"field2\":\"value26\"},{\"field1\":27,\"field2\":\"value27\"},{\"field1\":28,\"field2\":\"value28\"}]",
+        headers: {
+          'Authorization'=>'Bearer EdaNNN68y',
+  	      'Connection'=>'Keep-Alive',
+          'Content-Type'=>'application/json',
+          'Format'=>'syslog',
+          'Structure'=>'simple'
+        })
+      .to_return(:status => 200, :body => "ok")
+  end
+
+  def sample_record()
+    [
+      {'field1' => 26, 'field2' => 'value26'},
+      {'field1' => 27, 'field2' => 'value27'},
+      {'field1' => 28, 'field2' => 'value28'}
+    ]
+  end
+
+  def create_http_client
+    Fluent::Plugin::HttpClient.new(
+      DEFAULT_URL, true, 
+        {
+            'Authorization'=>'Bearer EdaNNN68y',
+            'Connection'=>'Keep-Alive',
+            'Content-Type'=>'application/json',
+            'Format'=>'syslog',
+            'Host'=>'local.endpoint:3000',
+            'Structure'=>'simple'
+        }, [500, 510], 60, 60, Logger.new(STDOUT))
+  end
+
+  def test_retry_on_response_status_code
+    http_client = create_http_client()
+    stub_server_returns_500
+    assert_raise RuntimeError do
+      http_client.post(JSON.dump(sample_record()))
+    end
+  end
+
+  def test_server_raise_error
+    http_client = create_http_client()
+    stub_server_raise_error
+    assert_raise IOError do
+      http_client.post(JSON.dump(sample_record()))
+    end
+  end
+
+  def test_post_logs
+    stub_post_logs
+    http_client = create_http_client()
+    http_client.post(JSON.dump(sample_record()))
+
+    stub_post_logs
+    http_client.post(JSON.dump(sample_record()))
+  end
+end

data/test/plugin/test_out_vmware_log_intelligence.rb

@@ -0,0 +1,154 @@
+# Copyright 2018 VMware, Inc.
+# SPDX-License-Identifier: MIT
+
+require 'helper'
+require 'yaml'
+
+class LogIntelligenceOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::LogIntelligenceOutput).configure(conf)
+  end
+
+  def test_configure
+    config = %[
+     endpoint_url  http://localhost:9200/li
+   ]
+
+    instance = create_driver(config).instance
+    assert_equal 'http://localhost:9200/li', instance.endpoint_url
+    assert_equal '', instance.instance_eval { @http_retry_statuses }
+    assert_equal [], instance.instance_eval { @statuses }
+    assert_equal 60, instance.instance_eval { @read_timeout }
+    assert_equal 60, instance.instance_eval { @open_timeout }
+    assert_equal({}, instance.instance_eval { @headers })
+  end
+
+  def test_full_configure
+    config = %[
+     @type http_buffered
+     endpoint_url https://local.endpoint:3000/dummy/xyz
+     verify_ssl false
+     <headers>
+       Content-Type application/json
+       Authorization Bearer EdaNNN68y
+       structure simple
+       format syslog
+     </headers>
+     <buffer>
+       chunk_limit_records 3
+       flush_interval 12s
+       retry_max_times 3
+     </buffer>
+   ]
+
+    instance = create_driver(config).instance
+    assert_equal 'https://local.endpoint:3000/dummy/xyz', instance.endpoint_url
+    assert_equal '', instance.http_retry_statuses
+    assert_equal [], instance.instance_eval { @statuses }
+    assert_equal 60, instance.read_timeout
+    assert_equal 60, instance.open_timeout
+    assert_equal({
+                     "Authorization"=>"Bearer EdaNNN68y",
+                     "Content-Type"=>"application/json",
+                     "structure"=>"simple",
+                     "format"=>"syslog"}, instance.instance_eval { @headers })
+  end
+
+  def test_invalid_endpoint
+    assert_raise Fluent::ConfigError do
+      create_driver('endpoint_url \\@3')
+    end
+
+    assert_raise Fluent::ConfigError do
+      create_driver('endpoint_url google.com')
+    end
+  end
+
+  def test_json_with_special_char1
+    $log = Logger.new(STDOUT)
+    $log.level = Logger::DEBUG
+    input = {"host" => "192.168.0.1", "log" => "2019-12-02 02:04:13.556787 I | mvcc: finished scheduled compaction at 2259 (took 822.573\xC2\xB5s)"}
+    output = [{"host" => "192.168.0.1", "text" => "2019-12-02 02:04:13.556787 I | mvcc: finished scheduled compaction at 2259 (took 822.573\xC2\xB5s)"}]
+    $log.debug("INPUT: #{input}")
+    $log.debug("OUTPUT: #{output}")
+    verify_write(input, output)
+  end
+  
+  def test_json_with_special_char2
+    $log = Logger.new(STDOUT)
+    $log.level = Logger::DEBUG
+    input = {"host" => "192.168.0.1", "log" => "2019-12-02 02:04:13.556787 I | mvcc: finished scheduled compaction at 2259 (took 678.559µs)"}
+    output = [{"host" => "192.168.0.1", "text" => "2019-12-02 02:04:13.556787 I | mvcc: finished scheduled compaction at 2259 (took 678.559µs)"}]
+    $log.debug("INPUT: #{input}")
+    $log.debug("OUTPUT: #{output}")
+    verify_write(input, output)
+  end
+
+  def test_json_with_log
+    input = {"host" => "192.168.0.1", "log" => "machine reboot"}
+    output = [{"host" => "192.168.0.1", "text" => "machine reboot"}]
+    verify_write(input, output)
+  end
+
+  def test_json_with_msg
+    input = {"host" => "192.168.0.1", "msg" => "machine reboot"}
+    output = [{"host" => "192.168.0.1", "text" => "machine reboot"}]
+    verify_write(input, output)
+  end
+
+  def test_nested_json
+    input = {"host" => "192.168.0.1", "properties" => {"type" => "windows"}, "msg" => "machine reboot"}
+    output = [{"host" => "192.168.0.1", "properties_type" => "windows", "text" => "machine reboot"}]
+    verify_write(input, output)
+  end
+
+  def test_nested_json_with_log
+    input = {"host" => "192.168.0.1", "properties" => {"type" => "windows"}, "message" => "machine reboot"}
+    output = [{"host" => "192.168.0.1", "properties_type" => "windows", "text" => "machine reboot"}]
+    verify_write(input, output)
+  end
+
+  def test_json_with_new_line
+    input = {"host" => "192.168.0.1", "log" => "\\n"}
+    output = [{}]
+    verify_write(input, output)
+  end
+
+  def test_json_with_multiple_log_formats
+    input = {"host" => "192.168.0.1", "log" => "custom log:1", "message" => "custom message:2", "msg" => "custom msg:3"}
+    output = [{"host" => "192.168.0.1", "text" => "custom log:1 custom message:2 custom msg:3"}]
+    verify_write(input, output)
+  end
+
+  # For any null values, its key and values are not being populated to output.
+  def test_json_with_null_value
+    input = {"host" => "192.168.0.1", "source" => nil, "log" => "abc"}
+    output = [{"host" => "192.168.0.1", "text" => "abc"}]
+    verify_write(input, output)
+  end
+
+  # like '/', '-', '\', '.', etc. replace with '_'
+  def test_json_with_sperators
+    input = {"host" => "192.168.0.1", "/properties" => {"type-" => "windows"}, "msg" => "123"}
+    output = [{"host" => "192.168.0.1", "_properties_type_" => "windows", "text" => "123"}]
+    verify_write(input, output)
+  end
+
+  def verify_write(input, output)
+    config = %[
+     endpoint_url  http://localhost:9200/li
+   ]
+    stub = stub_request(:post, "http://localhost:9200/li").with(body: output.to_json).
+        to_return(status: 200, body: "", headers: {})
+
+    driver = create_driver(config)
+    driver.run(default_tag: 'test') do
+      driver.feed(input)
+    end
+    assert_requested(stub)
+  end
+end

metadata

@@ -0,0 +1,238 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-vmware-aria-log-intelligence
+version: !ruby/object:Gem::Version
+  version: 2.0.7
+platform: ruby
+authors:
+- Alexander Blagoev
+- Chaur Wu
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-08-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.20
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.20
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.8
+- !ruby/object:Gem::Dependency
+  name: myslog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-mysqlslowquery
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.9
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.4
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-detect-exceptions
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.12
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.12
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-concat
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-kubernetes_metadata_filter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-multi-format-parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-postgres
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+description: Send Fluentd buffered logs to VMware Log Intelligence
+email: gwu@vmware.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.rdoc
+files:
+- Gemfile
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- fluent-plugin-vmware-log-intelligence.gemspec
+- lib/fluent/plugin/http_client.rb
+- lib/fluent/plugin/out_vmware_log_intelligence.rb
+- test/helper.rb
+- test/plugin/test_http_client.rb
+- test/plugin/test_out_vmware_log_intelligence.rb
+homepage: http://github.com/vmware/fluent-plugin-vmware-log-intelligence
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3.1
+signing_key: 
+specification_version: 4
+summary: Fluentd buffered output plugin for VMware Log Intelligence
+test_files: []