fluent-plugin-sumologic_output 1.7.1 → 1.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38f85777973860ce39f213613b943139ae0304be0b69e13b2b1cde495a08e2d5
-  data.tar.gz: b4838ac669ee2ae899b06236d57a6cf01144370b9f19ae0fe56f900051058936
+  metadata.gz: 8ecb31a6366d5d0d352759cd11053747029c07d65267d410c5dfb4826b487ad8
+  data.tar.gz: 43b5911ccfeecd4e9d5a3f56b040678fb5e089a1793668b61559544e73427697
 SHA512:
-  metadata.gz: 604cc97685ba67fe6bbee0dd519b4228f66a4ca13ef721371cc8adb56d253f0a4f710ff1f00270675ce8e95526d951d22a4cc2ab0e03c17168fc7021d8c39c71
-  data.tar.gz: 61ab2d7c22d5f41d17cf3a4427db0f4c352ee5863b47b40b960279262ef047a5203e8ef98af5626ec0951625b520382a1003db7f9a320b1b408ffd5f573d67c4
+  metadata.gz: 590654de132ef18f36d6ef3671e7fec0a6b813c7e23beb2c602fd259521e405ef043cdefd2ba96b6454c72d91d88e9be9b3baa89467893c9448e36003a09655b
+  data.tar.gz: 0ab2c898106c02e8cb01af1f34231d230528378dd71173c569521be35c5b8427951f0219ff049b114f912d9fc2e92924b9023159c2fd187425cbf7b16fccd4e0

data/.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# Learn about CODEOWNERS file format:
+#  https://help.github.com/en/articles/about-code-owners
+
+* @SumoLogic/open-source-collection-team
+

data/.gitignore

@@ -2,4 +2,5 @@
 *.rbc
 Gemfile.lock
 .idea/
-TAGS
+TAGS
+.vagrant

data/CHANGELOG.md

@@ -2,6 +2,15 @@
 
 All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
 
+<a name="1.7.3"></a>
+# [1.7.3] (2021-10-19)
+- Expose httpclient send_timeout [#66](https://github.com/SumoLogic/fluentd-output-sumologic/pull/68)
+- Fix json parsing [#69](https://github.com/SumoLogic/fluentd-output-sumologic/pull/69)
+
+<a name="1.7.2"></a>
+# [1.7.2] (2020-11-23)
+- Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)
+
 <a name="1.7.1"></a>
 # [1.7.1] (2020-04-28)
 - Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)

data/README.md

@@ -28,6 +28,7 @@ Configuration options for fluent.conf are:
   * json_merge - Same as json but merge content of `log_key` into the top level and strip `log_key`
 * `log_key` - Used to specify the key when merging json or sending logs in text format (default `message`)
 * `open_timeout` - Set timeout seconds to wait until connection is opened.
+* `send_timeout` - Timeout for sending to SumoLogic in seconds. Don't modify unless you see `HTTPClient::SendTimeoutError` in your Fluentd logs. (default `120`)
 * `add_timestamp` - Add `timestamp` (or `timestamp_key`) field to logs before sending to sumologic (default `true`)
 * `timestamp_key` - Field name when `add_timestamp` is on (default `timestamp`)
 * `proxy_uri` - Add the `uri` of the `proxy` environment if present.

data/Vagrantfile

@@ -0,0 +1,27 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+unless Vagrant.has_plugin?("vagrant-disksize")
+  puts "vagrant-disksize plugin unavailable\n" +
+       "please install it via 'vagrant plugin install vagrant-disksize'"
+  exit 1
+end
+
+Vagrant.configure('2') do |config|
+  config.vm.box = 'ubuntu/focal64'
+  config.disksize.size = '50GB'
+  config.vm.box_check_update = false
+  config.vm.host_name = 'fluentd-output-sumologic'
+  config.vm.network :private_network, ip: "192.168.78.45"
+
+  config.vm.provider 'virtualbox' do |vb|
+    vb.gui = false
+    vb.cpus = 8
+    vb.memory = 16384
+    vb.name = 'fluentd-output-sumologic'
+  end
+
+  config.vm.provision 'shell', path: 'vagrant/provision.sh'
+
+  config.vm.synced_folder ".", "/sumologic"
+end

data/fluent-plugin-sumologic_output.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sumologic_output"
-  gem.version       = "1.7.1"
+  gem.version       = "1.7.4"
   gem.authors       = ["Steven Adams", "Frank Reno"]
   gem.email         = ["stevezau@gmail.com", "frank.reno@me.com"]
   gem.description   = %q{Output plugin to SumoLogic HTTP Endpoint}

data/lib/fluent/plugin/out_sumologic.rb

@@ -1,5 +1,6 @@
 require 'fluent/plugin/output'
 require 'net/https'
+require 'json'
 require 'yajl'
 require 'httpclient'
 require 'zlib'
@@ -12,12 +13,13 @@ class SumologicConnection
   COMPRESS_DEFLATE = 'deflate'
   COMPRESS_GZIP = 'gzip'
 
-  def initialize(endpoint, verify_ssl, connect_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding)
+  def initialize(endpoint, verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding, logger)
     @endpoint = endpoint
     @sumo_client = sumo_client
-    create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
+    create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
     @compress = compress_enabled
     @compress_encoding = (compress_encoding ||= COMPRESS_GZIP).downcase
+    @logger = logger
 
     unless [COMPRESS_DEFLATE, COMPRESS_GZIP].include? @compress_encoding
       raise "Invalid compression encoding #{@compress_encoding} must be gzip or deflate"
@@ -29,6 +31,29 @@ class SumologicConnection
     unless response.ok?
       raise RuntimeError, "Failed to send data to HTTP Source. #{response.code} - #{response.body}"
     end
+
+    # response is 20x, check response content
+    return if response.content.length == 0
+    
+    # if we get a non-empty response, check it 
+    begin
+      response_map = JSON.load(response.content)
+    rescue JSON::ParserError
+      @logger.warn "Error decoding receiver response: #{response.content}"
+      return
+    end
+
+    # log a warning with the present keys
+    response_keys = ["id", "code", "status", "message", "errors"]
+    log_params = []
+    response_keys.each do |key|
+      if response_map.has_key?(key) then
+        value = response_map[key]
+        log_params.append("#{key}: #{value}")
+      end
+    end
+    log_params_str = log_params.join(", ")
+    @logger.warn "There was an issue sending data: #{log_params_str}"
   end
 
   def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields, dimensions)
@@ -69,10 +94,11 @@ class SumologicConnection
     verify_ssl==true ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
   end
 
-  def create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
+  def create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
     @http                        = HTTPClient.new(proxy_uri)
     @http.ssl_config.verify_mode = ssl_options(verify_ssl)
     @http.connect_timeout        = connect_timeout
+    @http.send_timeout           = send_timeout
     if disable_cookies
       @http.cookie_manager       = nil
     end
@@ -126,6 +152,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   config_param :verify_ssl, :bool, :default => true
   config_param :delimiter, :string, :default => "."
   config_param :open_timeout, :integer, :default => 60
+  config_param :send_timeout, :integer, :default => 120
   config_param :add_timestamp, :bool, :default => true
   config_param :timestamp_key, :string, :default => 'timestamp'
   config_param :proxy_uri, :string, :default => nil
@@ -210,11 +237,13 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
       conf['endpoint'],
       conf['verify_ssl'],
       conf['open_timeout'].to_i,
+      conf['send_timeout'].to_i,
       conf['proxy_uri'],
       conf['disable_cookies'],
       conf['sumo_client'],
       conf['compress'],
-      conf['compress_encoding']
+      conf['compress_encoding'],
+      log,
       )
     super
   end
@@ -249,8 +278,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   def dump_log(log)
     log.delete('_sumo_metadata')
     begin
-      parser = Yajl::Parser.new
-      hash = parser.parse(log[@log_key])
+      hash = JSON.parse(log[@log_key])
       log[@log_key] = hash
       Yajl.dump(log)
     rescue
@@ -293,6 +321,19 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
     time.to_s.length == 13 ? time : time * 1000
   end
 
+  # Convert log to string and strip it
+  def log_to_str(log)
+    if log.is_a?(Array) or log.is_a?(Hash)
+      log = Yajl.dump(log)
+    end
+
+    unless log.nil?
+      log.strip!
+    end
+
+    return log
+  end
+
   # This method is called every flush interval. Write the buffer chunk
   def write(chunk)
     messages_list = {}
@@ -313,10 +354,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
       when 'logs'
         case log_format
         when 'text'
-          log = record[@log_key]
-          unless log.nil?
-            log.strip!
-          end
+          log = log_to_str(record[@log_key])
         when 'json_merge'
           if @add_timestamp
             record = { @timestamp_key => sumo_timestamp(time) }.merge(record)
@@ -334,10 +372,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
           log = dump_log(record)
         end
       when 'metrics'
-        log = record[@log_key]
-        unless log.nil?
-          log.strip!
-        end
+        log = log_to_str(record[@log_key])
       end
 
       unless log.nil?

data/test/plugin/test_out_sumologic.rb

@@ -671,4 +671,158 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
-end
+  def test_emit_text_from_array
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      text
+      source_category test
+      source_host     test
+      source_name     test
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'bar', 'message' => ['test', 'test2']})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: '["test","test2"]',
+                     times:1
+  end
+
+  def test_emit_text_from_dict
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      text
+      source_category test
+      source_host     test
+      source_name     test
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'bar', 'message' => {'test': 'test2', 'test3': 'test4'}})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: '{"test":"test2","test3":"test4"}',
+                     times:1
+  end
+
+  def test_emit_fields_string_based
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"}'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":{"foo":"bar","message":"test"}}\z/,
+                     times:1
+  end
+  
+  def test_emit_fields_invalid_json_string_based_1
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\": \\"test\\""}\z/,
+                     times:1
+  end
+  
+  def test_emit_fields_invalid_json_string_based_2
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"foo": "bar", "message"'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\""}\z/,
+                     times:1
+  end
+  
+  def test_emit_fields_invalid_json_string_based_3
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '"foo\": \"bar\", \"mess'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":"\\"foo\\\\\\": \\\\\\"bar\\\\\\", \\\\\\"mess"}\z/,
+                     times:1
+  end
+
+  def test_warning_response_from_receiver
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      endpoint #{endpoint}
+    }
+    testdata = [
+      [
+        '{"id":"1TIRY-KGIVX-TPQRJ","errors":[{"code":"internal.error","message":"Internal server error."}]}', 
+        'There was an issue sending data: id: 1TIRY-KGIVX-TPQRJ, errors: [{"code"=>"internal.error", "message"=>"Internal server error."}]'
+      ],
+      [
+        '{"id":"1TIRY-KGIVX-TPQRX","code": 200, "status": "Fields dropped", "message": "Dropped fields above the 30 field limit"}', 
+        'There was an issue sending data: id: 1TIRY-KGIVX-TPQRX, code: 200, status: Fields dropped, message: Dropped fields above the 30 field limit'
+      ],
+    ]
+    time = event_time
+
+    testdata.each do |data, log|
+      driver = create_driver(config)
+      stub_request(:post, endpoint).to_return(body: data, headers: {content_type: 'application/json'})
+      driver.run do
+        driver.feed("test", time, {"message": "test"})
+      end
+      assert_equal driver.logs.length, 1
+      assert driver.logs[0].end_with?(log + "\n")
+    end
+  end
+
+end

data/vagrant/README.md

@@ -0,0 +1,43 @@
+# Vagrant
+
+## Prerequisites
+
+Please install the following:
+
+- [VirtualBox](https://www.virtualbox.org/)
+- [Vagrant](https://www.vagrantup.com/)
+- [vagrant-disksize](https://github.com/sprotheroe/vagrant-disksize) plugin
+
+### MacOS
+
+```bash
+brew cask install virtualbox
+brew cask install vagrant
+vagrant plugin install vagrant-disksize
+```
+
+## Setting up
+
+You can set up the Vagrant environment with just one command:
+
+```bash
+vagrant up
+```
+
+After successfull installation you can ssh to the virtual machine with:
+
+```bash
+vagrant ssh
+```
+
+NOTICE: The directory with fluentd-output-sumologic repository on the host is synced with `/sumologic/` directory on the virtual machine.
+
+## Runing tests
+
+You can run tests using following commands:
+
+```bash
+cd /sumologic
+bundle install
+bundle exec rake
+```

data/vagrant/provision.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -x
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get --yes upgrade
+apt-get --yes install apt-transport-https
+
+echo "export EDITOR=vim" >> /home/vagrant/.bashrc
+
+# Install docker
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+add-apt-repository \
+   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+   $(lsb_release -cs) \
+   stable"
+apt-get install -y docker-ce docker-ce-cli containerd.io
+usermod -aG docker vagrant
+
+# Install make
+apt-get install -y make
+
+# install requirements for ruby
+snap install ruby --channel=2.6/stable --classic
+su vagrant -c 'gem install bundler:2.1.4'
+apt install -y gcc g++ libsnappy-dev libicu-dev zlib1g-dev cmake pkg-config libssl-dev

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sumologic_output
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.7.4
 platform: ruby
 authors:
 - Steven Adams
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-28 00:00:00.000000000 Z
+date: 2022-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -103,6 +103,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
 - ".gitignore"
 - ".travis.yml"
 - CHANGELOG.md
@@ -110,11 +111,14 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- Vagrantfile
 - ci/build.sh
 - fluent-plugin-sumologic_output.gemspec
 - lib/fluent/plugin/out_sumologic.rb
 - test/helper.rb
 - test/plugin/test_out_sumologic.rb
+- vagrant/README.md
+- vagrant/provision.sh
 homepage: https://github.com/SumoLogic/fluentd-output-sumologic
 licenses:
 - Apache-2.0
@@ -134,8 +138,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Output plugin to SumoLogic HTTP Endpoint