fluent-plugin-sumologic_output 1.7.2 → 1.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3bf755e8ecd488457805d948614c43b3657de815889dd341091d11f338ad995
-  data.tar.gz: 20163f7c5ac543a010e53fce506dbaed10bafe5d46830579a3101558f5b61fa4
+  metadata.gz: 52b0c77e900d61ba360421933440383208033b29b0f08b012250519189872f24
+  data.tar.gz: ff86f952744e650e30ba94d1185f4d270a3d8d6e7bfee5f30a0810e043a9483f
 SHA512:
-  metadata.gz: 35cf3af9a561fd22e810dea880570007613b7cb9ae87307dd46ea3ff8d9e005b96c9ebf19f1aafd58b6ae4495509f51406b7788029f7fb26144f54b60f22370a
-  data.tar.gz: 6b5e6a13ed50d798c4c0f8225e771a2c5977373373aed316cbe1a5369a2af22aef56c86e474aefd6ca847e41c496c287a191c23c1224d2a49075794b420949e4
+  metadata.gz: 74c43d262f35e2767d170a05dbec58911590c0851a724f6c4adb0047a26601383e4d8fa5e2671c845bbd89a65781fb0d35e4bc2f8d7ed6bd0f0e376d6c7ec958
+  data.tar.gz: 73b8ea2a6dbf4593eca7d4a702eeea14f27728bd7aa28ddc15e968301a5a2eb21f76ae18f1be2dd759f42cb357c2ceda92aec49e136755d76c954ebded889cdd

data/.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# Learn about CODEOWNERS file format:
+#  https://help.github.com/en/articles/about-code-owners
+
+* @SumoLogic/open-source-collection-team
+

data/.gitignore

@@ -2,4 +2,5 @@
 *.rbc
 Gemfile.lock
 .idea/
-TAGS
+TAGS
+.vagrant

data/README.md

@@ -28,6 +28,7 @@ Configuration options for fluent.conf are:
   * json_merge - Same as json but merge content of `log_key` into the top level and strip `log_key`
 * `log_key` - Used to specify the key when merging json or sending logs in text format (default `message`)
 * `open_timeout` - Set timeout seconds to wait until connection is opened.
+* `send_timeout` - Timeout for sending to SumoLogic in seconds. Don't modify unless you see `HTTPClient::SendTimeoutError` in your Fluentd logs. (default `120`)
 * `add_timestamp` - Add `timestamp` (or `timestamp_key`) field to logs before sending to sumologic (default `true`)
 * `timestamp_key` - Field name when `add_timestamp` is on (default `timestamp`)
 * `proxy_uri` - Add the `uri` of the `proxy` environment if present.

data/Vagrantfile

@@ -0,0 +1,27 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+unless Vagrant.has_plugin?("vagrant-disksize")
+  puts "vagrant-disksize plugin unavailable\n" +
+       "please install it via 'vagrant plugin install vagrant-disksize'"
+  exit 1
+end
+
+Vagrant.configure('2') do |config|
+  config.vm.box = 'ubuntu/focal64'
+  config.disksize.size = '50GB'
+  config.vm.box_check_update = false
+  config.vm.host_name = 'fluentd-output-sumologic'
+  config.vm.network :private_network, ip: "192.168.78.45"
+
+  config.vm.provider 'virtualbox' do |vb|
+    vb.gui = false
+    vb.cpus = 8
+    vb.memory = 16384
+    vb.name = 'fluentd-output-sumologic'
+  end
+
+  config.vm.provision 'shell', path: 'vagrant/provision.sh'
+
+  config.vm.synced_folder ".", "/sumologic"
+end

data/fluent-plugin-sumologic_output.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sumologic_output"
-  gem.version       = "1.7.2"
+  gem.version       = "1.7.3"
   gem.authors       = ["Steven Adams", "Frank Reno"]
   gem.email         = ["stevezau@gmail.com", "frank.reno@me.com"]
   gem.description   = %q{Output plugin to SumoLogic HTTP Endpoint}

data/lib/fluent/plugin/out_sumologic.rb

@@ -1,5 +1,6 @@
 require 'fluent/plugin/output'
 require 'net/https'
+require 'json'
 require 'yajl'
 require 'httpclient'
 require 'zlib'
@@ -12,10 +13,10 @@ class SumologicConnection
   COMPRESS_DEFLATE = 'deflate'
   COMPRESS_GZIP = 'gzip'
 
-  def initialize(endpoint, verify_ssl, connect_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding)
+  def initialize(endpoint, verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding)
     @endpoint = endpoint
     @sumo_client = sumo_client
-    create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
+    create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
     @compress = compress_enabled
     @compress_encoding = (compress_encoding ||= COMPRESS_GZIP).downcase
 
@@ -69,10 +70,11 @@ class SumologicConnection
     verify_ssl==true ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
   end
 
-  def create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
+  def create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
     @http                        = HTTPClient.new(proxy_uri)
     @http.ssl_config.verify_mode = ssl_options(verify_ssl)
     @http.connect_timeout        = connect_timeout
+    @http.send_timeout           = send_timeout
     if disable_cookies
       @http.cookie_manager       = nil
     end
@@ -126,6 +128,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   config_param :verify_ssl, :bool, :default => true
   config_param :delimiter, :string, :default => "."
   config_param :open_timeout, :integer, :default => 60
+  config_param :send_timeout, :integer, :default => 120
   config_param :add_timestamp, :bool, :default => true
   config_param :timestamp_key, :string, :default => 'timestamp'
   config_param :proxy_uri, :string, :default => nil
@@ -210,6 +213,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
       conf['endpoint'],
       conf['verify_ssl'],
       conf['open_timeout'].to_i,
+      conf['send_timeout'].to_i,
       conf['proxy_uri'],
       conf['disable_cookies'],
       conf['sumo_client'],
@@ -249,8 +253,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   def dump_log(log)
     log.delete('_sumo_metadata')
     begin
-      parser = Yajl::Parser.new
-      hash = parser.parse(log[@log_key])
+      hash = JSON.parse(log[@log_key])
       log[@log_key] = hash
       Yajl.dump(log)
     rescue

data/test/plugin/test_out_sumologic.rb

@@ -713,4 +713,88 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
+  def test_emit_fields_string_based
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"}'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":{"foo":"bar","message":"test"}}\z/,
+                     times:1
+  end
+  
+  def test_emit_fields_invalid_json_string_based_1
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\": \\"test\\""}\z/,
+                     times:1
+  end
+  
+  def test_emit_fields_invalid_json_string_based_2
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"foo": "bar", "message"'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\""}\z/,
+                     times:1
+  end
+  
+  def test_emit_fields_invalid_json_string_based_3
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '"foo\": \"bar\", \"mess'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":"\\"foo\\\\\\": \\\\\\"bar\\\\\\", \\\\\\"mess"}\z/,
+                     times:1
+  end  
+
 end

data/vagrant/README.md

@@ -0,0 +1,43 @@
+# Vagrant
+
+## Prerequisites
+
+Please install the following:
+
+- [VirtualBox](https://www.virtualbox.org/)
+- [Vagrant](https://www.vagrantup.com/)
+- [vagrant-disksize](https://github.com/sprotheroe/vagrant-disksize) plugin
+
+### MacOS
+
+```bash
+brew cask install virtualbox
+brew cask install vagrant
+vagrant plugin install vagrant-disksize
+```
+
+## Setting up
+
+You can set up the Vagrant environment with just one command:
+
+```bash
+vagrant up
+```
+
+After successfull installation you can ssh to the virtual machine with:
+
+```bash
+vagrant ssh
+```
+
+NOTICE: The directory with fluentd-output-sumologic repository on the host is synced with `/sumologic/` directory on the virtual machine.
+
+## Runing tests
+
+You can run tests using following commands:
+
+```bash
+cd /sumologic
+bundle install
+bundle exec rake
+```

data/vagrant/provision.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -x
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get --yes upgrade
+apt-get --yes install apt-transport-https
+
+echo "export EDITOR=vim" >> /home/vagrant/.bashrc
+
+# Install docker
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+add-apt-repository \
+   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+   $(lsb_release -cs) \
+   stable"
+apt-get install -y docker-ce docker-ce-cli containerd.io
+usermod -aG docker vagrant
+
+# Install make
+apt-get install -y make
+
+# install requirements for ruby
+snap install ruby --channel=2.6/stable --classic
+su vagrant -c 'gem install bundler:2.1.4'
+apt install -y gcc g++ libsnappy-dev libicu-dev zlib1g-dev cmake pkg-config libssl-dev

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sumologic_output
 version: !ruby/object:Gem::Version
-  version: 1.7.2
+  version: 1.7.3
 platform: ruby
 authors:
 - Steven Adams
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-23 00:00:00.000000000 Z
+date: 2021-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -103,6 +103,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
 - ".gitignore"
 - ".travis.yml"
 - CHANGELOG.md
@@ -110,11 +111,14 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- Vagrantfile
 - ci/build.sh
 - fluent-plugin-sumologic_output.gemspec
 - lib/fluent/plugin/out_sumologic.rb
 - test/helper.rb
 - test/plugin/test_out_sumologic.rb
+- vagrant/README.md
+- vagrant/provision.sh
 homepage: https://github.com/SumoLogic/fluentd-output-sumologic
 licenses:
 - Apache-2.0