fluent-plugin-sumologic_output 1.7.2 → 1.7.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3bf755e8ecd488457805d948614c43b3657de815889dd341091d11f338ad995
-  data.tar.gz: 20163f7c5ac543a010e53fce506dbaed10bafe5d46830579a3101558f5b61fa4
+  metadata.gz: 10221368588a63fe0f97cb60695466a78a12eae80c13f736b5ae54879e4a5b4e
+  data.tar.gz: 2b99181908e9df7b2bcae1e6534a9311d72464cb74923a77f3a7489eeeb13f94
 SHA512:
-  metadata.gz: 35cf3af9a561fd22e810dea880570007613b7cb9ae87307dd46ea3ff8d9e005b96c9ebf19f1aafd58b6ae4495509f51406b7788029f7fb26144f54b60f22370a
-  data.tar.gz: 6b5e6a13ed50d798c4c0f8225e771a2c5977373373aed316cbe1a5369a2af22aef56c86e474aefd6ca847e41c496c287a191c23c1224d2a49075794b420949e4
+  metadata.gz: 245aa88c7c415099db251eabf2c6b9e80a64d657cdf4df90e365115e962cbeec76279cab1d4d5dc87de1b6387475c0e6363a2be24c55d21953d4bc852727d869
+  data.tar.gz: c9a870b1252ca73b2565a4627e29ee5501ce9da6caa88834932d0edee05e45f573a5f7aeae183968e303786615622487188287236b1d12d8234ac992f25e822a

data/.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# Learn about CODEOWNERS file format:
+#  https://help.github.com/en/articles/about-code-owners
+
+* @SumoLogic/open-source-collection-team
+

data/.gitignore

@@ -2,4 +2,5 @@
 *.rbc
 Gemfile.lock
 .idea/
-TAGS
+TAGS
+.vagrant

data/CHANGELOG.md

@@ -2,6 +2,15 @@
 
 All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
 
+<a name="1.7.3"></a>
+# [1.7.3] (2021-10-19)
+- Expose httpclient send_timeout [#66](https://github.com/SumoLogic/fluentd-output-sumologic/pull/68)
+- Fix json parsing [#69](https://github.com/SumoLogic/fluentd-output-sumologic/pull/69)
+
+<a name="1.7.2"></a>
+# [1.7.2] (2020-11-23)
+- Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)
+
 <a name="1.7.1"></a>
 # [1.7.1] (2020-04-28)
 - Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)

data/README.md

@@ -28,6 +28,7 @@ Configuration options for fluent.conf are:
   * json_merge - Same as json but merge content of `log_key` into the top level and strip `log_key`
 * `log_key` - Used to specify the key when merging json or sending logs in text format (default `message`)
 * `open_timeout` - Set timeout seconds to wait until connection is opened.
+* `send_timeout` - Timeout for sending to SumoLogic in seconds. Don't modify unless you see `HTTPClient::SendTimeoutError` in your Fluentd logs. (default `120`)
 * `add_timestamp` - Add `timestamp` (or `timestamp_key`) field to logs before sending to sumologic (default `true`)
 * `timestamp_key` - Field name when `add_timestamp` is on (default `timestamp`)
 * `proxy_uri` - Add the `uri` of the `proxy` environment if present.

data/Vagrantfile

@@ -0,0 +1,29 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+unless Vagrant.has_plugin?("vagrant-disksize")
+  puts "vagrant-disksize plugin unavailable\n" +
+       "please install it via 'vagrant plugin install vagrant-disksize'"
+  exit 1
+end
+
+Vagrant.configure('2') do |config|
+  config.vm.box = 'ubuntu/focal64'
+  config.disksize.size = '50GB'
+  config.vm.box_check_update = false
+  config.vm.host_name = 'fluentd-output-sumologic'
+  # Vbox 6.1.28+ restricts host-only adapters to 192.168.56.0/21
+  # See: https://www.virtualbox.org/manual/ch06.html#network_hostonly
+  config.vm.network :private_network, ip: "192.168.56.45"
+
+  config.vm.provider 'virtualbox' do |vb|
+    vb.gui = false
+    vb.cpus = 8
+    vb.memory = 16384
+    vb.name = 'fluentd-output-sumologic'
+  end
+
+  config.vm.provision 'shell', path: 'vagrant/provision.sh'
+
+  config.vm.synced_folder ".", "/sumologic"
+end

data/fluent-plugin-sumologic_output.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sumologic_output"
-  gem.version       = "1.7.2"
+  gem.version       = "1.7.5"
   gem.authors       = ["Steven Adams", "Frank Reno"]
   gem.email         = ["stevezau@gmail.com", "frank.reno@me.com"]
   gem.description   = %q{Output plugin to SumoLogic HTTP Endpoint}

data/lib/fluent/plugin/out_sumologic.rb

@@ -1,5 +1,6 @@
 require 'fluent/plugin/output'
 require 'net/https'
+require 'json'
 require 'yajl'
 require 'httpclient'
 require 'zlib'
@@ -12,12 +13,13 @@ class SumologicConnection
   COMPRESS_DEFLATE = 'deflate'
   COMPRESS_GZIP = 'gzip'
 
-  def initialize(endpoint, verify_ssl, connect_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding)
+  def initialize(endpoint, verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding, logger)
     @endpoint = endpoint
     @sumo_client = sumo_client
-    create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
+    create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
     @compress = compress_enabled
     @compress_encoding = (compress_encoding ||= COMPRESS_GZIP).downcase
+    @logger = logger
 
     unless [COMPRESS_DEFLATE, COMPRESS_GZIP].include? @compress_encoding
       raise "Invalid compression encoding #{@compress_encoding} must be gzip or deflate"
@@ -29,6 +31,29 @@ class SumologicConnection
     unless response.ok?
       raise RuntimeError, "Failed to send data to HTTP Source. #{response.code} - #{response.body}"
     end
+
+    # response is 20x, check response content
+    return if response.content.length == 0
+    
+    # if we get a non-empty response, check it 
+    begin
+      response_map = JSON.load(response.content)
+    rescue JSON::ParserError
+      @logger.warn "Error decoding receiver response: #{response.content}"
+      return
+    end
+
+    # log a warning with the present keys
+    response_keys = ["id", "code", "status", "message", "errors"]
+    log_params = []
+    response_keys.each do |key|
+      if response_map.has_key?(key) then
+        value = response_map[key]
+        log_params.append("#{key}: #{value}")
+      end
+    end
+    log_params_str = log_params.join(", ")
+    @logger.warn "There was an issue sending data: #{log_params_str}"
   end
 
   def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields, dimensions)
@@ -69,10 +94,11 @@ class SumologicConnection
     verify_ssl==true ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
   end
 
-  def create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
+  def create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
     @http                        = HTTPClient.new(proxy_uri)
     @http.ssl_config.verify_mode = ssl_options(verify_ssl)
     @http.connect_timeout        = connect_timeout
+    @http.send_timeout           = send_timeout
     if disable_cookies
       @http.cookie_manager       = nil
     end
@@ -126,6 +152,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   config_param :verify_ssl, :bool, :default => true
   config_param :delimiter, :string, :default => "."
   config_param :open_timeout, :integer, :default => 60
+  config_param :send_timeout, :integer, :default => 120
   config_param :add_timestamp, :bool, :default => true
   config_param :timestamp_key, :string, :default => 'timestamp'
   config_param :proxy_uri, :string, :default => nil
@@ -210,11 +237,13 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
       conf['endpoint'],
       conf['verify_ssl'],
       conf['open_timeout'].to_i,
+      conf['send_timeout'].to_i,
       conf['proxy_uri'],
       conf['disable_cookies'],
       conf['sumo_client'],
       conf['compress'],
-      conf['compress_encoding']
+      conf['compress_encoding'],
+      log,
       )
     super
   end
@@ -249,8 +278,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   def dump_log(log)
     log.delete('_sumo_metadata')
     begin
-      parser = Yajl::Parser.new
-      hash = parser.parse(log[@log_key])
+      hash = JSON.parse(log[@log_key])
       log[@log_key] = hash
       Yajl.dump(log)
     rescue
@@ -369,6 +397,8 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
         fields = [fields,@custom_fields].compact.join(",")
       end
 
+      @log.debug { "Sending #{messages.count} #{@data_type} records with source category '#{source_category}', source host '#{source_host}', source name '#{source_name}'." }
+
       @sumo_conn.publish(
           messages.join("\n"),
           source_host         =source_host,

data/test/plugin/test_out_sumologic.rb

@@ -713,4 +713,116 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
+  def test_emit_fields_string_based
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"}'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":{"foo":"bar","message":"test"}}\z/,
+                     times:1
+  end
+  
+  def test_emit_fields_invalid_json_string_based_1
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\": \\"test\\""}\z/,
+                     times:1
+  end
+  
+  def test_emit_fields_invalid_json_string_based_2
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"foo": "bar", "message"'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\""}\z/,
+                     times:1
+  end
+  
+  def test_emit_fields_invalid_json_string_based_3
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '"foo\": \"bar\", \"mess'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":"\\"foo\\\\\\": \\\\\\"bar\\\\\\", \\\\\\"mess"}\z/,
+                     times:1
+  end
+
+  def test_warning_response_from_receiver
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      endpoint #{endpoint}
+    }
+    testdata = [
+      [
+        '{"id":"1TIRY-KGIVX-TPQRJ","errors":[{"code":"internal.error","message":"Internal server error."}]}', 
+        'There was an issue sending data: id: 1TIRY-KGIVX-TPQRJ, errors: [{"code"=>"internal.error", "message"=>"Internal server error."}]'
+      ],
+      [
+        '{"id":"1TIRY-KGIVX-TPQRX","code": 200, "status": "Fields dropped", "message": "Dropped fields above the 30 field limit"}', 
+        'There was an issue sending data: id: 1TIRY-KGIVX-TPQRX, code: 200, status: Fields dropped, message: Dropped fields above the 30 field limit'
+      ],
+    ]
+    time = event_time
+
+    testdata.each do |data, log|
+      driver = create_driver(config)
+      stub_request(:post, endpoint).to_return(body: data, headers: {content_type: 'application/json'})
+      driver.run do
+        driver.feed("test", time, {"message": "test"})
+      end
+      assert_equal driver.logs.length, 1
+      assert driver.logs[0].end_with?(log + "\n")
+    end
+  end
+
 end

data/vagrant/README.md

@@ -0,0 +1,43 @@
+# Vagrant
+
+## Prerequisites
+
+Please install the following:
+
+- [VirtualBox](https://www.virtualbox.org/)
+- [Vagrant](https://www.vagrantup.com/)
+- [vagrant-disksize](https://github.com/sprotheroe/vagrant-disksize) plugin
+
+### MacOS
+
+```bash
+brew cask install virtualbox
+brew cask install vagrant
+vagrant plugin install vagrant-disksize
+```
+
+## Setting up
+
+You can set up the Vagrant environment with just one command:
+
+```bash
+vagrant up
+```
+
+After successfull installation you can ssh to the virtual machine with:
+
+```bash
+vagrant ssh
+```
+
+NOTICE: The directory with fluentd-output-sumologic repository on the host is synced with `/sumologic/` directory on the virtual machine.
+
+## Runing tests
+
+You can run tests using following commands:
+
+```bash
+cd /sumologic
+bundle install
+bundle exec rake
+```

data/vagrant/provision.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -x
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get --yes upgrade
+apt-get --yes install apt-transport-https
+
+echo "export EDITOR=vim" >> /home/vagrant/.bashrc
+
+# Install docker
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+add-apt-repository \
+   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+   $(lsb_release -cs) \
+   stable"
+apt-get install -y docker-ce docker-ce-cli containerd.io
+usermod -aG docker vagrant
+
+# Install make
+apt-get install -y make
+
+# install requirements for ruby
+snap install ruby --channel=2.6/stable --classic
+su vagrant -c 'gem install bundler:2.1.4'
+apt install -y gcc g++ libsnappy-dev libicu-dev zlib1g-dev cmake pkg-config libssl-dev

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sumologic_output
 version: !ruby/object:Gem::Version
-  version: 1.7.2
+  version: 1.7.5
 platform: ruby
 authors:
 - Steven Adams
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-23 00:00:00.000000000 Z
+date: 2022-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -103,6 +103,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
 - ".gitignore"
 - ".travis.yml"
 - CHANGELOG.md
@@ -110,11 +111,14 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- Vagrantfile
 - ci/build.sh
 - fluent-plugin-sumologic_output.gemspec
 - lib/fluent/plugin/out_sumologic.rb
 - test/helper.rb
 - test/plugin/test_out_sumologic.rb
+- vagrant/README.md
+- vagrant/provision.sh
 homepage: https://github.com/SumoLogic/fluentd-output-sumologic
 licenses:
 - Apache-2.0