fluent-plugin-sumologic_output 1.7.2 → 1.7.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c3bf755e8ecd488457805d948614c43b3657de815889dd341091d11f338ad995
4
- data.tar.gz: 20163f7c5ac543a010e53fce506dbaed10bafe5d46830579a3101558f5b61fa4
3
+ metadata.gz: 10221368588a63fe0f97cb60695466a78a12eae80c13f736b5ae54879e4a5b4e
4
+ data.tar.gz: 2b99181908e9df7b2bcae1e6534a9311d72464cb74923a77f3a7489eeeb13f94
5
5
  SHA512:
6
- metadata.gz: 35cf3af9a561fd22e810dea880570007613b7cb9ae87307dd46ea3ff8d9e005b96c9ebf19f1aafd58b6ae4495509f51406b7788029f7fb26144f54b60f22370a
7
- data.tar.gz: 6b5e6a13ed50d798c4c0f8225e771a2c5977373373aed316cbe1a5369a2af22aef56c86e474aefd6ca847e41c496c287a191c23c1224d2a49075794b420949e4
6
+ metadata.gz: 245aa88c7c415099db251eabf2c6b9e80a64d657cdf4df90e365115e962cbeec76279cab1d4d5dc87de1b6387475c0e6363a2be24c55d21953d4bc852727d869
7
+ data.tar.gz: c9a870b1252ca73b2565a4627e29ee5501ce9da6caa88834932d0edee05e45f573a5f7aeae183968e303786615622487188287236b1d12d8234ac992f25e822a
@@ -0,0 +1,5 @@
1
+ # Learn about CODEOWNERS file format:
2
+ # https://help.github.com/en/articles/about-code-owners
3
+
4
+ * @SumoLogic/open-source-collection-team
5
+
data/.gitignore CHANGED
@@ -2,4 +2,5 @@
2
2
  *.rbc
3
3
  Gemfile.lock
4
4
  .idea/
5
- TAGS
5
+ TAGS
6
+ .vagrant
data/CHANGELOG.md CHANGED
@@ -2,6 +2,15 @@
2
2
 
3
3
  All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
4
4
 
5
+ <a name="1.7.3"></a>
6
+ # [1.7.3] (2021-10-19)
7
+ - Expose httpclient send_timeout [#66](https://github.com/SumoLogic/fluentd-output-sumologic/pull/68)
8
+ - Fix json parsing [#69](https://github.com/SumoLogic/fluentd-output-sumologic/pull/69)
9
+
10
+ <a name="1.7.2"></a>
11
+ # [1.7.2] (2020-11-23)
12
+ - Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)
13
+
5
14
  <a name="1.7.1"></a>
6
15
  # [1.7.1] (2020-04-28)
7
16
  - Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)
data/README.md CHANGED
@@ -28,6 +28,7 @@ Configuration options for fluent.conf are:
28
28
  * json_merge - Same as json but merge content of `log_key` into the top level and strip `log_key`
29
29
  * `log_key` - Used to specify the key when merging json or sending logs in text format (default `message`)
30
30
  * `open_timeout` - Set timeout seconds to wait until connection is opened.
31
+ * `send_timeout` - Timeout for sending to SumoLogic in seconds. Don't modify unless you see `HTTPClient::SendTimeoutError` in your Fluentd logs. (default `120`)
31
32
  * `add_timestamp` - Add `timestamp` (or `timestamp_key`) field to logs before sending to sumologic (default `true`)
32
33
  * `timestamp_key` - Field name when `add_timestamp` is on (default `timestamp`)
33
34
  * `proxy_uri` - Add the `uri` of the `proxy` environment if present.
data/Vagrantfile ADDED
@@ -0,0 +1,29 @@
1
+ # -*- mode: ruby -*-
2
+ # vi: set ft=ruby :
3
+
4
+ unless Vagrant.has_plugin?("vagrant-disksize")
5
+ puts "vagrant-disksize plugin unavailable\n" +
6
+ "please install it via 'vagrant plugin install vagrant-disksize'"
7
+ exit 1
8
+ end
9
+
10
+ Vagrant.configure('2') do |config|
11
+ config.vm.box = 'ubuntu/focal64'
12
+ config.disksize.size = '50GB'
13
+ config.vm.box_check_update = false
14
+ config.vm.host_name = 'fluentd-output-sumologic'
15
+ # Vbox 6.1.28+ restricts host-only adapters to 192.168.56.0/21
16
+ # See: https://www.virtualbox.org/manual/ch06.html#network_hostonly
17
+ config.vm.network :private_network, ip: "192.168.56.45"
18
+
19
+ config.vm.provider 'virtualbox' do |vb|
20
+ vb.gui = false
21
+ vb.cpus = 8
22
+ vb.memory = 16384
23
+ vb.name = 'fluentd-output-sumologic'
24
+ end
25
+
26
+ config.vm.provision 'shell', path: 'vagrant/provision.sh'
27
+
28
+ config.vm.synced_folder ".", "/sumologic"
29
+ end
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |gem|
6
6
  gem.name = "fluent-plugin-sumologic_output"
7
- gem.version = "1.7.2"
7
+ gem.version = "1.7.5"
8
8
  gem.authors = ["Steven Adams", "Frank Reno"]
9
9
  gem.email = ["stevezau@gmail.com", "frank.reno@me.com"]
10
10
  gem.description = %q{Output plugin to SumoLogic HTTP Endpoint}
@@ -1,5 +1,6 @@
1
1
  require 'fluent/plugin/output'
2
2
  require 'net/https'
3
+ require 'json'
3
4
  require 'yajl'
4
5
  require 'httpclient'
5
6
  require 'zlib'
@@ -12,12 +13,13 @@ class SumologicConnection
12
13
  COMPRESS_DEFLATE = 'deflate'
13
14
  COMPRESS_GZIP = 'gzip'
14
15
 
15
- def initialize(endpoint, verify_ssl, connect_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding)
16
+ def initialize(endpoint, verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding, logger)
16
17
  @endpoint = endpoint
17
18
  @sumo_client = sumo_client
18
- create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
19
+ create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
19
20
  @compress = compress_enabled
20
21
  @compress_encoding = (compress_encoding ||= COMPRESS_GZIP).downcase
22
+ @logger = logger
21
23
 
22
24
  unless [COMPRESS_DEFLATE, COMPRESS_GZIP].include? @compress_encoding
23
25
  raise "Invalid compression encoding #{@compress_encoding} must be gzip or deflate"
@@ -29,6 +31,29 @@ class SumologicConnection
29
31
  unless response.ok?
30
32
  raise RuntimeError, "Failed to send data to HTTP Source. #{response.code} - #{response.body}"
31
33
  end
34
+
35
+ # response is 20x, check response content
36
+ return if response.content.length == 0
37
+
38
+ # if we get a non-empty response, check it
39
+ begin
40
+ response_map = JSON.load(response.content)
41
+ rescue JSON::ParserError
42
+ @logger.warn "Error decoding receiver response: #{response.content}"
43
+ return
44
+ end
45
+
46
+ # log a warning with the present keys
47
+ response_keys = ["id", "code", "status", "message", "errors"]
48
+ log_params = []
49
+ response_keys.each do |key|
50
+ if response_map.has_key?(key) then
51
+ value = response_map[key]
52
+ log_params.append("#{key}: #{value}")
53
+ end
54
+ end
55
+ log_params_str = log_params.join(", ")
56
+ @logger.warn "There was an issue sending data: #{log_params_str}"
32
57
  end
33
58
 
34
59
  def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields, dimensions)
@@ -69,10 +94,11 @@ class SumologicConnection
69
94
  verify_ssl==true ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
70
95
  end
71
96
 
72
- def create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
97
+ def create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
73
98
  @http = HTTPClient.new(proxy_uri)
74
99
  @http.ssl_config.verify_mode = ssl_options(verify_ssl)
75
100
  @http.connect_timeout = connect_timeout
101
+ @http.send_timeout = send_timeout
76
102
  if disable_cookies
77
103
  @http.cookie_manager = nil
78
104
  end
@@ -126,6 +152,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
126
152
  config_param :verify_ssl, :bool, :default => true
127
153
  config_param :delimiter, :string, :default => "."
128
154
  config_param :open_timeout, :integer, :default => 60
155
+ config_param :send_timeout, :integer, :default => 120
129
156
  config_param :add_timestamp, :bool, :default => true
130
157
  config_param :timestamp_key, :string, :default => 'timestamp'
131
158
  config_param :proxy_uri, :string, :default => nil
@@ -210,11 +237,13 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
210
237
  conf['endpoint'],
211
238
  conf['verify_ssl'],
212
239
  conf['open_timeout'].to_i,
240
+ conf['send_timeout'].to_i,
213
241
  conf['proxy_uri'],
214
242
  conf['disable_cookies'],
215
243
  conf['sumo_client'],
216
244
  conf['compress'],
217
- conf['compress_encoding']
245
+ conf['compress_encoding'],
246
+ log,
218
247
  )
219
248
  super
220
249
  end
@@ -249,8 +278,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
249
278
  def dump_log(log)
250
279
  log.delete('_sumo_metadata')
251
280
  begin
252
- parser = Yajl::Parser.new
253
- hash = parser.parse(log[@log_key])
281
+ hash = JSON.parse(log[@log_key])
254
282
  log[@log_key] = hash
255
283
  Yajl.dump(log)
256
284
  rescue
@@ -369,6 +397,8 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
369
397
  fields = [fields,@custom_fields].compact.join(",")
370
398
  end
371
399
 
400
+ @log.debug { "Sending #{messages.count} #{@data_type} records with source category '#{source_category}', source host '#{source_host}', source name '#{source_name}'." }
401
+
372
402
  @sumo_conn.publish(
373
403
  messages.join("\n"),
374
404
  source_host =source_host,
@@ -713,4 +713,116 @@ class SumologicOutput < Test::Unit::TestCase
713
713
  times:1
714
714
  end
715
715
 
716
+ def test_emit_fields_string_based
717
+ config = %{
718
+ endpoint https://collectors.sumologic.com/v1/receivers/http/1234
719
+ log_format fields
720
+ source_category test
721
+ source_host test
722
+ source_name test
723
+
724
+ }
725
+ driver = create_driver(config)
726
+ time = event_time
727
+ stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
728
+ driver.run do
729
+ driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"}'})
730
+ end
731
+ assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
732
+ headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
733
+ body: /\A{"timestamp":\d+.,"message":{"foo":"bar","message":"test"}}\z/,
734
+ times:1
735
+ end
736
+
737
+ def test_emit_fields_invalid_json_string_based_1
738
+ config = %{
739
+ endpoint https://collectors.sumologic.com/v1/receivers/http/1234
740
+ log_format fields
741
+ source_category test
742
+ source_host test
743
+ source_name test
744
+
745
+ }
746
+ driver = create_driver(config)
747
+ time = event_time
748
+ stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
749
+ driver.run do
750
+ driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"'})
751
+ end
752
+ assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
753
+ headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
754
+ body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\": \\"test\\""}\z/,
755
+ times:1
756
+ end
757
+
758
+ def test_emit_fields_invalid_json_string_based_2
759
+ config = %{
760
+ endpoint https://collectors.sumologic.com/v1/receivers/http/1234
761
+ log_format fields
762
+ source_category test
763
+ source_host test
764
+ source_name test
765
+
766
+ }
767
+ driver = create_driver(config)
768
+ time = event_time
769
+ stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
770
+ driver.run do
771
+ driver.feed("output.test", time, {'message' => '{"foo": "bar", "message"'})
772
+ end
773
+ assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
774
+ headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
775
+ body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\""}\z/,
776
+ times:1
777
+ end
778
+
779
+ def test_emit_fields_invalid_json_string_based_3
780
+ config = %{
781
+ endpoint https://collectors.sumologic.com/v1/receivers/http/1234
782
+ log_format fields
783
+ source_category test
784
+ source_host test
785
+ source_name test
786
+
787
+ }
788
+ driver = create_driver(config)
789
+ time = event_time
790
+ stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
791
+ driver.run do
792
+ driver.feed("output.test", time, {'message' => '"foo\": \"bar\", \"mess'})
793
+ end
794
+ assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
795
+ headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
796
+ body: /\A{"timestamp":\d+.,"message":"\\"foo\\\\\\": \\\\\\"bar\\\\\\", \\\\\\"mess"}\z/,
797
+ times:1
798
+ end
799
+
800
+ def test_warning_response_from_receiver
801
+ endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
802
+ config = %{
803
+ endpoint #{endpoint}
804
+ }
805
+ testdata = [
806
+ [
807
+ '{"id":"1TIRY-KGIVX-TPQRJ","errors":[{"code":"internal.error","message":"Internal server error."}]}',
808
+ 'There was an issue sending data: id: 1TIRY-KGIVX-TPQRJ, errors: [{"code"=>"internal.error", "message"=>"Internal server error."}]'
809
+ ],
810
+ [
811
+ '{"id":"1TIRY-KGIVX-TPQRX","code": 200, "status": "Fields dropped", "message": "Dropped fields above the 30 field limit"}',
812
+ 'There was an issue sending data: id: 1TIRY-KGIVX-TPQRX, code: 200, status: Fields dropped, message: Dropped fields above the 30 field limit'
813
+ ],
814
+ ]
815
+ time = event_time
816
+
817
+ testdata.each do |data, log|
818
+ driver = create_driver(config)
819
+ stub_request(:post, endpoint).to_return(body: data, headers: {content_type: 'application/json'})
820
+ driver.run do
821
+ driver.feed("test", time, {"message": "test"})
822
+ end
823
+ assert_equal driver.logs.length, 1
824
+ assert driver.logs[0].end_with?(log + "\n")
825
+ end
826
+ end
827
+
716
828
  end
data/vagrant/README.md ADDED
@@ -0,0 +1,43 @@
1
+ # Vagrant
2
+
3
+ ## Prerequisites
4
+
5
+ Please install the following:
6
+
7
+ - [VirtualBox](https://www.virtualbox.org/)
8
+ - [Vagrant](https://www.vagrantup.com/)
9
+ - [vagrant-disksize](https://github.com/sprotheroe/vagrant-disksize) plugin
10
+
11
+ ### MacOS
12
+
13
+ ```bash
14
+ brew cask install virtualbox
15
+ brew cask install vagrant
16
+ vagrant plugin install vagrant-disksize
17
+ ```
18
+
19
+ ## Setting up
20
+
21
+ You can set up the Vagrant environment with just one command:
22
+
23
+ ```bash
24
+ vagrant up
25
+ ```
26
+
27
+ After successfull installation you can ssh to the virtual machine with:
28
+
29
+ ```bash
30
+ vagrant ssh
31
+ ```
32
+
33
+ NOTICE: The directory with fluentd-output-sumologic repository on the host is synced with `/sumologic/` directory on the virtual machine.
34
+
35
+ ## Runing tests
36
+
37
+ You can run tests using following commands:
38
+
39
+ ```bash
40
+ cd /sumologic
41
+ bundle install
42
+ bundle exec rake
43
+ ```
@@ -0,0 +1,27 @@
1
+ #!/bin/bash
2
+
3
+ set -x
4
+
5
+ export DEBIAN_FRONTEND=noninteractive
6
+ apt-get update
7
+ apt-get --yes upgrade
8
+ apt-get --yes install apt-transport-https
9
+
10
+ echo "export EDITOR=vim" >> /home/vagrant/.bashrc
11
+
12
+ # Install docker
13
+ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
14
+ add-apt-repository \
15
+ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
16
+ $(lsb_release -cs) \
17
+ stable"
18
+ apt-get install -y docker-ce docker-ce-cli containerd.io
19
+ usermod -aG docker vagrant
20
+
21
+ # Install make
22
+ apt-get install -y make
23
+
24
+ # install requirements for ruby
25
+ snap install ruby --channel=2.6/stable --classic
26
+ su vagrant -c 'gem install bundler:2.1.4'
27
+ apt install -y gcc g++ libsnappy-dev libicu-dev zlib1g-dev cmake pkg-config libssl-dev
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-sumologic_output
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.7.2
4
+ version: 1.7.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Steven Adams
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-11-23 00:00:00.000000000 Z
12
+ date: 2022-04-11 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler
@@ -103,6 +103,7 @@ executables: []
103
103
  extensions: []
104
104
  extra_rdoc_files: []
105
105
  files:
106
+ - ".github/CODEOWNERS"
106
107
  - ".gitignore"
107
108
  - ".travis.yml"
108
109
  - CHANGELOG.md
@@ -110,11 +111,14 @@ files:
110
111
  - LICENSE
111
112
  - README.md
112
113
  - Rakefile
114
+ - Vagrantfile
113
115
  - ci/build.sh
114
116
  - fluent-plugin-sumologic_output.gemspec
115
117
  - lib/fluent/plugin/out_sumologic.rb
116
118
  - test/helper.rb
117
119
  - test/plugin/test_out_sumologic.rb
120
+ - vagrant/README.md
121
+ - vagrant/provision.sh
118
122
  homepage: https://github.com/SumoLogic/fluentd-output-sumologic
119
123
  licenses:
120
124
  - Apache-2.0