fluent-plugin-sumologic_output 1.7.1 → 1.7.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 38f85777973860ce39f213613b943139ae0304be0b69e13b2b1cde495a08e2d5
4
- data.tar.gz: b4838ac669ee2ae899b06236d57a6cf01144370b9f19ae0fe56f900051058936
3
+ metadata.gz: 8ecb31a6366d5d0d352759cd11053747029c07d65267d410c5dfb4826b487ad8
4
+ data.tar.gz: 43b5911ccfeecd4e9d5a3f56b040678fb5e089a1793668b61559544e73427697
5
5
  SHA512:
6
- metadata.gz: 604cc97685ba67fe6bbee0dd519b4228f66a4ca13ef721371cc8adb56d253f0a4f710ff1f00270675ce8e95526d951d22a4cc2ab0e03c17168fc7021d8c39c71
7
- data.tar.gz: 61ab2d7c22d5f41d17cf3a4427db0f4c352ee5863b47b40b960279262ef047a5203e8ef98af5626ec0951625b520382a1003db7f9a320b1b408ffd5f573d67c4
6
+ metadata.gz: 590654de132ef18f36d6ef3671e7fec0a6b813c7e23beb2c602fd259521e405ef043cdefd2ba96b6454c72d91d88e9be9b3baa89467893c9448e36003a09655b
7
+ data.tar.gz: 0ab2c898106c02e8cb01af1f34231d230528378dd71173c569521be35c5b8427951f0219ff049b114f912d9fc2e92924b9023159c2fd187425cbf7b16fccd4e0
@@ -0,0 +1,5 @@
1
+ # Learn about CODEOWNERS file format:
2
+ # https://help.github.com/en/articles/about-code-owners
3
+
4
+ * @SumoLogic/open-source-collection-team
5
+
data/.gitignore CHANGED
@@ -2,4 +2,5 @@
2
2
  *.rbc
3
3
  Gemfile.lock
4
4
  .idea/
5
- TAGS
5
+ TAGS
6
+ .vagrant
data/CHANGELOG.md CHANGED
@@ -2,6 +2,15 @@
2
2
 
3
3
  All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
4
4
 
5
+ <a name="1.7.3"></a>
6
+ # [1.7.3] (2021-10-19)
7
+ - Expose httpclient send_timeout [#66](https://github.com/SumoLogic/fluentd-output-sumologic/pull/68)
8
+ - Fix json parsing [#69](https://github.com/SumoLogic/fluentd-output-sumologic/pull/69)
9
+
10
+ <a name="1.7.2"></a>
11
+ # [1.7.2] (2020-11-23)
12
+ - Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)
13
+
5
14
  <a name="1.7.1"></a>
6
15
  # [1.7.1] (2020-04-28)
7
16
  - Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)
data/README.md CHANGED
@@ -28,6 +28,7 @@ Configuration options for fluent.conf are:
28
28
  * json_merge - Same as json but merge content of `log_key` into the top level and strip `log_key`
29
29
  * `log_key` - Used to specify the key when merging json or sending logs in text format (default `message`)
30
30
  * `open_timeout` - Set timeout seconds to wait until connection is opened.
31
+ * `send_timeout` - Timeout for sending to SumoLogic in seconds. Don't modify unless you see `HTTPClient::SendTimeoutError` in your Fluentd logs. (default `120`)
31
32
  * `add_timestamp` - Add `timestamp` (or `timestamp_key`) field to logs before sending to sumologic (default `true`)
32
33
  * `timestamp_key` - Field name when `add_timestamp` is on (default `timestamp`)
33
34
  * `proxy_uri` - Add the `uri` of the `proxy` environment if present.
data/Vagrantfile ADDED
@@ -0,0 +1,27 @@
1
+ # -*- mode: ruby -*-
2
+ # vi: set ft=ruby :
3
+
4
+ unless Vagrant.has_plugin?("vagrant-disksize")
5
+ puts "vagrant-disksize plugin unavailable\n" +
6
+ "please install it via 'vagrant plugin install vagrant-disksize'"
7
+ exit 1
8
+ end
9
+
10
+ Vagrant.configure('2') do |config|
11
+ config.vm.box = 'ubuntu/focal64'
12
+ config.disksize.size = '50GB'
13
+ config.vm.box_check_update = false
14
+ config.vm.host_name = 'fluentd-output-sumologic'
15
+ config.vm.network :private_network, ip: "192.168.78.45"
16
+
17
+ config.vm.provider 'virtualbox' do |vb|
18
+ vb.gui = false
19
+ vb.cpus = 8
20
+ vb.memory = 16384
21
+ vb.name = 'fluentd-output-sumologic'
22
+ end
23
+
24
+ config.vm.provision 'shell', path: 'vagrant/provision.sh'
25
+
26
+ config.vm.synced_folder ".", "/sumologic"
27
+ end
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |gem|
6
6
  gem.name = "fluent-plugin-sumologic_output"
7
- gem.version = "1.7.1"
7
+ gem.version = "1.7.4"
8
8
  gem.authors = ["Steven Adams", "Frank Reno"]
9
9
  gem.email = ["stevezau@gmail.com", "frank.reno@me.com"]
10
10
  gem.description = %q{Output plugin to SumoLogic HTTP Endpoint}
@@ -1,5 +1,6 @@
1
1
  require 'fluent/plugin/output'
2
2
  require 'net/https'
3
+ require 'json'
3
4
  require 'yajl'
4
5
  require 'httpclient'
5
6
  require 'zlib'
@@ -12,12 +13,13 @@ class SumologicConnection
12
13
  COMPRESS_DEFLATE = 'deflate'
13
14
  COMPRESS_GZIP = 'gzip'
14
15
 
15
- def initialize(endpoint, verify_ssl, connect_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding)
16
+ def initialize(endpoint, verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding, logger)
16
17
  @endpoint = endpoint
17
18
  @sumo_client = sumo_client
18
- create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
19
+ create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
19
20
  @compress = compress_enabled
20
21
  @compress_encoding = (compress_encoding ||= COMPRESS_GZIP).downcase
22
+ @logger = logger
21
23
 
22
24
  unless [COMPRESS_DEFLATE, COMPRESS_GZIP].include? @compress_encoding
23
25
  raise "Invalid compression encoding #{@compress_encoding} must be gzip or deflate"
@@ -29,6 +31,29 @@ class SumologicConnection
29
31
  unless response.ok?
30
32
  raise RuntimeError, "Failed to send data to HTTP Source. #{response.code} - #{response.body}"
31
33
  end
34
+
35
+ # response is 20x, check response content
36
+ return if response.content.length == 0
37
+
38
+ # if we get a non-empty response, check it
39
+ begin
40
+ response_map = JSON.load(response.content)
41
+ rescue JSON::ParserError
42
+ @logger.warn "Error decoding receiver response: #{response.content}"
43
+ return
44
+ end
45
+
46
+ # log a warning with the present keys
47
+ response_keys = ["id", "code", "status", "message", "errors"]
48
+ log_params = []
49
+ response_keys.each do |key|
50
+ if response_map.has_key?(key) then
51
+ value = response_map[key]
52
+ log_params.append("#{key}: #{value}")
53
+ end
54
+ end
55
+ log_params_str = log_params.join(", ")
56
+ @logger.warn "There was an issue sending data: #{log_params_str}"
32
57
  end
33
58
 
34
59
  def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields, dimensions)
@@ -69,10 +94,11 @@ class SumologicConnection
69
94
  verify_ssl==true ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
70
95
  end
71
96
 
72
- def create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
97
+ def create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
73
98
  @http = HTTPClient.new(proxy_uri)
74
99
  @http.ssl_config.verify_mode = ssl_options(verify_ssl)
75
100
  @http.connect_timeout = connect_timeout
101
+ @http.send_timeout = send_timeout
76
102
  if disable_cookies
77
103
  @http.cookie_manager = nil
78
104
  end
@@ -126,6 +152,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
126
152
  config_param :verify_ssl, :bool, :default => true
127
153
  config_param :delimiter, :string, :default => "."
128
154
  config_param :open_timeout, :integer, :default => 60
155
+ config_param :send_timeout, :integer, :default => 120
129
156
  config_param :add_timestamp, :bool, :default => true
130
157
  config_param :timestamp_key, :string, :default => 'timestamp'
131
158
  config_param :proxy_uri, :string, :default => nil
@@ -210,11 +237,13 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
210
237
  conf['endpoint'],
211
238
  conf['verify_ssl'],
212
239
  conf['open_timeout'].to_i,
240
+ conf['send_timeout'].to_i,
213
241
  conf['proxy_uri'],
214
242
  conf['disable_cookies'],
215
243
  conf['sumo_client'],
216
244
  conf['compress'],
217
- conf['compress_encoding']
245
+ conf['compress_encoding'],
246
+ log,
218
247
  )
219
248
  super
220
249
  end
@@ -249,8 +278,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
249
278
  def dump_log(log)
250
279
  log.delete('_sumo_metadata')
251
280
  begin
252
- parser = Yajl::Parser.new
253
- hash = parser.parse(log[@log_key])
281
+ hash = JSON.parse(log[@log_key])
254
282
  log[@log_key] = hash
255
283
  Yajl.dump(log)
256
284
  rescue
@@ -293,6 +321,19 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
293
321
  time.to_s.length == 13 ? time : time * 1000
294
322
  end
295
323
 
324
+ # Convert log to string and strip it
325
+ def log_to_str(log)
326
+ if log.is_a?(Array) or log.is_a?(Hash)
327
+ log = Yajl.dump(log)
328
+ end
329
+
330
+ unless log.nil?
331
+ log.strip!
332
+ end
333
+
334
+ return log
335
+ end
336
+
296
337
  # This method is called every flush interval. Write the buffer chunk
297
338
  def write(chunk)
298
339
  messages_list = {}
@@ -313,10 +354,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
313
354
  when 'logs'
314
355
  case log_format
315
356
  when 'text'
316
- log = record[@log_key]
317
- unless log.nil?
318
- log.strip!
319
- end
357
+ log = log_to_str(record[@log_key])
320
358
  when 'json_merge'
321
359
  if @add_timestamp
322
360
  record = { @timestamp_key => sumo_timestamp(time) }.merge(record)
@@ -334,10 +372,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
334
372
  log = dump_log(record)
335
373
  end
336
374
  when 'metrics'
337
- log = record[@log_key]
338
- unless log.nil?
339
- log.strip!
340
- end
375
+ log = log_to_str(record[@log_key])
341
376
  end
342
377
 
343
378
  unless log.nil?
@@ -671,4 +671,158 @@ class SumologicOutput < Test::Unit::TestCase
671
671
  times:1
672
672
  end
673
673
 
674
- end
674
+ def test_emit_text_from_array
675
+ config = %{
676
+ endpoint https://collectors.sumologic.com/v1/receivers/http/1234
677
+ log_format text
678
+ source_category test
679
+ source_host test
680
+ source_name test
681
+
682
+ }
683
+ driver = create_driver(config)
684
+ time = event_time
685
+ stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
686
+ driver.run do
687
+ driver.feed("output.test", time, {'foo' => 'bar', 'message' => ['test', 'test2']})
688
+ end
689
+ assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
690
+ headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
691
+ body: '["test","test2"]',
692
+ times:1
693
+ end
694
+
695
+ def test_emit_text_from_dict
696
+ config = %{
697
+ endpoint https://collectors.sumologic.com/v1/receivers/http/1234
698
+ log_format text
699
+ source_category test
700
+ source_host test
701
+ source_name test
702
+
703
+ }
704
+ driver = create_driver(config)
705
+ time = event_time
706
+ stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
707
+ driver.run do
708
+ driver.feed("output.test", time, {'foo' => 'bar', 'message' => {'test': 'test2', 'test3': 'test4'}})
709
+ end
710
+ assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
711
+ headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
712
+ body: '{"test":"test2","test3":"test4"}',
713
+ times:1
714
+ end
715
+
716
+ def test_emit_fields_string_based
717
+ config = %{
718
+ endpoint https://collectors.sumologic.com/v1/receivers/http/1234
719
+ log_format fields
720
+ source_category test
721
+ source_host test
722
+ source_name test
723
+
724
+ }
725
+ driver = create_driver(config)
726
+ time = event_time
727
+ stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
728
+ driver.run do
729
+ driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"}'})
730
+ end
731
+ assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
732
+ headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
733
+ body: /\A{"timestamp":\d+.,"message":{"foo":"bar","message":"test"}}\z/,
734
+ times:1
735
+ end
736
+
737
+ def test_emit_fields_invalid_json_string_based_1
738
+ config = %{
739
+ endpoint https://collectors.sumologic.com/v1/receivers/http/1234
740
+ log_format fields
741
+ source_category test
742
+ source_host test
743
+ source_name test
744
+
745
+ }
746
+ driver = create_driver(config)
747
+ time = event_time
748
+ stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
749
+ driver.run do
750
+ driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"'})
751
+ end
752
+ assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
753
+ headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
754
+ body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\": \\"test\\""}\z/,
755
+ times:1
756
+ end
757
+
758
+ def test_emit_fields_invalid_json_string_based_2
759
+ config = %{
760
+ endpoint https://collectors.sumologic.com/v1/receivers/http/1234
761
+ log_format fields
762
+ source_category test
763
+ source_host test
764
+ source_name test
765
+
766
+ }
767
+ driver = create_driver(config)
768
+ time = event_time
769
+ stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
770
+ driver.run do
771
+ driver.feed("output.test", time, {'message' => '{"foo": "bar", "message"'})
772
+ end
773
+ assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
774
+ headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
775
+ body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\""}\z/,
776
+ times:1
777
+ end
778
+
779
+ def test_emit_fields_invalid_json_string_based_3
780
+ config = %{
781
+ endpoint https://collectors.sumologic.com/v1/receivers/http/1234
782
+ log_format fields
783
+ source_category test
784
+ source_host test
785
+ source_name test
786
+
787
+ }
788
+ driver = create_driver(config)
789
+ time = event_time
790
+ stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
791
+ driver.run do
792
+ driver.feed("output.test", time, {'message' => '"foo\": \"bar\", \"mess'})
793
+ end
794
+ assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
795
+ headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
796
+ body: /\A{"timestamp":\d+.,"message":"\\"foo\\\\\\": \\\\\\"bar\\\\\\", \\\\\\"mess"}\z/,
797
+ times:1
798
+ end
799
+
800
+ def test_warning_response_from_receiver
801
+ endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
802
+ config = %{
803
+ endpoint #{endpoint}
804
+ }
805
+ testdata = [
806
+ [
807
+ '{"id":"1TIRY-KGIVX-TPQRJ","errors":[{"code":"internal.error","message":"Internal server error."}]}',
808
+ 'There was an issue sending data: id: 1TIRY-KGIVX-TPQRJ, errors: [{"code"=>"internal.error", "message"=>"Internal server error."}]'
809
+ ],
810
+ [
811
+ '{"id":"1TIRY-KGIVX-TPQRX","code": 200, "status": "Fields dropped", "message": "Dropped fields above the 30 field limit"}',
812
+ 'There was an issue sending data: id: 1TIRY-KGIVX-TPQRX, code: 200, status: Fields dropped, message: Dropped fields above the 30 field limit'
813
+ ],
814
+ ]
815
+ time = event_time
816
+
817
+ testdata.each do |data, log|
818
+ driver = create_driver(config)
819
+ stub_request(:post, endpoint).to_return(body: data, headers: {content_type: 'application/json'})
820
+ driver.run do
821
+ driver.feed("test", time, {"message": "test"})
822
+ end
823
+ assert_equal driver.logs.length, 1
824
+ assert driver.logs[0].end_with?(log + "\n")
825
+ end
826
+ end
827
+
828
+ end
data/vagrant/README.md ADDED
@@ -0,0 +1,43 @@
1
+ # Vagrant
2
+
3
+ ## Prerequisites
4
+
5
+ Please install the following:
6
+
7
+ - [VirtualBox](https://www.virtualbox.org/)
8
+ - [Vagrant](https://www.vagrantup.com/)
9
+ - [vagrant-disksize](https://github.com/sprotheroe/vagrant-disksize) plugin
10
+
11
+ ### MacOS
12
+
13
+ ```bash
14
+ brew cask install virtualbox
15
+ brew cask install vagrant
16
+ vagrant plugin install vagrant-disksize
17
+ ```
18
+
19
+ ## Setting up
20
+
21
+ You can set up the Vagrant environment with just one command:
22
+
23
+ ```bash
24
+ vagrant up
25
+ ```
26
+
27
+ After successfull installation you can ssh to the virtual machine with:
28
+
29
+ ```bash
30
+ vagrant ssh
31
+ ```
32
+
33
+ NOTICE: The directory with fluentd-output-sumologic repository on the host is synced with `/sumologic/` directory on the virtual machine.
34
+
35
+ ## Runing tests
36
+
37
+ You can run tests using following commands:
38
+
39
+ ```bash
40
+ cd /sumologic
41
+ bundle install
42
+ bundle exec rake
43
+ ```
@@ -0,0 +1,27 @@
1
+ #!/bin/bash
2
+
3
+ set -x
4
+
5
+ export DEBIAN_FRONTEND=noninteractive
6
+ apt-get update
7
+ apt-get --yes upgrade
8
+ apt-get --yes install apt-transport-https
9
+
10
+ echo "export EDITOR=vim" >> /home/vagrant/.bashrc
11
+
12
+ # Install docker
13
+ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
14
+ add-apt-repository \
15
+ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
16
+ $(lsb_release -cs) \
17
+ stable"
18
+ apt-get install -y docker-ce docker-ce-cli containerd.io
19
+ usermod -aG docker vagrant
20
+
21
+ # Install make
22
+ apt-get install -y make
23
+
24
+ # install requirements for ruby
25
+ snap install ruby --channel=2.6/stable --classic
26
+ su vagrant -c 'gem install bundler:2.1.4'
27
+ apt install -y gcc g++ libsnappy-dev libicu-dev zlib1g-dev cmake pkg-config libssl-dev
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-sumologic_output
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.7.1
4
+ version: 1.7.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Steven Adams
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-04-28 00:00:00.000000000 Z
12
+ date: 2022-04-08 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler
@@ -103,6 +103,7 @@ executables: []
103
103
  extensions: []
104
104
  extra_rdoc_files: []
105
105
  files:
106
+ - ".github/CODEOWNERS"
106
107
  - ".gitignore"
107
108
  - ".travis.yml"
108
109
  - CHANGELOG.md
@@ -110,11 +111,14 @@ files:
110
111
  - LICENSE
111
112
  - README.md
112
113
  - Rakefile
114
+ - Vagrantfile
113
115
  - ci/build.sh
114
116
  - fluent-plugin-sumologic_output.gemspec
115
117
  - lib/fluent/plugin/out_sumologic.rb
116
118
  - test/helper.rb
117
119
  - test/plugin/test_out_sumologic.rb
120
+ - vagrant/README.md
121
+ - vagrant/provision.sh
118
122
  homepage: https://github.com/SumoLogic/fluentd-output-sumologic
119
123
  licenses:
120
124
  - Apache-2.0
@@ -134,8 +138,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
134
138
  - !ruby/object:Gem::Version
135
139
  version: '0'
136
140
  requirements: []
137
- rubyforge_project:
138
- rubygems_version: 2.7.7
141
+ rubygems_version: 3.0.8
139
142
  signing_key:
140
143
  specification_version: 4
141
144
  summary: Output plugin to SumoLogic HTTP Endpoint