fluent-plugin-sumologic_output 1.7.2 → 1.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/CODEOWNERS +5 -0
- data/.gitignore +2 -1
- data/CHANGELOG.md +9 -0
- data/README.md +1 -0
- data/Vagrantfile +29 -0
- data/fluent-plugin-sumologic_output.gemspec +1 -1
- data/lib/fluent/plugin/out_sumologic.rb +36 -6
- data/test/plugin/test_out_sumologic.rb +112 -0
- data/vagrant/README.md +43 -0
- data/vagrant/provision.sh +27 -0
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 10221368588a63fe0f97cb60695466a78a12eae80c13f736b5ae54879e4a5b4e
|
4
|
+
data.tar.gz: 2b99181908e9df7b2bcae1e6534a9311d72464cb74923a77f3a7489eeeb13f94
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 245aa88c7c415099db251eabf2c6b9e80a64d657cdf4df90e365115e962cbeec76279cab1d4d5dc87de1b6387475c0e6363a2be24c55d21953d4bc852727d869
|
7
|
+
data.tar.gz: c9a870b1252ca73b2565a4627e29ee5501ce9da6caa88834932d0edee05e45f573a5f7aeae183968e303786615622487188287236b1d12d8234ac992f25e822a
|
data/.github/CODEOWNERS
ADDED
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,15 @@
|
|
2
2
|
|
3
3
|
All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
|
4
4
|
|
5
|
+
<a name="1.7.3"></a>
|
6
|
+
# [1.7.3] (2021-10-19)
|
7
|
+
- Expose httpclient send_timeout [#66](https://github.com/SumoLogic/fluentd-output-sumologic/pull/68)
|
8
|
+
- Fix json parsing [#69](https://github.com/SumoLogic/fluentd-output-sumologic/pull/69)
|
9
|
+
|
10
|
+
<a name="1.7.2"></a>
|
11
|
+
# [1.7.2] (2020-11-23)
|
12
|
+
- Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)
|
13
|
+
|
5
14
|
<a name="1.7.1"></a>
|
6
15
|
# [1.7.1] (2020-04-28)
|
7
16
|
- Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)
|
data/README.md
CHANGED
@@ -28,6 +28,7 @@ Configuration options for fluent.conf are:
|
|
28
28
|
* json_merge - Same as json but merge content of `log_key` into the top level and strip `log_key`
|
29
29
|
* `log_key` - Used to specify the key when merging json or sending logs in text format (default `message`)
|
30
30
|
* `open_timeout` - Set timeout seconds to wait until connection is opened.
|
31
|
+
* `send_timeout` - Timeout for sending to SumoLogic in seconds. Don't modify unless you see `HTTPClient::SendTimeoutError` in your Fluentd logs. (default `120`)
|
31
32
|
* `add_timestamp` - Add `timestamp` (or `timestamp_key`) field to logs before sending to sumologic (default `true`)
|
32
33
|
* `timestamp_key` - Field name when `add_timestamp` is on (default `timestamp`)
|
33
34
|
* `proxy_uri` - Add the `uri` of the `proxy` environment if present.
|
data/Vagrantfile
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# -*- mode: ruby -*-
|
2
|
+
# vi: set ft=ruby :
|
3
|
+
|
4
|
+
unless Vagrant.has_plugin?("vagrant-disksize")
|
5
|
+
puts "vagrant-disksize plugin unavailable\n" +
|
6
|
+
"please install it via 'vagrant plugin install vagrant-disksize'"
|
7
|
+
exit 1
|
8
|
+
end
|
9
|
+
|
10
|
+
Vagrant.configure('2') do |config|
|
11
|
+
config.vm.box = 'ubuntu/focal64'
|
12
|
+
config.disksize.size = '50GB'
|
13
|
+
config.vm.box_check_update = false
|
14
|
+
config.vm.host_name = 'fluentd-output-sumologic'
|
15
|
+
# Vbox 6.1.28+ restricts host-only adapters to 192.168.56.0/21
|
16
|
+
# See: https://www.virtualbox.org/manual/ch06.html#network_hostonly
|
17
|
+
config.vm.network :private_network, ip: "192.168.56.45"
|
18
|
+
|
19
|
+
config.vm.provider 'virtualbox' do |vb|
|
20
|
+
vb.gui = false
|
21
|
+
vb.cpus = 8
|
22
|
+
vb.memory = 16384
|
23
|
+
vb.name = 'fluentd-output-sumologic'
|
24
|
+
end
|
25
|
+
|
26
|
+
config.vm.provision 'shell', path: 'vagrant/provision.sh'
|
27
|
+
|
28
|
+
config.vm.synced_folder ".", "/sumologic"
|
29
|
+
end
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = "fluent-plugin-sumologic_output"
|
7
|
-
gem.version = "1.7.
|
7
|
+
gem.version = "1.7.5"
|
8
8
|
gem.authors = ["Steven Adams", "Frank Reno"]
|
9
9
|
gem.email = ["stevezau@gmail.com", "frank.reno@me.com"]
|
10
10
|
gem.description = %q{Output plugin to SumoLogic HTTP Endpoint}
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'fluent/plugin/output'
|
2
2
|
require 'net/https'
|
3
|
+
require 'json'
|
3
4
|
require 'yajl'
|
4
5
|
require 'httpclient'
|
5
6
|
require 'zlib'
|
@@ -12,12 +13,13 @@ class SumologicConnection
|
|
12
13
|
COMPRESS_DEFLATE = 'deflate'
|
13
14
|
COMPRESS_GZIP = 'gzip'
|
14
15
|
|
15
|
-
def initialize(endpoint, verify_ssl, connect_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding)
|
16
|
+
def initialize(endpoint, verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding, logger)
|
16
17
|
@endpoint = endpoint
|
17
18
|
@sumo_client = sumo_client
|
18
|
-
create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
|
19
|
+
create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
|
19
20
|
@compress = compress_enabled
|
20
21
|
@compress_encoding = (compress_encoding ||= COMPRESS_GZIP).downcase
|
22
|
+
@logger = logger
|
21
23
|
|
22
24
|
unless [COMPRESS_DEFLATE, COMPRESS_GZIP].include? @compress_encoding
|
23
25
|
raise "Invalid compression encoding #{@compress_encoding} must be gzip or deflate"
|
@@ -29,6 +31,29 @@ class SumologicConnection
|
|
29
31
|
unless response.ok?
|
30
32
|
raise RuntimeError, "Failed to send data to HTTP Source. #{response.code} - #{response.body}"
|
31
33
|
end
|
34
|
+
|
35
|
+
# response is 20x, check response content
|
36
|
+
return if response.content.length == 0
|
37
|
+
|
38
|
+
# if we get a non-empty response, check it
|
39
|
+
begin
|
40
|
+
response_map = JSON.load(response.content)
|
41
|
+
rescue JSON::ParserError
|
42
|
+
@logger.warn "Error decoding receiver response: #{response.content}"
|
43
|
+
return
|
44
|
+
end
|
45
|
+
|
46
|
+
# log a warning with the present keys
|
47
|
+
response_keys = ["id", "code", "status", "message", "errors"]
|
48
|
+
log_params = []
|
49
|
+
response_keys.each do |key|
|
50
|
+
if response_map.has_key?(key) then
|
51
|
+
value = response_map[key]
|
52
|
+
log_params.append("#{key}: #{value}")
|
53
|
+
end
|
54
|
+
end
|
55
|
+
log_params_str = log_params.join(", ")
|
56
|
+
@logger.warn "There was an issue sending data: #{log_params_str}"
|
32
57
|
end
|
33
58
|
|
34
59
|
def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields, dimensions)
|
@@ -69,10 +94,11 @@ class SumologicConnection
|
|
69
94
|
verify_ssl==true ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
|
70
95
|
end
|
71
96
|
|
72
|
-
def create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
|
97
|
+
def create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
|
73
98
|
@http = HTTPClient.new(proxy_uri)
|
74
99
|
@http.ssl_config.verify_mode = ssl_options(verify_ssl)
|
75
100
|
@http.connect_timeout = connect_timeout
|
101
|
+
@http.send_timeout = send_timeout
|
76
102
|
if disable_cookies
|
77
103
|
@http.cookie_manager = nil
|
78
104
|
end
|
@@ -126,6 +152,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
|
|
126
152
|
config_param :verify_ssl, :bool, :default => true
|
127
153
|
config_param :delimiter, :string, :default => "."
|
128
154
|
config_param :open_timeout, :integer, :default => 60
|
155
|
+
config_param :send_timeout, :integer, :default => 120
|
129
156
|
config_param :add_timestamp, :bool, :default => true
|
130
157
|
config_param :timestamp_key, :string, :default => 'timestamp'
|
131
158
|
config_param :proxy_uri, :string, :default => nil
|
@@ -210,11 +237,13 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
|
|
210
237
|
conf['endpoint'],
|
211
238
|
conf['verify_ssl'],
|
212
239
|
conf['open_timeout'].to_i,
|
240
|
+
conf['send_timeout'].to_i,
|
213
241
|
conf['proxy_uri'],
|
214
242
|
conf['disable_cookies'],
|
215
243
|
conf['sumo_client'],
|
216
244
|
conf['compress'],
|
217
|
-
conf['compress_encoding']
|
245
|
+
conf['compress_encoding'],
|
246
|
+
log,
|
218
247
|
)
|
219
248
|
super
|
220
249
|
end
|
@@ -249,8 +278,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
|
|
249
278
|
def dump_log(log)
|
250
279
|
log.delete('_sumo_metadata')
|
251
280
|
begin
|
252
|
-
|
253
|
-
hash = parser.parse(log[@log_key])
|
281
|
+
hash = JSON.parse(log[@log_key])
|
254
282
|
log[@log_key] = hash
|
255
283
|
Yajl.dump(log)
|
256
284
|
rescue
|
@@ -369,6 +397,8 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
|
|
369
397
|
fields = [fields,@custom_fields].compact.join(",")
|
370
398
|
end
|
371
399
|
|
400
|
+
@log.debug { "Sending #{messages.count} #{@data_type} records with source category '#{source_category}', source host '#{source_host}', source name '#{source_name}'." }
|
401
|
+
|
372
402
|
@sumo_conn.publish(
|
373
403
|
messages.join("\n"),
|
374
404
|
source_host =source_host,
|
@@ -713,4 +713,116 @@ class SumologicOutput < Test::Unit::TestCase
|
|
713
713
|
times:1
|
714
714
|
end
|
715
715
|
|
716
|
+
def test_emit_fields_string_based
|
717
|
+
config = %{
|
718
|
+
endpoint https://collectors.sumologic.com/v1/receivers/http/1234
|
719
|
+
log_format fields
|
720
|
+
source_category test
|
721
|
+
source_host test
|
722
|
+
source_name test
|
723
|
+
|
724
|
+
}
|
725
|
+
driver = create_driver(config)
|
726
|
+
time = event_time
|
727
|
+
stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
|
728
|
+
driver.run do
|
729
|
+
driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"}'})
|
730
|
+
end
|
731
|
+
assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
|
732
|
+
headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
|
733
|
+
body: /\A{"timestamp":\d+.,"message":{"foo":"bar","message":"test"}}\z/,
|
734
|
+
times:1
|
735
|
+
end
|
736
|
+
|
737
|
+
def test_emit_fields_invalid_json_string_based_1
|
738
|
+
config = %{
|
739
|
+
endpoint https://collectors.sumologic.com/v1/receivers/http/1234
|
740
|
+
log_format fields
|
741
|
+
source_category test
|
742
|
+
source_host test
|
743
|
+
source_name test
|
744
|
+
|
745
|
+
}
|
746
|
+
driver = create_driver(config)
|
747
|
+
time = event_time
|
748
|
+
stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
|
749
|
+
driver.run do
|
750
|
+
driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"'})
|
751
|
+
end
|
752
|
+
assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
|
753
|
+
headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
|
754
|
+
body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\": \\"test\\""}\z/,
|
755
|
+
times:1
|
756
|
+
end
|
757
|
+
|
758
|
+
def test_emit_fields_invalid_json_string_based_2
|
759
|
+
config = %{
|
760
|
+
endpoint https://collectors.sumologic.com/v1/receivers/http/1234
|
761
|
+
log_format fields
|
762
|
+
source_category test
|
763
|
+
source_host test
|
764
|
+
source_name test
|
765
|
+
|
766
|
+
}
|
767
|
+
driver = create_driver(config)
|
768
|
+
time = event_time
|
769
|
+
stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
|
770
|
+
driver.run do
|
771
|
+
driver.feed("output.test", time, {'message' => '{"foo": "bar", "message"'})
|
772
|
+
end
|
773
|
+
assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
|
774
|
+
headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
|
775
|
+
body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\""}\z/,
|
776
|
+
times:1
|
777
|
+
end
|
778
|
+
|
779
|
+
def test_emit_fields_invalid_json_string_based_3
|
780
|
+
config = %{
|
781
|
+
endpoint https://collectors.sumologic.com/v1/receivers/http/1234
|
782
|
+
log_format fields
|
783
|
+
source_category test
|
784
|
+
source_host test
|
785
|
+
source_name test
|
786
|
+
|
787
|
+
}
|
788
|
+
driver = create_driver(config)
|
789
|
+
time = event_time
|
790
|
+
stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
|
791
|
+
driver.run do
|
792
|
+
driver.feed("output.test", time, {'message' => '"foo\": \"bar\", \"mess'})
|
793
|
+
end
|
794
|
+
assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
|
795
|
+
headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
|
796
|
+
body: /\A{"timestamp":\d+.,"message":"\\"foo\\\\\\": \\\\\\"bar\\\\\\", \\\\\\"mess"}\z/,
|
797
|
+
times:1
|
798
|
+
end
|
799
|
+
|
800
|
+
def test_warning_response_from_receiver
|
801
|
+
endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
|
802
|
+
config = %{
|
803
|
+
endpoint #{endpoint}
|
804
|
+
}
|
805
|
+
testdata = [
|
806
|
+
[
|
807
|
+
'{"id":"1TIRY-KGIVX-TPQRJ","errors":[{"code":"internal.error","message":"Internal server error."}]}',
|
808
|
+
'There was an issue sending data: id: 1TIRY-KGIVX-TPQRJ, errors: [{"code"=>"internal.error", "message"=>"Internal server error."}]'
|
809
|
+
],
|
810
|
+
[
|
811
|
+
'{"id":"1TIRY-KGIVX-TPQRX","code": 200, "status": "Fields dropped", "message": "Dropped fields above the 30 field limit"}',
|
812
|
+
'There was an issue sending data: id: 1TIRY-KGIVX-TPQRX, code: 200, status: Fields dropped, message: Dropped fields above the 30 field limit'
|
813
|
+
],
|
814
|
+
]
|
815
|
+
time = event_time
|
816
|
+
|
817
|
+
testdata.each do |data, log|
|
818
|
+
driver = create_driver(config)
|
819
|
+
stub_request(:post, endpoint).to_return(body: data, headers: {content_type: 'application/json'})
|
820
|
+
driver.run do
|
821
|
+
driver.feed("test", time, {"message": "test"})
|
822
|
+
end
|
823
|
+
assert_equal driver.logs.length, 1
|
824
|
+
assert driver.logs[0].end_with?(log + "\n")
|
825
|
+
end
|
826
|
+
end
|
827
|
+
|
716
828
|
end
|
data/vagrant/README.md
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
# Vagrant
|
2
|
+
|
3
|
+
## Prerequisites
|
4
|
+
|
5
|
+
Please install the following:
|
6
|
+
|
7
|
+
- [VirtualBox](https://www.virtualbox.org/)
|
8
|
+
- [Vagrant](https://www.vagrantup.com/)
|
9
|
+
- [vagrant-disksize](https://github.com/sprotheroe/vagrant-disksize) plugin
|
10
|
+
|
11
|
+
### MacOS
|
12
|
+
|
13
|
+
```bash
|
14
|
+
brew cask install virtualbox
|
15
|
+
brew cask install vagrant
|
16
|
+
vagrant plugin install vagrant-disksize
|
17
|
+
```
|
18
|
+
|
19
|
+
## Setting up
|
20
|
+
|
21
|
+
You can set up the Vagrant environment with just one command:
|
22
|
+
|
23
|
+
```bash
|
24
|
+
vagrant up
|
25
|
+
```
|
26
|
+
|
27
|
+
After successfull installation you can ssh to the virtual machine with:
|
28
|
+
|
29
|
+
```bash
|
30
|
+
vagrant ssh
|
31
|
+
```
|
32
|
+
|
33
|
+
NOTICE: The directory with fluentd-output-sumologic repository on the host is synced with `/sumologic/` directory on the virtual machine.
|
34
|
+
|
35
|
+
## Runing tests
|
36
|
+
|
37
|
+
You can run tests using following commands:
|
38
|
+
|
39
|
+
```bash
|
40
|
+
cd /sumologic
|
41
|
+
bundle install
|
42
|
+
bundle exec rake
|
43
|
+
```
|
@@ -0,0 +1,27 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
set -x
|
4
|
+
|
5
|
+
export DEBIAN_FRONTEND=noninteractive
|
6
|
+
apt-get update
|
7
|
+
apt-get --yes upgrade
|
8
|
+
apt-get --yes install apt-transport-https
|
9
|
+
|
10
|
+
echo "export EDITOR=vim" >> /home/vagrant/.bashrc
|
11
|
+
|
12
|
+
# Install docker
|
13
|
+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
|
14
|
+
add-apt-repository \
|
15
|
+
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
|
16
|
+
$(lsb_release -cs) \
|
17
|
+
stable"
|
18
|
+
apt-get install -y docker-ce docker-ce-cli containerd.io
|
19
|
+
usermod -aG docker vagrant
|
20
|
+
|
21
|
+
# Install make
|
22
|
+
apt-get install -y make
|
23
|
+
|
24
|
+
# install requirements for ruby
|
25
|
+
snap install ruby --channel=2.6/stable --classic
|
26
|
+
su vagrant -c 'gem install bundler:2.1.4'
|
27
|
+
apt install -y gcc g++ libsnappy-dev libicu-dev zlib1g-dev cmake pkg-config libssl-dev
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sumologic_output
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.7.
|
4
|
+
version: 1.7.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Steven Adams
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2022-04-11 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -103,6 +103,7 @@ executables: []
|
|
103
103
|
extensions: []
|
104
104
|
extra_rdoc_files: []
|
105
105
|
files:
|
106
|
+
- ".github/CODEOWNERS"
|
106
107
|
- ".gitignore"
|
107
108
|
- ".travis.yml"
|
108
109
|
- CHANGELOG.md
|
@@ -110,11 +111,14 @@ files:
|
|
110
111
|
- LICENSE
|
111
112
|
- README.md
|
112
113
|
- Rakefile
|
114
|
+
- Vagrantfile
|
113
115
|
- ci/build.sh
|
114
116
|
- fluent-plugin-sumologic_output.gemspec
|
115
117
|
- lib/fluent/plugin/out_sumologic.rb
|
116
118
|
- test/helper.rb
|
117
119
|
- test/plugin/test_out_sumologic.rb
|
120
|
+
- vagrant/README.md
|
121
|
+
- vagrant/provision.sh
|
118
122
|
homepage: https://github.com/SumoLogic/fluentd-output-sumologic
|
119
123
|
licenses:
|
120
124
|
- Apache-2.0
|