fluent-plugin-logzio 0.0.16 → 0.0.21
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.whitesource +8 -0
- data/README.md +9 -0
- data/fluent-plugin-logzio.gemspec +5 -5
- data/lib/fluent/plugin/out_logzio_buffered.rb +46 -6
- data/spec/lib/fluent/plugin/out_logzio_buffered_spec.rb +2 -2
- data/spec/lib/fluent/plugin/out_logzio_multi_bulk_spec.rb +2 -2
- data/spec/lib/fluent/plugin/out_logzio_record_too_large_spec.rb +2 -2
- metadata +8 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: a6b850642fbf62e80b5033d03300ea4cbb3e600484836ae411877546785182a3
|
4
|
+
data.tar.gz: 2a40dc72118892e5bdf8bbf2becf44e6e2bf8688470034ae0152fb918b05a481
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b98047acec6ffededc5c44f22ac3f2b451da2fc5c3b6a9d0b32f7d1e1a5e838f06c5ab1cdb4754f30500817a74bfebcbf0e92d1583b14d48f085ce348a203e7b
|
7
|
+
data.tar.gz: c05ed2eb34bd102938d8b00708ffa6ac0978b5420d46001925dcf0a9042e9b46ec14868b42b6396eb0c0be80663d81579186c58ed1fb2650643bd6ce52f10fa1
|
data/.whitesource
ADDED
data/README.md
CHANGED
@@ -68,8 +68,17 @@ This is an **example** only. Your needs in production may vary!
|
|
68
68
|
* **retry_count** How many times to resend failed bulks. Defaults to 4 times.
|
69
69
|
* **retry_sleep** How long to sleep initially between retries, exponential step-off. Initial default is 2s.
|
70
70
|
* **bulk_limit** Limit to the size of the Logz.io upload bulk. Defaults to 1000000 bytes leaving about 24kB for overhead.
|
71
|
+
* **bulk_limit_warning_limit** Limit to the size of the Logz.io warning message when a record exceeds bulk_limit to prevent a recursion when Fluent warnings are sent to the Logz.io output. Defaults to nil (no truncation).
|
72
|
+
* **proxy_uri** Your proxy uri. Default is nil
|
73
|
+
* **proxy_cert** Your proxy cert. Default is nil
|
74
|
+
* **gzip** should the plugin ship the logs in gzip compression. Default is false
|
75
|
+
|
71
76
|
|
72
77
|
## Release Notes
|
78
|
+
- 0.0.21: Update gem `net-http-persistent` to 3.x.
|
79
|
+
- 0.0.20: Support gzip compression
|
80
|
+
- 0.0.18: Support proxy_uri and proxy_cert in the configuration file. Put logzio output plugin class under Fluent::Plugin module and thus work with multi workers.
|
81
|
+
- 0.0.17: Optional truncate log messages when they are exceeding bulk size in warning logs
|
73
82
|
- 0.0.16: More fluentD 1.0+ adjustments
|
74
83
|
- 0.0.15: Support FluentD 1.0+. Split the chunk into bulk uploads, decoupling `chunk_limit_size`/`buffer_chunk_limit` from Logz.io bulk limit. Tunable `bulk_limit` and initial `retry_sleep`.
|
75
84
|
- 0.0.14: Refactor send function to handle more cases, and retry in case of logzio connection failure.
|
@@ -4,9 +4,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |s|
|
6
6
|
s.name = 'fluent-plugin-logzio'
|
7
|
-
s.version = '0.0.
|
8
|
-
s.authors = ['Yury Kotov', 'Roi Rav-Hon', 'Arcadiy Ivanov']
|
9
|
-
s.email = ['bairkan@gmail.com', 'roi@logz.io', 'arcadiy@ivanov.biz']
|
7
|
+
s.version = '0.0.21'
|
8
|
+
s.authors = ['Yury Kotov', 'Roi Rav-Hon', 'Arcadiy Ivanov', 'Miri Ignatiev']
|
9
|
+
s.email = ['bairkan@gmail.com', 'roi@logz.io', 'arcadiy@ivanov.biz', 'miri.ignatiev@logz.io']
|
10
10
|
s.homepage = 'https://github.com/logzio/fluent-plugin-logzio'
|
11
11
|
s.summary = %q{Fluentd plugin for output to Logz.io}
|
12
12
|
s.description = %q{Fluentd pluging (fluent.org) for output to Logz.io (logz.io)}
|
@@ -19,10 +19,10 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.require_paths = ['lib']
|
20
20
|
s.required_ruby_version = Gem::Requirement.new('>= 2.1.0')
|
21
21
|
|
22
|
-
s.add_dependency 'net-http-persistent', '~>
|
22
|
+
s.add_dependency 'net-http-persistent', '~> 3.0'
|
23
23
|
s.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
|
24
24
|
s.add_development_dependency 'rake', '~> 12.3'
|
25
25
|
s.add_development_dependency 'bundler', '~> 1.16'
|
26
26
|
s.add_development_dependency 'rspec', '~> 3.7'
|
27
27
|
s.add_development_dependency 'test-unit', '~> 3.2'
|
28
|
-
end
|
28
|
+
end
|
@@ -1,7 +1,9 @@
|
|
1
1
|
require 'time'
|
2
2
|
require 'fluent/plugin/output'
|
3
|
+
require 'zlib'
|
4
|
+
require 'stringio'
|
3
5
|
|
4
|
-
module Fluent
|
6
|
+
module Fluent::Plugin
|
5
7
|
class LogzioOutputBuffered < Output
|
6
8
|
Fluent::Plugin.register_output('logzio_buffered', self)
|
7
9
|
|
@@ -13,22 +15,40 @@ module Fluent
|
|
13
15
|
config_param :retry_count, :integer, default: 4 # How many times to resend failed bulks. Undocumented because not suppose to be changed
|
14
16
|
config_param :retry_sleep, :integer, default: 2 # How long to sleep initially between retries, exponential step-off
|
15
17
|
config_param :bulk_limit, :integer, default: 1000000 # Make sure submission to LogzIO does not exceed 1MB limit and leave some overhead
|
18
|
+
config_param :bulk_limit_warning_limit, :integer, default: nil # If fluent warnings are sent to the Logzio output, truncating is necessary to prevent a recursion
|
16
19
|
config_param :http_idle_timeout, :integer, default: 5
|
17
20
|
config_param :output_tags_fieldname, :string, default: 'fluentd_tags'
|
21
|
+
config_param :proxy_uri, :string, default: nil
|
22
|
+
config_param :proxy_cert, :string, default: nil
|
23
|
+
config_param :gzip, :bool, default: false # False for backward compatibility
|
18
24
|
|
19
25
|
def configure(conf)
|
20
26
|
super
|
21
27
|
compat_parameters_convert(conf, :buffer)
|
22
28
|
|
23
29
|
log.debug "Logz.io URL #{@endpoint_url}"
|
30
|
+
|
31
|
+
if conf['proxy_uri']
|
32
|
+
log.debug "Proxy #{@proxy_uri}"
|
33
|
+
ENV['http_proxy'] = @proxy_uri
|
34
|
+
end
|
35
|
+
|
36
|
+
if conf['proxy_cert']
|
37
|
+
log.debug "Proxy #{@proxy_cert}"
|
38
|
+
ENV['SSL_CERT_FILE'] = @proxy_cert
|
39
|
+
end
|
40
|
+
|
24
41
|
end
|
25
42
|
|
26
43
|
def start
|
27
44
|
super
|
28
45
|
require 'net/http/persistent'
|
29
46
|
@uri = URI @endpoint_url
|
30
|
-
@http = Net::HTTP::Persistent.new 'fluent-plugin-logzio', :ENV
|
47
|
+
@http = Net::HTTP::Persistent.new name: 'fluent-plugin-logzio', proxy: :ENV
|
31
48
|
@http.headers['Content-Type'] = 'text/plain'
|
49
|
+
if @gzip
|
50
|
+
@http.headers['Content-Encoding'] = 'gzip'
|
51
|
+
end
|
32
52
|
@http.idle_timeout = @http_idle_timeout
|
33
53
|
@http.socket_options << [Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1]
|
34
54
|
|
@@ -43,6 +63,10 @@ module Fluent
|
|
43
63
|
true
|
44
64
|
end
|
45
65
|
|
66
|
+
def multi_workers_ready?
|
67
|
+
true
|
68
|
+
end
|
69
|
+
|
46
70
|
def format(tag, time, record)
|
47
71
|
if time.is_a?(Fluent::EventTime)
|
48
72
|
sec_frac = time.to_f
|
@@ -74,7 +98,13 @@ module Fluent
|
|
74
98
|
end
|
75
99
|
|
76
100
|
if record_size > @bulk_limit
|
77
|
-
|
101
|
+
if @bulk_limit_warning_limit.is_a?(Integer)
|
102
|
+
log.warn "Record with size #{record_size} exceeds #{@bulk_limit} and can't be sent to Logz.io. Record starts with (truncated at #{@bulk_limit_warning_limit} characters): #{json_record[0,@bulk_limit_warning_limit]}"
|
103
|
+
# Send the full message to debug facility
|
104
|
+
log.debug "Record with size #{record_size} exceeds #{@bulk_limit} and can't be sent to Logz.io. Record is: #{json_record}"
|
105
|
+
else
|
106
|
+
log.warn "Record with size #{record_size} exceeds #{@bulk_limit} and can't be sent to Logz.io. Record is: #{json_record}"
|
107
|
+
end
|
78
108
|
next
|
79
109
|
end
|
80
110
|
if bulk_size + record_size > @bulk_limit
|
@@ -98,7 +128,9 @@ module Fluent
|
|
98
128
|
|
99
129
|
# Logz.io bulk http endpoint expecting log line with \n delimiter
|
100
130
|
post.body = bulk_records.join("\n")
|
101
|
-
|
131
|
+
if gzip
|
132
|
+
post.body = compress(post.body)
|
133
|
+
end
|
102
134
|
sleep_interval = @retry_sleep
|
103
135
|
|
104
136
|
begin
|
@@ -114,14 +146,14 @@ module Fluent
|
|
114
146
|
log.info "Got 400 code from Logz.io. This means that some of your logs are too big, or badly formatted. Response: #{response.body}"
|
115
147
|
should_retry = false
|
116
148
|
else
|
117
|
-
log.
|
149
|
+
log.warn "Got HTTP #{response.code} from Logz.io, not giving up just yet (Try #{counter + 1}/#{@retry_count})"
|
118
150
|
end
|
119
151
|
else
|
120
152
|
log.debug "Successfully sent bulk of #{bulk_records.size} records, size #{bulk_size}B to Logz.io"
|
121
153
|
should_retry = false
|
122
154
|
end
|
123
155
|
rescue StandardError => e
|
124
|
-
log.
|
156
|
+
log.warn "Error connecting to Logz.io. Got exception: #{e} (Try #{counter + 1}/#{@retry_count})"
|
125
157
|
end
|
126
158
|
|
127
159
|
if should_retry
|
@@ -139,5 +171,13 @@ module Fluent
|
|
139
171
|
log.error "Got unexpected exception! Here: #{e}"
|
140
172
|
end
|
141
173
|
end
|
174
|
+
|
175
|
+
def compress(string)
|
176
|
+
wio = StringIO.new("w")
|
177
|
+
w_gz = Zlib::GzipWriter.new(wio)
|
178
|
+
w_gz.write(string)
|
179
|
+
w_gz.close
|
180
|
+
wio.string
|
181
|
+
end
|
142
182
|
end
|
143
183
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
describe 'Fluent::LogzioOutputBuffered' do
|
4
|
-
let(:driver) { Fluent::Test::Driver::Output.new(Fluent::LogzioOutputBuffered).configure(config) }
|
3
|
+
describe 'Fluent::Plugin::LogzioOutputBuffered' do
|
4
|
+
let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::LogzioOutputBuffered).configure(config) }
|
5
5
|
let(:config) do
|
6
6
|
%[
|
7
7
|
endpoint_url https://logz.io?token=123
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
describe 'Fluent::LogzioOutputBuffered' do
|
4
|
-
let(:driver) { Fluent::Test::Driver::Output.new(Fluent::LogzioOutputBuffered).configure(config) }
|
3
|
+
describe 'Fluent::Plugin::LogzioOutputBuffered' do
|
4
|
+
let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::LogzioOutputBuffered).configure(config) }
|
5
5
|
let(:config) do
|
6
6
|
%[
|
7
7
|
endpoint_url https://logz.io?token=123
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
describe 'Fluent::LogzioOutputBuffered' do
|
4
|
-
let(:driver) { Fluent::Test::Driver::Output.new(Fluent::LogzioOutputBuffered).configure(config) }
|
3
|
+
describe 'Fluent::Plugin::LogzioOutputBuffered' do
|
4
|
+
let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::LogzioOutputBuffered).configure(config) }
|
5
5
|
let(:config) do
|
6
6
|
%[
|
7
7
|
endpoint_url https://logz.io?token=123
|
metadata
CHANGED
@@ -1,16 +1,17 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-logzio
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.21
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yury Kotov
|
8
8
|
- Roi Rav-Hon
|
9
9
|
- Arcadiy Ivanov
|
10
|
+
- Miri Ignatiev
|
10
11
|
autorequire:
|
11
12
|
bindir: bin
|
12
13
|
cert_chain: []
|
13
|
-
date:
|
14
|
+
date: 2021-03-01 00:00:00.000000000 Z
|
14
15
|
dependencies:
|
15
16
|
- !ruby/object:Gem::Dependency
|
16
17
|
name: net-http-persistent
|
@@ -18,14 +19,14 @@ dependencies:
|
|
18
19
|
requirements:
|
19
20
|
- - "~>"
|
20
21
|
- !ruby/object:Gem::Version
|
21
|
-
version: '
|
22
|
+
version: '3.0'
|
22
23
|
type: :runtime
|
23
24
|
prerelease: false
|
24
25
|
version_requirements: !ruby/object:Gem::Requirement
|
25
26
|
requirements:
|
26
27
|
- - "~>"
|
27
28
|
- !ruby/object:Gem::Version
|
28
|
-
version: '
|
29
|
+
version: '3.0'
|
29
30
|
- !ruby/object:Gem::Dependency
|
30
31
|
name: fluentd
|
31
32
|
requirement: !ruby/object:Gem::Requirement
|
@@ -107,12 +108,14 @@ email:
|
|
107
108
|
- bairkan@gmail.com
|
108
109
|
- roi@logz.io
|
109
110
|
- arcadiy@ivanov.biz
|
111
|
+
- miri.ignatiev@logz.io
|
110
112
|
executables: []
|
111
113
|
extensions: []
|
112
114
|
extra_rdoc_files: []
|
113
115
|
files:
|
114
116
|
- ".gitignore"
|
115
117
|
- ".rspec"
|
118
|
+
- ".whitesource"
|
116
119
|
- Gemfile
|
117
120
|
- LICENSE
|
118
121
|
- README.md
|
@@ -142,8 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
142
145
|
- !ruby/object:Gem::Version
|
143
146
|
version: '0'
|
144
147
|
requirements: []
|
145
|
-
|
146
|
-
rubygems_version: 2.5.1
|
148
|
+
rubygems_version: 3.0.3
|
147
149
|
signing_key:
|
148
150
|
specification_version: 4
|
149
151
|
summary: Fluentd plugin for output to Logz.io
|