fluent-plugin-logzio 0.1.0 → 0.2.0.pre.test1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +12 -2
- data/fluent-plugin-logzio.gemspec +2 -1
- data/lib/fluent/plugin/out_logzio_buffered.rb +78 -7
- metadata +18 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8ff307f46dc9f34f4f4cdd60374c340c81ab328c6e1485d8cf77d7c833709b9e
|
4
|
+
data.tar.gz: 286c954bb9c4a1e1ead99edf25e96a77556254b58afdb74dcccba4773e1a0a10
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 85ff70985c0fda5b6220c8cf6e0fa1306525940828c03c75d097bebe9cda65e6e6097f020454c06e4b17137d02a7b77c1bf492c5e0261f2dad45336357e554ab
|
7
|
+
data.tar.gz: dafe99b38c2d4f9e33efa7c71334d21c6493318b9d407a90cb13a5d9b0877df36af0bc047f166697ec0b816be3b79018298a6a4742a78191c5836dfd275b0515
|
data/README.md
CHANGED
@@ -70,11 +70,21 @@ This is an **example** only. Your needs in production may vary!
|
|
70
70
|
* **bulk_limit** Limit to the size of the Logz.io upload bulk. Defaults to 1000000 bytes leaving about 24kB for overhead.
|
71
71
|
* **bulk_limit_warning_limit** Limit to the size of the Logz.io warning message when a record exceeds bulk_limit to prevent a recursion when Fluent warnings are sent to the Logz.io output. Defaults to nil (no truncation).
|
72
72
|
* **proxy_uri** Your proxy uri. Default is nil. For example: "`my.ip:12345`".
|
73
|
-
* **proxy_cert** Your proxy cert. Default is nil
|
74
|
-
* **gzip** should the plugin ship the logs in gzip compression. Default is false
|
73
|
+
* **proxy_cert** Your proxy cert. Default is nil.
|
74
|
+
* **gzip** should the plugin ship the logs in gzip compression. Default is false.
|
75
|
+
|
76
|
+
|
77
|
+
## Plugin metrics:
|
78
|
+
|
79
|
+
| Metric Name | Description | Type | Example |
|
80
|
+
| --- | --- | --- | --- |
|
81
|
+
| `logzio_status_codes` | Status codes received from Logz.io | Gauge | `logzio_status_codes{type="logzio_buffered",plugin_id="out_logzio",status_code="500"}` |
|
75
82
|
|
76
83
|
|
77
84
|
## Release Notes
|
85
|
+
- **0.2.0**:
|
86
|
+
- Do not retry on 400 and 401. For 400 - try to fix log and resend.
|
87
|
+
- Generate a metric (`logzio_status_codes`) for response codes from Logz.io.
|
78
88
|
- **0.1.0**:
|
79
89
|
- Use fluentd's retry instead of retry in code (raise exception on non-2xx response).
|
80
90
|
- 0.0.22: Update gem `net-http-persistent` to 4.x.
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |s|
|
6
6
|
s.name = 'fluent-plugin-logzio'
|
7
|
-
s.version = '0.
|
7
|
+
s.version = '0.2.0-test1'
|
8
8
|
s.authors = ['Yury Kotov', 'Roi Rav-Hon', 'Arcadiy Ivanov', 'Miri Bar']
|
9
9
|
s.email = ['bairkan@gmail.com', 'roi@logz.io', 'arcadiy@ivanov.biz', 'miri.ignatiev@logz.io']
|
10
10
|
s.homepage = 'https://github.com/logzio/fluent-plugin-logzio'
|
@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
|
|
21
21
|
|
22
22
|
s.add_dependency 'net-http-persistent', '~> 4.0'
|
23
23
|
s.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
|
24
|
+
s.add_runtime_dependency 'prometheus-client', '>= 2.1.0'
|
24
25
|
s.add_development_dependency 'rake', '~> 12.3'
|
25
26
|
s.add_development_dependency 'bundler', '~> 1.16'
|
26
27
|
s.add_development_dependency 'rspec', '~> 3.7'
|
@@ -6,6 +6,7 @@ require 'stringio'
|
|
6
6
|
module Fluent::Plugin
|
7
7
|
class LogzioOutputBuffered < Output
|
8
8
|
Fluent::Plugin.register_output('logzio_buffered', self)
|
9
|
+
class RetryableResponse < StandardError; end
|
9
10
|
|
10
11
|
helpers :compat_parameters
|
11
12
|
|
@@ -37,9 +38,23 @@ module Fluent::Plugin
|
|
37
38
|
log.debug "Proxy #{@proxy_cert}"
|
38
39
|
ENV['SSL_CERT_FILE'] = @proxy_cert
|
39
40
|
end
|
41
|
+
@metric_labels = {
|
42
|
+
type: 'logzio_buffered',
|
43
|
+
plugin_id: 'out_logzio',
|
44
|
+
}
|
45
|
+
@metrics = {
|
46
|
+
status_codes: get_gauge(
|
47
|
+
:logzio_status_codes,
|
48
|
+
'Status codes received from Logz.io', {"status_code":""}),
|
49
|
+
}
|
40
50
|
|
41
51
|
end
|
42
52
|
|
53
|
+
def initialize
|
54
|
+
super
|
55
|
+
@registry = ::Prometheus::Client.registry
|
56
|
+
end
|
57
|
+
|
43
58
|
def start
|
44
59
|
super
|
45
60
|
require 'net/http/persistent'
|
@@ -121,6 +136,27 @@ module Fluent::Plugin
|
|
121
136
|
end
|
122
137
|
|
123
138
|
def send_bulk(bulk_records, bulk_size)
|
139
|
+
response = do_post(bulk_records, bulk_size)
|
140
|
+
|
141
|
+
@metrics[:status_codes].increment(labels: merge_labels({'status_code': response.code.to_s}))
|
142
|
+
log.info "Status code from logzio: #{response.code}"
|
143
|
+
|
144
|
+
if not response.code.start_with?('2')
|
145
|
+
log.info "here here here"
|
146
|
+
if response.code == '400'
|
147
|
+
log.warn "Received #{response.code} from Logzio. Some logs may be malformed or too long. Valid logs were succesfully sent into the system. Will try to proccess and send oversized logs. Response body: #{response.body}"
|
148
|
+
process_code_400(bulk_records, Yajl.load(response.body))
|
149
|
+
elsif response.code == '401'
|
150
|
+
log.error "Received #{response.code} from Logzio. Unauthorized, please check your logs shipping token. Will not retry sending. Response body: #{response.body}"
|
151
|
+
else
|
152
|
+
log.debug "Failed request body: #{post.body}"
|
153
|
+
log.error "Error while sending POST to #{@uri}: #{response.body}"
|
154
|
+
raise RetryableResponse, "Logzio listener returned (#{response.code}) for #{@uri}: #{response.body}", []
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def do_post(bulk_records, bulk_size)
|
124
160
|
log.debug "Sending a bulk of #{bulk_records.size} records, size #{bulk_size}B to Logz.io"
|
125
161
|
|
126
162
|
# Setting our request
|
@@ -136,16 +172,39 @@ module Fluent::Plugin
|
|
136
172
|
response = @http.request @uri, post
|
137
173
|
rescue Net::HTTP::Persistent::Error => e
|
138
174
|
raise e.cause
|
175
|
+
return response
|
139
176
|
end
|
177
|
+
end
|
140
178
|
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
179
|
+
def process_code_400(bulk_records, response_body)
|
180
|
+
max_log_field_size_bytes = 32000
|
181
|
+
oversized_logs_counter = response_body['oversizedLines'].to_i
|
182
|
+
new_bulk = []
|
183
|
+
for log_record in bulk_records
|
184
|
+
log.info "Oversized lines: #{oversized_logs_counter}" # todo
|
185
|
+
if oversized_logs_counter == 0
|
186
|
+
log.debug "No malformed lines, breaking"
|
187
|
+
break
|
188
|
+
end
|
189
|
+
new_log = Yajl.load(log_record)
|
190
|
+
msg_size = new_log['message'].size
|
191
|
+
# Handle oversized log:
|
192
|
+
if msg_size >= max_log_field_size_bytes
|
193
|
+
new_log['message'] = new_log['message'][0, max_log_field_size_bytes - 1]
|
194
|
+
log.debug "new log: #{new_log}"
|
195
|
+
new_bulk.append(Yajl.dump(new_log))
|
196
|
+
oversized_logs_counter -= 1
|
197
|
+
end
|
198
|
+
end
|
199
|
+
if new_bulk.size > 0
|
200
|
+
log.debug "Number of fixed bad logs to send: #{new_bulk.size}"
|
201
|
+
response = do_post(new_bulk, new_bulk.size)
|
202
|
+
if response.code.start_with?('2')
|
203
|
+
log.info "Succesfully sent bad logs"
|
204
|
+
else
|
205
|
+
log.warn "While trying to send fixed bad logs, got #{response.code} from Logz.io, will not try to re-send"
|
206
|
+
end
|
146
207
|
end
|
147
|
-
|
148
|
-
raise "Logzio listener returned (#{response.code}) for #{@uri}: #{response.body}" if resp_err
|
149
208
|
end
|
150
209
|
|
151
210
|
def compress(string)
|
@@ -155,5 +214,17 @@ module Fluent::Plugin
|
|
155
214
|
w_gz.close
|
156
215
|
wio.string
|
157
216
|
end
|
217
|
+
|
218
|
+
def merge_labels(extra_labels= {})
|
219
|
+
@metric_labels.merge extra_labels
|
220
|
+
end
|
221
|
+
|
222
|
+
def get_gauge(name, docstring, extra_labels = {})
|
223
|
+
if @registry.exist?(name)
|
224
|
+
@registry.get(name)
|
225
|
+
else
|
226
|
+
@registry.gauge(name, docstring: docstring, labels: @metric_labels.keys + extra_labels.keys)
|
227
|
+
end
|
228
|
+
end
|
158
229
|
end
|
159
230
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-logzio
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0.pre.test1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yury Kotov
|
@@ -11,7 +11,7 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date: 2023-
|
14
|
+
date: 2023-07-12 00:00:00.000000000 Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: net-http-persistent
|
@@ -47,6 +47,20 @@ dependencies:
|
|
47
47
|
- - "<"
|
48
48
|
- !ruby/object:Gem::Version
|
49
49
|
version: '2'
|
50
|
+
- !ruby/object:Gem::Dependency
|
51
|
+
name: prometheus-client
|
52
|
+
requirement: !ruby/object:Gem::Requirement
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: 2.1.0
|
57
|
+
type: :runtime
|
58
|
+
prerelease: false
|
59
|
+
version_requirements: !ruby/object:Gem::Requirement
|
60
|
+
requirements:
|
61
|
+
- - ">="
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: 2.1.0
|
50
64
|
- !ruby/object:Gem::Dependency
|
51
65
|
name: rake
|
52
66
|
requirement: !ruby/object:Gem::Requirement
|
@@ -142,9 +156,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
142
156
|
version: 2.1.0
|
143
157
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
144
158
|
requirements:
|
145
|
-
- - "
|
159
|
+
- - ">"
|
146
160
|
- !ruby/object:Gem::Version
|
147
|
-
version:
|
161
|
+
version: 1.3.1
|
148
162
|
requirements: []
|
149
163
|
rubygems_version: 3.0.3.1
|
150
164
|
signing_key:
|