fluent-plugin-logzio 0.1.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +19 -2
- data/fluent-plugin-logzio.gemspec +2 -1
- data/lib/fluent/plugin/out_logzio_buffered.rb +76 -7
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1f0dd01a3030ad599d784e84b7e1032c37cd540f7b642ba39aecf87bfa90df9f
|
4
|
+
data.tar.gz: '08e24b873ce51e284fad88609fa4ef81889f2bfb07b45d553b0a1a827d39a401'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2529a44b97de6d8fd2cebee1664b8617531970596237bb4235148beccc1aee8ceebd22cde6cd220846c2289032dc58976777f7bd7e1598732c57b6d0167ccbae
|
7
|
+
data.tar.gz: 8710e58b271bf793d5122df4290a9f15e02219ae45e2321f212956b7632e682a5f03ab0f0c3cdb494156109af3b2e1c2392fbac21623017a70a6597207f8fffc
|
data/README.md
CHANGED
@@ -70,13 +70,28 @@ This is an **example** only. Your needs in production may vary!
|
|
70
70
|
* **bulk_limit** Limit to the size of the Logz.io upload bulk. Defaults to 1000000 bytes leaving about 24kB for overhead.
|
71
71
|
* **bulk_limit_warning_limit** Limit to the size of the Logz.io warning message when a record exceeds bulk_limit to prevent a recursion when Fluent warnings are sent to the Logz.io output. Defaults to nil (no truncation).
|
72
72
|
* **proxy_uri** Your proxy uri. Default is nil. For example: "`my.ip:12345`".
|
73
|
-
* **proxy_cert** Your proxy cert. Default is nil
|
74
|
-
* **gzip** should the plugin ship the logs in gzip compression. Default is false
|
73
|
+
* **proxy_cert** Your proxy cert. Default is nil.
|
74
|
+
* **gzip** should the plugin ship the logs in gzip compression. Default is false.
|
75
|
+
|
76
|
+
|
77
|
+
## Plugin metrics:
|
78
|
+
|
79
|
+
| Metric Name | Description | Type | Example |
|
80
|
+
| --- | --- | --- | --- |
|
81
|
+
| `logzio_status_codes` | Status codes received from Logz.io | Gauge | `logzio_status_codes{type="logzio_buffered",plugin_id="out_logzio",status_code="500"}` |
|
75
82
|
|
76
83
|
|
77
84
|
## Release Notes
|
85
|
+
- **0.2.1**:
|
86
|
+
- Do not retry on 400 and 401. For 400 - try to fix log and resend.
|
87
|
+
- Generate a metric (`logzio_status_codes`) for response codes from Logz.io.
|
88
|
+
- **0.2.0**: N/A - version was yanked. Please refer to **0.2.1**.
|
78
89
|
- **0.1.0**:
|
79
90
|
- Use fluentd's retry instead of retry in code (raise exception on non-2xx response).
|
91
|
+
|
92
|
+
<details>
|
93
|
+
<summary markdown="span"> Expand to check old versions </summary>
|
94
|
+
|
80
95
|
- 0.0.22: Update gem `net-http-persistent` to 4.x.
|
81
96
|
- 0.0.21: Update gem `net-http-persistent` to 3.x.
|
82
97
|
- 0.0.20: Support gzip compression
|
@@ -87,3 +102,5 @@ This is an **example** only. Your needs in production may vary!
|
|
87
102
|
- 0.0.14: Refactor send function to handle more cases, and retry in case of logzio connection failure.
|
88
103
|
- 0.0.13: BREAKING - Removed non-buffered version. It's really not efficient, and should just not be used. If you are using this version, you should change to the buffered one.
|
89
104
|
- 0.0.12: Catch exception when parsing YAML to ignore (instead of crash) not valid logs.
|
105
|
+
|
106
|
+
</details>
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |s|
|
6
6
|
s.name = 'fluent-plugin-logzio'
|
7
|
-
s.version = '0.1
|
7
|
+
s.version = '0.2.1'
|
8
8
|
s.authors = ['Yury Kotov', 'Roi Rav-Hon', 'Arcadiy Ivanov', 'Miri Bar']
|
9
9
|
s.email = ['bairkan@gmail.com', 'roi@logz.io', 'arcadiy@ivanov.biz', 'miri.ignatiev@logz.io']
|
10
10
|
s.homepage = 'https://github.com/logzio/fluent-plugin-logzio'
|
@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
|
|
21
21
|
|
22
22
|
s.add_dependency 'net-http-persistent', '~> 4.0'
|
23
23
|
s.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
|
24
|
+
s.add_runtime_dependency 'prometheus-client', '>= 2.1.0'
|
24
25
|
s.add_development_dependency 'rake', '~> 12.3'
|
25
26
|
s.add_development_dependency 'bundler', '~> 1.16'
|
26
27
|
s.add_development_dependency 'rspec', '~> 3.7'
|
@@ -6,6 +6,7 @@ require 'stringio'
|
|
6
6
|
module Fluent::Plugin
|
7
7
|
class LogzioOutputBuffered < Output
|
8
8
|
Fluent::Plugin.register_output('logzio_buffered', self)
|
9
|
+
class RetryableResponse < StandardError; end
|
9
10
|
|
10
11
|
helpers :compat_parameters
|
11
12
|
|
@@ -37,9 +38,23 @@ module Fluent::Plugin
|
|
37
38
|
log.debug "Proxy #{@proxy_cert}"
|
38
39
|
ENV['SSL_CERT_FILE'] = @proxy_cert
|
39
40
|
end
|
41
|
+
@metric_labels = {
|
42
|
+
type: 'logzio_buffered',
|
43
|
+
plugin_id: 'out_logzio',
|
44
|
+
}
|
45
|
+
@metrics = {
|
46
|
+
status_codes: get_gauge(
|
47
|
+
:logzio_status_codes,
|
48
|
+
'Status codes received from Logz.io', {"status_code":""}),
|
49
|
+
}
|
40
50
|
|
41
51
|
end
|
42
52
|
|
53
|
+
def initialize
|
54
|
+
super
|
55
|
+
@registry = ::Prometheus::Client.registry
|
56
|
+
end
|
57
|
+
|
43
58
|
def start
|
44
59
|
super
|
45
60
|
require 'net/http/persistent'
|
@@ -121,6 +136,25 @@ module Fluent::Plugin
|
|
121
136
|
end
|
122
137
|
|
123
138
|
def send_bulk(bulk_records, bulk_size)
|
139
|
+
response = do_post(bulk_records, bulk_size)
|
140
|
+
|
141
|
+
@metrics[:status_codes].increment(labels: merge_labels({'status_code': response.code.to_s}))
|
142
|
+
|
143
|
+
if not response.code.start_with?('2')
|
144
|
+
if response.code == '400'
|
145
|
+
log.warn "Received #{response.code} from Logzio. Some logs may be malformed or too long. Valid logs were succesfully sent into the system. Will try to proccess and send oversized logs. Response body: #{response.body}"
|
146
|
+
process_code_400(bulk_records, Yajl.load(response.body))
|
147
|
+
elsif response.code == '401'
|
148
|
+
log.error "Received #{response.code} from Logzio. Unauthorized, please check your logs shipping token. Will not retry sending. Response body: #{response.body}"
|
149
|
+
else
|
150
|
+
log.debug "Failed request body: #{post.body}"
|
151
|
+
log.error "Error while sending POST to #{@uri}: #{response.body}"
|
152
|
+
raise RetryableResponse, "Logzio listener returned (#{response.code}) for #{@uri}: #{response.body}", []
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
def do_post(bulk_records, bulk_size)
|
124
158
|
log.debug "Sending a bulk of #{bulk_records.size} records, size #{bulk_size}B to Logz.io"
|
125
159
|
|
126
160
|
# Setting our request
|
@@ -136,16 +170,39 @@ module Fluent::Plugin
|
|
136
170
|
response = @http.request @uri, post
|
137
171
|
rescue Net::HTTP::Persistent::Error => e
|
138
172
|
raise e.cause
|
173
|
+
return response
|
139
174
|
end
|
175
|
+
end
|
140
176
|
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
177
|
+
def process_code_400(bulk_records, response_body)
|
178
|
+
max_log_field_size_bytes = 32000
|
179
|
+
oversized_logs_counter = response_body['oversizedLines'].to_i
|
180
|
+
new_bulk = []
|
181
|
+
for log_record in bulk_records
|
182
|
+
log.info "Oversized lines: #{oversized_logs_counter}" # todo
|
183
|
+
if oversized_logs_counter == 0
|
184
|
+
log.debug "No malformed lines, breaking"
|
185
|
+
break
|
186
|
+
end
|
187
|
+
new_log = Yajl.load(log_record)
|
188
|
+
msg_size = new_log['message'].size
|
189
|
+
# Handle oversized log:
|
190
|
+
if msg_size >= max_log_field_size_bytes
|
191
|
+
new_log['message'] = new_log['message'][0, max_log_field_size_bytes - 1]
|
192
|
+
log.debug "new log: #{new_log}"
|
193
|
+
new_bulk.append(Yajl.dump(new_log))
|
194
|
+
oversized_logs_counter -= 1
|
195
|
+
end
|
196
|
+
end
|
197
|
+
if new_bulk.size > 0
|
198
|
+
log.debug "Number of fixed bad logs to send: #{new_bulk.size}"
|
199
|
+
response = do_post(new_bulk, new_bulk.size)
|
200
|
+
if response.code.start_with?('2')
|
201
|
+
log.info "Succesfully sent bad logs"
|
202
|
+
else
|
203
|
+
log.warn "While trying to send fixed bad logs, got #{response.code} from Logz.io, will not try to re-send"
|
204
|
+
end
|
146
205
|
end
|
147
|
-
|
148
|
-
raise "Logzio listener returned (#{response.code}) for #{@uri}: #{response.body}" if resp_err
|
149
206
|
end
|
150
207
|
|
151
208
|
def compress(string)
|
@@ -155,5 +212,17 @@ module Fluent::Plugin
|
|
155
212
|
w_gz.close
|
156
213
|
wio.string
|
157
214
|
end
|
215
|
+
|
216
|
+
def merge_labels(extra_labels= {})
|
217
|
+
@metric_labels.merge extra_labels
|
218
|
+
end
|
219
|
+
|
220
|
+
def get_gauge(name, docstring, extra_labels = {})
|
221
|
+
if @registry.exist?(name)
|
222
|
+
@registry.get(name)
|
223
|
+
else
|
224
|
+
@registry.gauge(name, docstring: docstring, labels: @metric_labels.keys + extra_labels.keys)
|
225
|
+
end
|
226
|
+
end
|
158
227
|
end
|
159
228
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-logzio
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yury Kotov
|
@@ -11,7 +11,7 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date: 2023-
|
14
|
+
date: 2023-07-13 00:00:00.000000000 Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: net-http-persistent
|
@@ -47,6 +47,20 @@ dependencies:
|
|
47
47
|
- - "<"
|
48
48
|
- !ruby/object:Gem::Version
|
49
49
|
version: '2'
|
50
|
+
- !ruby/object:Gem::Dependency
|
51
|
+
name: prometheus-client
|
52
|
+
requirement: !ruby/object:Gem::Requirement
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: 2.1.0
|
57
|
+
type: :runtime
|
58
|
+
prerelease: false
|
59
|
+
version_requirements: !ruby/object:Gem::Requirement
|
60
|
+
requirements:
|
61
|
+
- - ">="
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: 2.1.0
|
50
64
|
- !ruby/object:Gem::Dependency
|
51
65
|
name: rake
|
52
66
|
requirement: !ruby/object:Gem::Requirement
|