fluent-plugin-logzio 0.1.0 → 0.2.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: de65dcc352aca420ba2469b1c31c0c3f135309b7728d501ca1eb27e09259e322
4
- data.tar.gz: b92016ff7b59405e6e9d45b24ae87fb0d60d396b06bc6ae81b681991d45a56ea
3
+ metadata.gz: 1f0dd01a3030ad599d784e84b7e1032c37cd540f7b642ba39aecf87bfa90df9f
4
+ data.tar.gz: '08e24b873ce51e284fad88609fa4ef81889f2bfb07b45d553b0a1a827d39a401'
5
5
  SHA512:
6
- metadata.gz: f167bd4d0d6c60612dff909d2490d734808eb07894c7396af321351ae741c8fc9b60193209ff92819a50f74089c4d72f91bf24393e27f74a2da3246fef22d5a3
7
- data.tar.gz: ccb0be4ca90d64ac4626adc6efbc05f46fde74bad6b505535c298f6b824a6e4e8d349a21128cddd211c212f730f9501c75b76c5f9c02efed096ea05e44a967de
6
+ metadata.gz: 2529a44b97de6d8fd2cebee1664b8617531970596237bb4235148beccc1aee8ceebd22cde6cd220846c2289032dc58976777f7bd7e1598732c57b6d0167ccbae
7
+ data.tar.gz: 8710e58b271bf793d5122df4290a9f15e02219ae45e2321f212956b7632e682a5f03ab0f0c3cdb494156109af3b2e1c2392fbac21623017a70a6597207f8fffc
data/README.md CHANGED
@@ -70,13 +70,28 @@ This is an **example** only. Your needs in production may vary!
70
70
  * **bulk_limit** Limit to the size of the Logz.io upload bulk. Defaults to 1000000 bytes leaving about 24kB for overhead.
71
71
  * **bulk_limit_warning_limit** Limit to the size of the Logz.io warning message when a record exceeds bulk_limit to prevent a recursion when Fluent warnings are sent to the Logz.io output. Defaults to nil (no truncation).
72
72
  * **proxy_uri** Your proxy uri. Default is nil. For example: "`my.ip:12345`".
73
- * **proxy_cert** Your proxy cert. Default is nil
74
- * **gzip** should the plugin ship the logs in gzip compression. Default is false
73
+ * **proxy_cert** Your proxy cert. Default is nil.
74
+ * **gzip** should the plugin ship the logs in gzip compression. Default is false.
75
+
76
+
77
+ ## Plugin metrics:
78
+
79
+ | Metric Name | Description | Type | Example |
80
+ | --- | --- | --- | --- |
81
+ | `logzio_status_codes` | Status codes received from Logz.io | Gauge | `logzio_status_codes{type="logzio_buffered",plugin_id="out_logzio",status_code="500"}` |
75
82
 
76
83
 
77
84
  ## Release Notes
85
+ - **0.2.1**:
86
+ - Do not retry on 400 and 401. For 400 - try to fix log and resend.
87
+ - Generate a metric (`logzio_status_codes`) for response codes from Logz.io.
88
+ - **0.2.0**: N/A - version was yanked. Please refer to **0.2.1**.
78
89
  - **0.1.0**:
79
90
  - Use fluentd's retry instead of retry in code (raise exception on non-2xx response).
91
+
92
+ <details>
93
+ <summary markdown="span"> Expand to check old versions </summary>
94
+
80
95
  - 0.0.22: Update gem `net-http-persistent` to 4.x.
81
96
  - 0.0.21: Update gem `net-http-persistent` to 3.x.
82
97
  - 0.0.20: Support gzip compression
@@ -87,3 +102,5 @@ This is an **example** only. Your needs in production may vary!
87
102
  - 0.0.14: Refactor send function to handle more cases, and retry in case of logzio connection failure.
88
103
  - 0.0.13: BREAKING - Removed non-buffered version. It's really not efficient, and should just not be used. If you are using this version, you should change to the buffered one.
89
104
  - 0.0.12: Catch exception when parsing YAML to ignore (instead of crash) not valid logs.
105
+
106
+ </details>
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |s|
6
6
  s.name = 'fluent-plugin-logzio'
7
- s.version = '0.1.0'
7
+ s.version = '0.2.1'
8
8
  s.authors = ['Yury Kotov', 'Roi Rav-Hon', 'Arcadiy Ivanov', 'Miri Bar']
9
9
  s.email = ['bairkan@gmail.com', 'roi@logz.io', 'arcadiy@ivanov.biz', 'miri.ignatiev@logz.io']
10
10
  s.homepage = 'https://github.com/logzio/fluent-plugin-logzio'
@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
21
21
 
22
22
  s.add_dependency 'net-http-persistent', '~> 4.0'
23
23
  s.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
24
+ s.add_runtime_dependency 'prometheus-client', '>= 2.1.0'
24
25
  s.add_development_dependency 'rake', '~> 12.3'
25
26
  s.add_development_dependency 'bundler', '~> 1.16'
26
27
  s.add_development_dependency 'rspec', '~> 3.7'
@@ -6,6 +6,7 @@ require 'stringio'
6
6
  module Fluent::Plugin
7
7
  class LogzioOutputBuffered < Output
8
8
  Fluent::Plugin.register_output('logzio_buffered', self)
9
+ class RetryableResponse < StandardError; end
9
10
 
10
11
  helpers :compat_parameters
11
12
 
@@ -37,9 +38,23 @@ module Fluent::Plugin
37
38
  log.debug "Proxy #{@proxy_cert}"
38
39
  ENV['SSL_CERT_FILE'] = @proxy_cert
39
40
  end
41
+ @metric_labels = {
42
+ type: 'logzio_buffered',
43
+ plugin_id: 'out_logzio',
44
+ }
45
+ @metrics = {
46
+ status_codes: get_gauge(
47
+ :logzio_status_codes,
48
+ 'Status codes received from Logz.io', {"status_code":""}),
49
+ }
40
50
 
41
51
  end
42
52
 
53
+ def initialize
54
+ super
55
+ @registry = ::Prometheus::Client.registry
56
+ end
57
+
43
58
  def start
44
59
  super
45
60
  require 'net/http/persistent'
@@ -121,6 +136,25 @@ module Fluent::Plugin
121
136
  end
122
137
 
123
138
  def send_bulk(bulk_records, bulk_size)
139
+ response = do_post(bulk_records, bulk_size)
140
+
141
+ @metrics[:status_codes].increment(labels: merge_labels({'status_code': response.code.to_s}))
142
+
143
+ if not response.code.start_with?('2')
144
+ if response.code == '400'
145
+ log.warn "Received #{response.code} from Logzio. Some logs may be malformed or too long. Valid logs were succesfully sent into the system. Will try to proccess and send oversized logs. Response body: #{response.body}"
146
+ process_code_400(bulk_records, Yajl.load(response.body))
147
+ elsif response.code == '401'
148
+ log.error "Received #{response.code} from Logzio. Unauthorized, please check your logs shipping token. Will not retry sending. Response body: #{response.body}"
149
+ else
150
+ log.debug "Failed request body: #{post.body}"
151
+ log.error "Error while sending POST to #{@uri}: #{response.body}"
152
+ raise RetryableResponse, "Logzio listener returned (#{response.code}) for #{@uri}: #{response.body}", []
153
+ end
154
+ end
155
+ end
156
+
157
+ def do_post(bulk_records, bulk_size)
124
158
  log.debug "Sending a bulk of #{bulk_records.size} records, size #{bulk_size}B to Logz.io"
125
159
 
126
160
  # Setting our request
@@ -136,16 +170,39 @@ module Fluent::Plugin
136
170
  response = @http.request @uri, post
137
171
  rescue Net::HTTP::Persistent::Error => e
138
172
  raise e.cause
173
+ return response
139
174
  end
175
+ end
140
176
 
141
- resp_err = response.code.to_s.start_with?('4') || response.code.to_s.start_with?('5')
142
-
143
- if not response.code.start_with?('2')
144
- log.debug "Failed request body: #{post.body}"
145
- log.error "Error while sending POST to #{@uri}: #{response.body}"
177
+ def process_code_400(bulk_records, response_body)
178
+ max_log_field_size_bytes = 32000
179
+ oversized_logs_counter = response_body['oversizedLines'].to_i
180
+ new_bulk = []
181
+ for log_record in bulk_records
182
+ log.info "Oversized lines: #{oversized_logs_counter}" # todo
183
+ if oversized_logs_counter == 0
184
+ log.debug "No malformed lines, breaking"
185
+ break
186
+ end
187
+ new_log = Yajl.load(log_record)
188
+ msg_size = new_log['message'].size
189
+ # Handle oversized log:
190
+ if msg_size >= max_log_field_size_bytes
191
+ new_log['message'] = new_log['message'][0, max_log_field_size_bytes - 1]
192
+ log.debug "new log: #{new_log}"
193
+ new_bulk.append(Yajl.dump(new_log))
194
+ oversized_logs_counter -= 1
195
+ end
196
+ end
197
+ if new_bulk.size > 0
198
+ log.debug "Number of fixed bad logs to send: #{new_bulk.size}"
199
+ response = do_post(new_bulk, new_bulk.size)
200
+ if response.code.start_with?('2')
201
+ log.info "Succesfully sent bad logs"
202
+ else
203
+ log.warn "While trying to send fixed bad logs, got #{response.code} from Logz.io, will not try to re-send"
204
+ end
146
205
  end
147
-
148
- raise "Logzio listener returned (#{response.code}) for #{@uri}: #{response.body}" if resp_err
149
206
  end
150
207
 
151
208
  def compress(string)
@@ -155,5 +212,17 @@ module Fluent::Plugin
155
212
  w_gz.close
156
213
  wio.string
157
214
  end
215
+
216
+ def merge_labels(extra_labels= {})
217
+ @metric_labels.merge extra_labels
218
+ end
219
+
220
+ def get_gauge(name, docstring, extra_labels = {})
221
+ if @registry.exist?(name)
222
+ @registry.get(name)
223
+ else
224
+ @registry.gauge(name, docstring: docstring, labels: @metric_labels.keys + extra_labels.keys)
225
+ end
226
+ end
158
227
  end
159
228
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-logzio
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yury Kotov
@@ -11,7 +11,7 @@ authors:
11
11
  autorequire:
12
12
  bindir: bin
13
13
  cert_chain: []
14
- date: 2023-05-01 00:00:00.000000000 Z
14
+ date: 2023-07-13 00:00:00.000000000 Z
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
17
  name: net-http-persistent
@@ -47,6 +47,20 @@ dependencies:
47
47
  - - "<"
48
48
  - !ruby/object:Gem::Version
49
49
  version: '2'
50
+ - !ruby/object:Gem::Dependency
51
+ name: prometheus-client
52
+ requirement: !ruby/object:Gem::Requirement
53
+ requirements:
54
+ - - ">="
55
+ - !ruby/object:Gem::Version
56
+ version: 2.1.0
57
+ type: :runtime
58
+ prerelease: false
59
+ version_requirements: !ruby/object:Gem::Requirement
60
+ requirements:
61
+ - - ">="
62
+ - !ruby/object:Gem::Version
63
+ version: 2.1.0
50
64
  - !ruby/object:Gem::Dependency
51
65
  name: rake
52
66
  requirement: !ruby/object:Gem::Requirement