logstash-output-sensors_analytics 0.1.2 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/logstash/outputs/sensors_analytics.rb +72 -20
- data/logstash-output-sensors_analytics.gemspec +1 -1
- metadata +10 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bbcf496f3c8862f0febbb3fbb018ee6a4275493437f9d7dd4137794a6e9cd214
|
4
|
+
data.tar.gz: fe931cc0014ae67b9bf6e11f0de3a4a8ed6dd2b06c24567f84edc09c11ad8ab3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cc258f6d684517793e9fd76f10000ee6c3cf74f6c89dcbe72974eff67e0a4b3956e55906aab5435fab87e1ec02ad10ab432199c7915e50883ddc00d056acb2b6
|
7
|
+
data.tar.gz: e86c0cc001ed9c7dba6820b81d99036e672daf2add49ef8dcfc04905931ba212c86b91067110beca6556537e91d4d28815860e4621df6c288533a9b0bc9f3a64
|
@@ -32,7 +32,7 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
|
|
32
32
|
# 开启 filebeat 状态记录
|
33
33
|
config :enable_filebeat_status_report, :validate => :boolean, :default => true
|
34
34
|
|
35
|
-
PLUGIN_VERSION = "0.1.
|
35
|
+
PLUGIN_VERSION = "0.1.4"
|
36
36
|
|
37
37
|
public
|
38
38
|
|
@@ -116,8 +116,17 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
|
|
116
116
|
|
117
117
|
buffer_item = @buffer_items[buffer_index(tag)]
|
118
118
|
buffer_item.buffer_receive(record)
|
119
|
-
rescue
|
120
|
-
|
119
|
+
rescue => ex
|
120
|
+
|
121
|
+
@logger.error("Could not process record",
|
122
|
+
:exception => ex,
|
123
|
+
:exceptionClassName => ex.class.name,
|
124
|
+
:exceptionMessage => ex.message,
|
125
|
+
:exceptionBacktrace => ex.backtrace,
|
126
|
+
:exceptionCause => ex.cause,
|
127
|
+
:exceptionInspect => ex.inspect,
|
128
|
+
:record => e.to_s)
|
129
|
+
|
121
130
|
@parse_error_count += 1
|
122
131
|
end
|
123
132
|
end
|
@@ -196,11 +205,15 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
|
|
196
205
|
url_send_count_sum[url] = 0
|
197
206
|
end
|
198
207
|
|
208
|
+
flush_error_count_sum = 0
|
209
|
+
|
199
210
|
@buffer_items.each do |buffer_item|
|
200
211
|
buffer_url_send_count = buffer_item.url_send_count
|
201
212
|
buffer_url_send_count.each do |url, count|
|
202
213
|
url_send_count_sum[url] += count
|
203
214
|
end
|
215
|
+
|
216
|
+
flush_error_count_sum += buffer_item.flush_error_count
|
204
217
|
end
|
205
218
|
|
206
219
|
total_send_count = 0
|
@@ -216,6 +229,7 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
|
|
216
229
|
:receive_count => @receive_count,
|
217
230
|
:send_count => total_send_count,
|
218
231
|
:parse_error_count => @parse_error_count,
|
232
|
+
:flush_error_count => flush_error_count_sum,
|
219
233
|
:url_send_count => url_send_count_sum)
|
220
234
|
@logger.info("Filebeat status Report: #{format_filebeat_report_and_clean}") if @enable_filebeat_status_report
|
221
235
|
end
|
@@ -228,6 +242,7 @@ class BufferItem
|
|
228
242
|
|
229
243
|
attr_accessor :buffer_state
|
230
244
|
attr_accessor :url_send_count
|
245
|
+
attr_accessor :flush_error_count
|
231
246
|
|
232
247
|
def initialize(option = {})
|
233
248
|
@client = option[:client]
|
@@ -236,6 +251,7 @@ class BufferItem
|
|
236
251
|
url_list.each do |url|
|
237
252
|
@url_send_count[url] = 0
|
238
253
|
end
|
254
|
+
@flush_error_count = 0
|
239
255
|
init_url_list(url_list, option[:index])
|
240
256
|
@logger = option[:logger]
|
241
257
|
|
@@ -254,8 +270,16 @@ class BufferItem
|
|
254
270
|
@logger.warn("Send failed, code: #{response.code}, body: #{response.body}")
|
255
271
|
return false
|
256
272
|
end
|
257
|
-
rescue =>
|
258
|
-
|
273
|
+
rescue => ex
|
274
|
+
|
275
|
+
@logger.error("Send failed",
|
276
|
+
:exception => ex,
|
277
|
+
:exceptionClassName => ex.class.name,
|
278
|
+
:exceptionMessage => ex.message,
|
279
|
+
:exceptionBacktrace => ex.backtrace,
|
280
|
+
:exceptionCause => ex.cause,
|
281
|
+
:exceptionInspect => ex.inspect)
|
282
|
+
|
259
283
|
return false
|
260
284
|
end
|
261
285
|
true
|
@@ -267,23 +291,51 @@ class BufferItem
|
|
267
291
|
# 如果当前 url 发送失败, 会尝试获取列表中下一个地址进行发送, 发送失败的 url 在 3 秒内不会再尝试发送
|
268
292
|
# 如果所有的 url 都被标记为发送失败, sleep 5 秒后重新获取
|
269
293
|
def flush(events, final)
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
until do_send(form_data, url_item[:url])
|
280
|
-
last_url = url_item[:url]
|
281
|
-
# 将发送失败的 url 标记为不可用
|
282
|
-
disable_url(url_item)
|
294
|
+
begin
|
295
|
+
wio = StringIO.new("w")
|
296
|
+
gzip_io = Zlib::GzipWriter.new(wio)
|
297
|
+
gzip_io.write(events.to_json)
|
298
|
+
gzip_io.close
|
299
|
+
data = Base64.strict_encode64(wio.string)
|
300
|
+
form_data = {"data_list" => data, "gzip" => 1}
|
301
|
+
|
283
302
|
url_item = obtain_url
|
284
|
-
|
303
|
+
|
304
|
+
until do_send(form_data, url_item[:url])
|
305
|
+
last_url = url_item[:url]
|
306
|
+
# 将发送失败的 url 标记为不可用
|
307
|
+
disable_url(url_item)
|
308
|
+
url_item = obtain_url
|
309
|
+
@logger.warn("Send failed, retry send data to another url", :last_url => last_url, :retry_url => url_item[:url])
|
310
|
+
end
|
311
|
+
@url_send_count[url_item[:url]] += events.length
|
312
|
+
rescue => ex
|
313
|
+
|
314
|
+
if events.length == 1
|
315
|
+
# 打印异常现场
|
316
|
+
@logger.error("Flush failed",
|
317
|
+
:exception => ex,
|
318
|
+
:exceptionClassName => ex.class.name,
|
319
|
+
:exceptionMessage => ex.message,
|
320
|
+
:exceptionBacktrace => ex.backtrace,
|
321
|
+
:exceptionCause => ex.cause,
|
322
|
+
:exceptionInspect => ex.inspect,
|
323
|
+
:events_data => events.to_s)
|
324
|
+
|
325
|
+
@flush_error_count += 1
|
326
|
+
|
327
|
+
elsif events.length > 1
|
328
|
+
# 降级处理: 将events 的批量发送,降级为指数级后退重试
|
329
|
+
pivot = events.length / 2
|
330
|
+
left = events.slice(0, pivot)
|
331
|
+
right = events.slice(pivot, events.length)
|
332
|
+
flush(left, final)
|
333
|
+
flush(right, final)
|
334
|
+
|
335
|
+
@logger.warn("Flush retry in exponential backoff!", :left_length => left.length, :right_length => right.length)
|
336
|
+
end
|
337
|
+
|
285
338
|
end
|
286
|
-
@url_send_count[url_item[:url]] += events.length
|
287
339
|
end
|
288
340
|
|
289
341
|
private
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-sensors_analytics'
|
3
|
-
s.version = '0.1.
|
3
|
+
s.version = '0.1.4'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = 'Output plugin for Sensors Analytics'
|
6
6
|
s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program.'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-sensors_analytics
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Feng Jiajie
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-10-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -20,8 +20,8 @@ dependencies:
|
|
20
20
|
- !ruby/object:Gem::Version
|
21
21
|
version: '2.99'
|
22
22
|
name: logstash-core-plugin-api
|
23
|
-
prerelease: false
|
24
23
|
type: :runtime
|
24
|
+
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
@@ -37,8 +37,8 @@ dependencies:
|
|
37
37
|
- !ruby/object:Gem::Version
|
38
38
|
version: '0'
|
39
39
|
name: logstash-codec-plain
|
40
|
-
prerelease: false
|
41
40
|
type: :runtime
|
41
|
+
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
44
|
- - ">="
|
@@ -54,8 +54,8 @@ dependencies:
|
|
54
54
|
- !ruby/object:Gem::Version
|
55
55
|
version: 8.0.0
|
56
56
|
name: logstash-mixin-http_client
|
57
|
-
prerelease: false
|
58
57
|
type: :runtime
|
58
|
+
prerelease: false
|
59
59
|
version_requirements: !ruby/object:Gem::Requirement
|
60
60
|
requirements:
|
61
61
|
- - ">="
|
@@ -71,8 +71,8 @@ dependencies:
|
|
71
71
|
- !ruby/object:Gem::Version
|
72
72
|
version: 0.0.23
|
73
73
|
name: stud
|
74
|
-
prerelease: false
|
75
74
|
type: :runtime
|
75
|
+
prerelease: false
|
76
76
|
version_requirements: !ruby/object:Gem::Requirement
|
77
77
|
requirements:
|
78
78
|
- - "~>"
|
@@ -85,8 +85,8 @@ dependencies:
|
|
85
85
|
- !ruby/object:Gem::Version
|
86
86
|
version: '0'
|
87
87
|
name: logstash-devutils
|
88
|
-
prerelease: false
|
89
88
|
type: :development
|
89
|
+
prerelease: false
|
90
90
|
version_requirements: !ruby/object:Gem::Requirement
|
91
91
|
requirements:
|
92
92
|
- - ">="
|
@@ -99,8 +99,8 @@ dependencies:
|
|
99
99
|
- !ruby/object:Gem::Version
|
100
100
|
version: '0'
|
101
101
|
name: sinatra
|
102
|
-
prerelease: false
|
103
102
|
type: :development
|
103
|
+
prerelease: false
|
104
104
|
version_requirements: !ruby/object:Gem::Requirement
|
105
105
|
requirements:
|
106
106
|
- - ">="
|
@@ -113,8 +113,8 @@ dependencies:
|
|
113
113
|
- !ruby/object:Gem::Version
|
114
114
|
version: '0'
|
115
115
|
name: webrick
|
116
|
-
prerelease: false
|
117
116
|
type: :development
|
117
|
+
prerelease: false
|
118
118
|
version_requirements: !ruby/object:Gem::Requirement
|
119
119
|
requirements:
|
120
120
|
- - ">="
|
@@ -158,8 +158,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
158
158
|
- !ruby/object:Gem::Version
|
159
159
|
version: '0'
|
160
160
|
requirements: []
|
161
|
-
|
162
|
-
rubygems_version: 2.7.6
|
161
|
+
rubygems_version: 3.0.6
|
163
162
|
signing_key:
|
164
163
|
specification_version: 4
|
165
164
|
summary: Output plugin for Sensors Analytics
|