fluent-plugin-light-core 0.1.8 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +2 -0
- data/fluent-plugin-light-core.gemspec +2 -1
- data/lib/fluent/plugin/filter_light_core.rb +72 -37
- data/sample/README.md +10 -0
- data/sample/source.conf +10 -0
- metadata +22 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: de6963e9c755b56d0c56eb2efe8d5144b66ea1463dd826c90ab71f5b65a72f37
|
4
|
+
data.tar.gz: f37743b84fc9a68940c488e06903a71725ffbb14a0a7a02c2203a19074f22940
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2d3fb8390aaf781220b4e288819c1f14c063a50c58ea850ef8b55ae63005ed3db03653b7e81f6cf349cfac04ca5259c2a4e2c73ec83003ed8290fefe9e187c1b
|
7
|
+
data.tar.gz: b89a10c9a2581a781536e06f4b7285c066ef1ec73f4a033bd646c7c2759e9de00a2a5227a5afa98ec406737390fd37c2212086a4cf33ccd7cab3addac4ec4787
|
data/Gemfile
CHANGED
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |spec|
|
5
5
|
spec.name = "fluent-plugin-light-core"
|
6
|
-
spec.version = "0.
|
6
|
+
spec.version = "0.2.2"
|
7
7
|
spec.authors = ["LIN LI"]
|
8
8
|
spec.email = ["l.li@alphabets.cn"]
|
9
9
|
|
@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
|
|
24
24
|
spec.add_development_dependency "rake", "~> 12.0"
|
25
25
|
spec.add_development_dependency "test-unit", "~> 3.0"
|
26
26
|
spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
|
27
|
+
spec.add_runtime_dependency "sentry-ruby", [">= 4.1.5", "< 5"]
|
27
28
|
end
|
@@ -14,8 +14,8 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
|
16
16
|
require 'fluent/plugin/filter'
|
17
|
-
require 'socket'
|
18
17
|
require 'json'
|
18
|
+
require 'sentry-ruby'
|
19
19
|
|
20
20
|
module Fluent
|
21
21
|
module Plugin
|
@@ -23,9 +23,8 @@ module Fluent
|
|
23
23
|
Fluent::Plugin.register_filter('light_core', self)
|
24
24
|
|
25
25
|
# 通知相关配置
|
26
|
-
config_param :
|
27
|
-
config_param :
|
28
|
-
config_param :host, :string, default: '255.255.255.255'
|
26
|
+
config_param :sentry, :bool, default: false
|
27
|
+
config_param :sentry_dsn, :string, default: ''
|
29
28
|
|
30
29
|
# Application 异常 settings
|
31
30
|
config_param :app_stream, :string, default: 'stderr'
|
@@ -35,32 +34,36 @@ module Fluent
|
|
35
34
|
|
36
35
|
# Nginx 异常 settings
|
37
36
|
config_param :lb_stream, :string, default: 'stderr'
|
38
|
-
config_param :lb_code, :array, default: ['500'], value_type: :string
|
39
|
-
config_param :lb_elapsed, :float, default:
|
37
|
+
config_param :lb_code, :array, default: ['400', '500'], value_type: :string
|
38
|
+
config_param :lb_elapsed, :float, default: 3
|
39
|
+
config_param :lb_ignore, :array, default: [], value_type: :string
|
40
40
|
|
41
41
|
# MongoDB 异常 settings
|
42
42
|
config_param :mongo_severity, :array, default: ['F', 'E'], value_type: :string
|
43
43
|
config_param :mongo_querytime, :float, default: 100
|
44
44
|
|
45
|
-
# 初始化
|
45
|
+
# 初始化 Sentry
|
46
46
|
def start
|
47
47
|
super
|
48
48
|
|
49
|
-
if @
|
50
|
-
log.info('init
|
51
|
-
|
52
|
-
|
49
|
+
if @sentry
|
50
|
+
log.info('init sentry')
|
51
|
+
Sentry.init do |config|
|
52
|
+
config.dsn = @sentry_dsn
|
53
|
+
|
54
|
+
# To activate performance monitoring, set one of these options.
|
55
|
+
# We recommend adjusting the value in production:
|
56
|
+
config.traces_sample_rate = 1
|
57
|
+
|
58
|
+
# config.background_worker_threads = 2
|
59
|
+
config.transport.timeout = 10
|
60
|
+
config.transport.open_timeout = 10
|
61
|
+
end
|
53
62
|
end
|
54
63
|
end
|
55
64
|
|
56
|
-
# 清理
|
65
|
+
# 清理
|
57
66
|
def shutdown
|
58
|
-
|
59
|
-
if @notice
|
60
|
-
log.info('close udp connection')
|
61
|
-
@udp.close if @udp
|
62
|
-
end
|
63
|
-
|
64
67
|
super
|
65
68
|
end
|
66
69
|
|
@@ -222,6 +225,7 @@ module Fluent
|
|
222
225
|
record.delete('file')
|
223
226
|
record.delete('time')
|
224
227
|
|
228
|
+
# 旧版本日志格式解析
|
225
229
|
if /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+\d{4} [A-Z]/.match(log)
|
226
230
|
|
227
231
|
item = /^(?<time>[^ ]*) (?<severity>[A-Z])\s* (?<component>(-|([^ ]*)))\s* \[(?<context>[^\]]*)\]\s* ((?<query>.*) protocol:op_query (?<querytime>[\d\.]+(?=ms))|(?<message>.*))/.match(log)
|
@@ -249,6 +253,30 @@ module Fluent
|
|
249
253
|
end
|
250
254
|
end
|
251
255
|
|
256
|
+
return record
|
257
|
+
else
|
258
|
+
|
259
|
+
# 版本4.4开始,默认日志为json格式
|
260
|
+
item = JSON.parse(log)
|
261
|
+
record['time'] = item['t']['$date']
|
262
|
+
record['severity'] = item['s']
|
263
|
+
record['component'] = item['c']
|
264
|
+
record['context'] = item['ctx']
|
265
|
+
record['identifier'] = item['id']
|
266
|
+
record['message'] = item['msg']
|
267
|
+
|
268
|
+
attributes = item['attr']
|
269
|
+
if attributes
|
270
|
+
record['querytime'] = attributes['durationMillis']
|
271
|
+
record['collection'] = attributes['ns']
|
272
|
+
record['command'] = attributes['command']
|
273
|
+
record['attr'] = attributes
|
274
|
+
end
|
275
|
+
|
276
|
+
record['tags'] = item['tags']
|
277
|
+
record['truncated'] = item['truncated']
|
278
|
+
record['size'] = item['size']
|
279
|
+
|
252
280
|
return record
|
253
281
|
end
|
254
282
|
|
@@ -258,15 +286,18 @@ module Fluent
|
|
258
286
|
# 确认是否发送通知
|
259
287
|
def notice(tag, record)
|
260
288
|
|
261
|
-
unless @
|
289
|
+
unless @sentry
|
262
290
|
return record
|
263
291
|
end
|
264
292
|
|
265
293
|
if tag == 'app'
|
294
|
+
|
295
|
+
# 未使用
|
266
296
|
if @app_stream && record['stream']
|
267
297
|
send(record) if record['stream'] == @app_stream
|
268
298
|
end
|
269
299
|
|
300
|
+
# 未使用
|
270
301
|
if @app_message.length > 0 && record['message']
|
271
302
|
@app_message.each do |pattern|
|
272
303
|
if pattern.match(record['message'])
|
@@ -277,27 +308,36 @@ module Fluent
|
|
277
308
|
end
|
278
309
|
|
279
310
|
if @app_status.length > 0 && record['status']
|
280
|
-
|
311
|
+
message = 'Status code abnormal : ' + record['url']
|
312
|
+
send(tag, message, record) if @app_status.include?(record['status'])
|
281
313
|
end
|
282
314
|
|
283
315
|
if @app_elapsed > 0 && record['elapsed']
|
284
|
-
|
316
|
+
message = 'Slow process : ' + record['url']
|
317
|
+
send(tag, message, record) if record['elapsed'].to_f >= @app_elapsed
|
285
318
|
end
|
286
319
|
|
287
320
|
return record
|
288
321
|
end
|
289
322
|
|
290
323
|
if tag == 'lb'
|
324
|
+
|
325
|
+
if @lb_ignore && record['path'] && @lb_ignore.include?(record['path'])
|
326
|
+
return record
|
327
|
+
end
|
328
|
+
|
291
329
|
if @lb_stream && record['stream']
|
292
|
-
send(record) if record['stream'] == @lb_stream
|
330
|
+
send(tag, 'Stderror', record) if record['stream'] == @lb_stream
|
293
331
|
end
|
294
332
|
|
295
333
|
if @lb_code.length > 0 && record['code']
|
296
|
-
|
334
|
+
message = 'Status code abnormal : ' + record['path']
|
335
|
+
send(tag, message, record) if @lb_code.include?(record['code'])
|
297
336
|
end
|
298
337
|
|
299
338
|
if @lb_elapsed > 0 && record['elapsed']
|
300
|
-
|
339
|
+
message = 'Slow request : ' + record['path']
|
340
|
+
send(tag, message, record) if record['elapsed'].to_f >= @lb_elapsed
|
301
341
|
end
|
302
342
|
|
303
343
|
return record
|
@@ -305,11 +345,14 @@ module Fluent
|
|
305
345
|
|
306
346
|
if tag == 'mongo'
|
307
347
|
if @mongo_severity.length > 0 && record['severity']
|
308
|
-
|
348
|
+
message = 'Severity level abnormal : ' + record['severity']
|
349
|
+
send(tag, message, record) if @mongo_severity.include?(record['severity'])
|
309
350
|
end
|
310
351
|
|
311
352
|
if @mongo_querytime > 0 && record['querytime']
|
312
|
-
|
353
|
+
message = 'Slow query'
|
354
|
+
message = message + ' : ' + record['collection'] if record['collection']
|
355
|
+
send(tag, message, record) if record['querytime'].to_f >= @mongo_querytime
|
313
356
|
end
|
314
357
|
|
315
358
|
return record
|
@@ -320,17 +363,9 @@ module Fluent
|
|
320
363
|
end
|
321
364
|
|
322
365
|
# 发送UDP请求
|
323
|
-
def send(record)
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
data = {
|
328
|
-
:query => '/api/log/notice',
|
329
|
-
:params => {
|
330
|
-
:data => record
|
331
|
-
}
|
332
|
-
}
|
333
|
-
@udp.send(data.to_json, 0, Socket.pack_sockaddr_in(@port, @host))
|
366
|
+
def send(tag, message, record)
|
367
|
+
log.debug('send udp notice')
|
368
|
+
Sentry.capture_message(message, :extra => record, :tags => {'log' => tag})
|
334
369
|
end
|
335
370
|
|
336
371
|
# 转数字
|
data/sample/README.md
CHANGED
@@ -32,3 +32,13 @@ Initialized empty Git repository in /Users/lilin/developer/light/fluent-plugin-l
|
|
32
32
|
|
33
33
|
% rm -f sample/*.pos
|
34
34
|
% fluentd -c sample/source.conf -p lib/fluent/plugin
|
35
|
+
|
36
|
+
## 添加 sentry 依赖
|
37
|
+
|
38
|
+
- 编辑Gemfile
|
39
|
+
|
40
|
+
gem "sentry-ruby"
|
41
|
+
|
42
|
+
- 安装依赖
|
43
|
+
|
44
|
+
gem install sentry-ruby
|
data/sample/source.conf
CHANGED
@@ -79,6 +79,16 @@
|
|
79
79
|
|
80
80
|
<filter **>
|
81
81
|
@type light_core
|
82
|
+
sentry true
|
83
|
+
sentry_dsn https://4447403a2a86437491d307ceb72898e5@sentry.zf.link/4
|
84
|
+
mongo_querytime 300
|
85
|
+
mongo_severity F,E
|
86
|
+
lb_stream stderr
|
87
|
+
lb_code 400,500
|
88
|
+
lb_elapsed 5
|
89
|
+
lb_ignore /api/environment/list,/api/certificate/list?condition%5Bkind%5D=SSL
|
90
|
+
app_status 500
|
91
|
+
app_elapsed 5
|
82
92
|
</filter>
|
83
93
|
|
84
94
|
# <match app>
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-light-core
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- LIN LI
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-11-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -72,6 +72,26 @@ dependencies:
|
|
72
72
|
- - "<"
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: '2'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: sentry-ruby
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: 4.1.5
|
82
|
+
- - "<"
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: '5'
|
85
|
+
type: :runtime
|
86
|
+
prerelease: false
|
87
|
+
version_requirements: !ruby/object:Gem::Requirement
|
88
|
+
requirements:
|
89
|
+
- - ">="
|
90
|
+
- !ruby/object:Gem::Version
|
91
|
+
version: 4.1.5
|
92
|
+
- - "<"
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '5'
|
75
95
|
description: light core fluent plugin. support mongodb, nginx and application
|
76
96
|
email:
|
77
97
|
- l.li@alphabets.cn
|