fluent-plugin-light-core 0.2.9 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +2 -0
- data/Gemfile.lock +7 -4
- data/fluent-plugin-light-core.gemspec +2 -1
- data/lib/fluent/plugin/filter_light_core.rb +64 -11
- data/sample/source.conf +112 -72
- metadata +22 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 153a8ac1fcd1f93b50ca3502cac1d2a941f283c707e50b6367f42b8c13e08cf7
|
|
4
|
+
data.tar.gz: a27cd131cdec3ecfeddb80d4d154809a6745ca652a229ec6970dad135cf57e1b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 44197cea697e733b50abad6bde8e7e645a2d3c4fee3161de2b131461dadb8bc5076581ec9905975c173bddc5b726ff45bf08dcb9442d7784e208c1c7b5af0f80
|
|
7
|
+
data.tar.gz: 961ad750fe2f00b933792e8278822e927942763a64db38515159e44b3ce4145c93d9801a5cf36fb73a9e3b6e07fdc195bbda69b2d3a895c1314860ae1d286363
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
fluent-plugin-light-core (0.
|
|
4
|
+
fluent-plugin-light-core (0.3.1)
|
|
5
5
|
fluentd (>= 1.14.2, < 2)
|
|
6
|
+
oj (>= 3.13, < 4)
|
|
6
7
|
sentry-ruby (>= 4.8.0, < 5)
|
|
7
8
|
|
|
8
9
|
GEM
|
|
9
10
|
remote: https://rubygems.org/
|
|
10
11
|
specs:
|
|
12
|
+
audit_log_parser (0.1.3)
|
|
11
13
|
concurrent-ruby (1.1.9)
|
|
12
14
|
cool.io (1.7.1)
|
|
13
15
|
faraday (1.8.0)
|
|
@@ -29,10 +31,10 @@ GEM
|
|
|
29
31
|
faraday-net_http_persistent (1.2.0)
|
|
30
32
|
faraday-patron (1.0.0)
|
|
31
33
|
faraday-rack (1.0.0)
|
|
32
|
-
fluentd (1.14.
|
|
34
|
+
fluentd (1.14.3)
|
|
33
35
|
bundler
|
|
34
36
|
cool.io (>= 1.4.5, < 2.0.0)
|
|
35
|
-
http_parser.rb (>= 0.5.1, < 0.
|
|
37
|
+
http_parser.rb (>= 0.5.1, < 0.9.0)
|
|
36
38
|
msgpack (>= 1.3.1, < 2.0.0)
|
|
37
39
|
serverengine (>= 2.2.2, < 3.0.0)
|
|
38
40
|
sigdump (~> 0.2.2)
|
|
@@ -41,7 +43,7 @@ GEM
|
|
|
41
43
|
tzinfo-data (~> 1.0)
|
|
42
44
|
webrick (>= 1.4.2, < 1.8.0)
|
|
43
45
|
yajl-ruby (~> 1.0)
|
|
44
|
-
http_parser.rb (0.
|
|
46
|
+
http_parser.rb (0.8.0)
|
|
45
47
|
msgpack (1.4.2)
|
|
46
48
|
multipart-post (2.1.1)
|
|
47
49
|
oj (3.13.9)
|
|
@@ -72,6 +74,7 @@ PLATFORMS
|
|
|
72
74
|
ruby
|
|
73
75
|
|
|
74
76
|
DEPENDENCIES
|
|
77
|
+
audit_log_parser
|
|
75
78
|
bundler (~> 1.14)
|
|
76
79
|
fluent-plugin-light-core!
|
|
77
80
|
oj (~> 3.13)
|
|
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
3
3
|
|
|
4
4
|
Gem::Specification.new do |spec|
|
|
5
5
|
spec.name = "fluent-plugin-light-core"
|
|
6
|
-
spec.version = "0.
|
|
6
|
+
spec.version = "0.3.3"
|
|
7
7
|
spec.authors = ["LIN LI"]
|
|
8
8
|
spec.email = ["l.li@alphabets.cn"]
|
|
9
9
|
|
|
@@ -26,4 +26,5 @@ Gem::Specification.new do |spec|
|
|
|
26
26
|
spec.add_runtime_dependency "fluentd", [">= 1.14.2", "< 2"]
|
|
27
27
|
spec.add_runtime_dependency "sentry-ruby", [">= 4.8.0", "< 5"]
|
|
28
28
|
spec.add_runtime_dependency "oj", [">= 3.13", "< 4"]
|
|
29
|
+
spec.add_runtime_dependency "audit_log_parser", [">= 0.1.3", "< 1"]
|
|
29
30
|
end
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
require 'fluent/plugin/filter'
|
|
17
17
|
require 'json'
|
|
18
18
|
require 'sentry-ruby'
|
|
19
|
+
require 'audit_log_parser'
|
|
19
20
|
|
|
20
21
|
module Fluent
|
|
21
22
|
module Plugin
|
|
@@ -41,6 +42,8 @@ module Fluent
|
|
|
41
42
|
# MongoDB 异常 settings
|
|
42
43
|
config_param :mongo_severity, :array, default: ['F', 'E'], value_type: :string
|
|
43
44
|
config_param :mongo_querytime, :float, default: 100
|
|
45
|
+
config_param :mongo_dataBytes, :float, default: 1073741824
|
|
46
|
+
config_param :mongo_reslen, :float, default: 20971520
|
|
44
47
|
|
|
45
48
|
# 初始化 Sentry
|
|
46
49
|
def start
|
|
@@ -69,34 +72,59 @@ module Fluent
|
|
|
69
72
|
|
|
70
73
|
# 主处理
|
|
71
74
|
def filter(tag, time, record)
|
|
72
|
-
|
|
75
|
+
|
|
76
|
+
# 应用
|
|
73
77
|
if ['app', 'service'].include? tag
|
|
74
78
|
record = filter_app(tag, time, record)
|
|
75
|
-
|
|
79
|
+
notice('app', record)
|
|
76
80
|
end
|
|
77
81
|
|
|
78
|
-
|
|
82
|
+
# 负载均衡
|
|
83
|
+
if ['lb'].include? tag
|
|
79
84
|
record = filter_lb(tag, time, record)
|
|
80
|
-
|
|
85
|
+
notice('lb', record)
|
|
81
86
|
end
|
|
82
87
|
|
|
83
|
-
|
|
88
|
+
# 数据库 - TODO: 其中mongo为旧tag删除预定
|
|
89
|
+
if ['mongo', 'master', 'secondary', 'arbiter'].include? tag
|
|
84
90
|
record = filter_mongo(tag, time, record)
|
|
85
|
-
|
|
91
|
+
notice('mongo', record)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
if ['syslog.messages', 'syslog.secure', 'syslog.audit'].include? tag
|
|
95
|
+
record = filter_syslog(tag, time, record)
|
|
86
96
|
end
|
|
87
97
|
|
|
98
|
+
record['environment'] = ENV['FLUENTD_ENV']
|
|
99
|
+
record['node'] = ENV['NODE_IP']
|
|
100
|
+
|
|
101
|
+
# 其他
|
|
88
102
|
record
|
|
89
103
|
|
|
90
104
|
end
|
|
91
105
|
|
|
106
|
+
# Parse syslog
|
|
107
|
+
def filter_syslog(tag, time, record)
|
|
108
|
+
|
|
109
|
+
if (tag == 'syslog.audit')
|
|
110
|
+
line = record['message']
|
|
111
|
+
return record unless line
|
|
112
|
+
|
|
113
|
+
record = AuditLogParser.parse_line(line, flatten: false)
|
|
114
|
+
record['time'] = Time.at(record["header"]["msg"][/[0-9]+/].to_i).to_s
|
|
115
|
+
return record
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
record['time'] = Time.at(time).to_s
|
|
119
|
+
return record
|
|
120
|
+
end
|
|
121
|
+
|
|
92
122
|
# Parse the application log
|
|
93
123
|
def filter_app(tag, time, record)
|
|
94
124
|
file = record['file'].split('/').last.split('_') # Parse log file name
|
|
95
125
|
log = record['log'] # Get detailed log content
|
|
96
126
|
|
|
97
127
|
# Set common items
|
|
98
|
-
# record['environment'] = Socket.gethostname.split('-')[0] # dev | prd
|
|
99
|
-
record['environment'] = ENV['FLUENTD_ENV'] # dev | prd
|
|
100
128
|
record['cid'] = file[0] # container id
|
|
101
129
|
record['cname'] = file[0].split('-')[1] # container name
|
|
102
130
|
record['ctime'] = record['time'] # container time
|
|
@@ -150,7 +178,6 @@ module Fluent
|
|
|
150
178
|
file = record['file'].split('/').last.split('_')
|
|
151
179
|
log = record['log']
|
|
152
180
|
|
|
153
|
-
record['environment'] = ENV['FLUENTD_ENV']
|
|
154
181
|
record['cid'] = file[0]
|
|
155
182
|
record['cname'] = tag
|
|
156
183
|
record['ctime'] = record['time']
|
|
@@ -216,7 +243,6 @@ module Fluent
|
|
|
216
243
|
file = record['file'].split('/').last.split('_')
|
|
217
244
|
log = record['log']
|
|
218
245
|
|
|
219
|
-
record['environment'] = ENV['FLUENTD_ENV']
|
|
220
246
|
record['cid'] = file[0]
|
|
221
247
|
record['cname'] = tag
|
|
222
248
|
record['ctime'] = record['time']
|
|
@@ -233,12 +259,21 @@ module Fluent
|
|
|
233
259
|
record['context'] = item['ctx']
|
|
234
260
|
record['identifier'] = item['id']
|
|
235
261
|
record['message'] = item['msg']
|
|
236
|
-
|
|
262
|
+
|
|
237
263
|
attributes = item['attr']
|
|
238
264
|
if attributes
|
|
239
265
|
record['querytime'] = attributes['durationMillis']
|
|
240
266
|
record['collection'] = attributes['ns']
|
|
241
267
|
record['command'] = attributes['command']
|
|
268
|
+
record['reslen'] = attributes['reslen']
|
|
269
|
+
storage = attributes['storage']
|
|
270
|
+
if storage
|
|
271
|
+
data = storage['data']
|
|
272
|
+
if data
|
|
273
|
+
record['bytesRead'] = data['bytesRead']
|
|
274
|
+
record['bytesWritten'] = data['bytesWritten']
|
|
275
|
+
end
|
|
276
|
+
end
|
|
242
277
|
record['attr'] = attributes
|
|
243
278
|
end
|
|
244
279
|
|
|
@@ -314,6 +349,24 @@ module Fluent
|
|
|
314
349
|
send(tag, message, record) if record['querytime'].to_f >= @mongo_querytime
|
|
315
350
|
end
|
|
316
351
|
|
|
352
|
+
if @mongo_dataBytes > 0 && record['bytesRead']
|
|
353
|
+
message = 'bytesRead data is:' + record['bytesRead'].to_s
|
|
354
|
+
message = message + ' : ' + record['collection'] if record['collection']
|
|
355
|
+
send(tag, message, record) if record['bytesRead'].to_f >= @mongo_dataBytes
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
if @mongo_dataBytes > 0 && record['bytesWritten']
|
|
359
|
+
message = 'bytesWritten data is:' + record['bytesWritten'].to_s
|
|
360
|
+
message = message + ' : ' + record['collection'] if record['collection']
|
|
361
|
+
send(tag, message, record) if record['bytesWritten'].to_f >= @mongo_dataBytes
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
if @mongo_reslen > 0 && record['reslen']
|
|
365
|
+
message = 'reslen data is:' + record['reslen'].to_s
|
|
366
|
+
message = message + ' : ' + record['collection'] if record['collection']
|
|
367
|
+
send(tag, message, record) if record['reslen'].to_f >= @mongo_reslen
|
|
368
|
+
end
|
|
369
|
+
|
|
317
370
|
return record
|
|
318
371
|
end
|
|
319
372
|
|
data/sample/source.conf
CHANGED
|
@@ -1,81 +1,119 @@
|
|
|
1
1
|
|
|
2
|
-
<source>
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
</source>
|
|
2
|
+
# <source>
|
|
3
|
+
# @type tail
|
|
4
|
+
# path sample/app*.log
|
|
5
|
+
# pos_file sample/source.app.pos
|
|
6
|
+
# tag app
|
|
7
|
+
# format json
|
|
8
|
+
# read_from_head true
|
|
9
|
+
# path_key file
|
|
10
|
+
# time_key time
|
|
11
|
+
# keep_time_key true
|
|
12
|
+
# time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
13
|
+
# </source>
|
|
14
14
|
|
|
15
|
-
<source>
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
</source>
|
|
15
|
+
# <source>
|
|
16
|
+
# @type tail
|
|
17
|
+
# path sample/ingress-nginx*.log
|
|
18
|
+
# pos_file sample/source.ingress-nginx.pos
|
|
19
|
+
# tag lb
|
|
20
|
+
# format json
|
|
21
|
+
# read_from_head true
|
|
22
|
+
# path_key file
|
|
23
|
+
# time_key time
|
|
24
|
+
# keep_time_key true
|
|
25
|
+
# time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
26
|
+
# </source>
|
|
27
27
|
|
|
28
|
-
<source>
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
</source>
|
|
28
|
+
# <source>
|
|
29
|
+
# @type tail
|
|
30
|
+
# path sample/hub*.log
|
|
31
|
+
# pos_file sample/source.hub.pos
|
|
32
|
+
# tag hub
|
|
33
|
+
# format json
|
|
34
|
+
# read_from_head true
|
|
35
|
+
# path_key file
|
|
36
|
+
# time_key time
|
|
37
|
+
# keep_time_key true
|
|
38
|
+
# time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
39
|
+
# </source>
|
|
40
40
|
|
|
41
|
-
<source>
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
</source>
|
|
41
|
+
# <source>
|
|
42
|
+
# @type tail
|
|
43
|
+
# path sample/db*.log
|
|
44
|
+
# pos_file sample/source.mongo.pos
|
|
45
|
+
# tag mongo
|
|
46
|
+
# format json
|
|
47
|
+
# read_from_head true
|
|
48
|
+
# path_key file
|
|
49
|
+
# time_key time
|
|
50
|
+
# keep_time_key true
|
|
51
|
+
# time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
52
|
+
# </source>
|
|
53
53
|
|
|
54
|
-
<source>
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
</source>
|
|
54
|
+
# <source>
|
|
55
|
+
# @type tail
|
|
56
|
+
# path sample/secondary*.log
|
|
57
|
+
# pos_file sample/source.secondary.pos
|
|
58
|
+
# tag secondary
|
|
59
|
+
# format json
|
|
60
|
+
# read_from_head true
|
|
61
|
+
# path_key file
|
|
62
|
+
# time_key time
|
|
63
|
+
# keep_time_key true
|
|
64
|
+
# time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
65
|
+
# </source>
|
|
66
66
|
|
|
67
|
-
<source>
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
</source>
|
|
67
|
+
# <source>
|
|
68
|
+
# @type tail
|
|
69
|
+
# path sample/arbiter*.log
|
|
70
|
+
# pos_file sample/source.arbiter.pos
|
|
71
|
+
# tag arbiter
|
|
72
|
+
# format json
|
|
73
|
+
# read_from_head true
|
|
74
|
+
# path_key file
|
|
75
|
+
# time_key time
|
|
76
|
+
# keep_time_key true
|
|
77
|
+
# time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
78
|
+
# </source>
|
|
79
|
+
|
|
80
|
+
#######################################
|
|
81
|
+
# syslog messages
|
|
82
|
+
#######################################
|
|
83
|
+
# <source>
|
|
84
|
+
# @type tail
|
|
85
|
+
# format syslog
|
|
86
|
+
# path sample/sys/messages.log
|
|
87
|
+
# pos_file sample/sys/messages.pos
|
|
88
|
+
# read_from_head true
|
|
89
|
+
# tag syslog.messages
|
|
90
|
+
# </source>
|
|
91
|
+
|
|
92
|
+
#######################################
|
|
93
|
+
# syslog secure
|
|
94
|
+
#######################################
|
|
95
|
+
# <source>
|
|
96
|
+
# @type tail
|
|
97
|
+
# format syslog
|
|
98
|
+
# path sample/sys/secure.log
|
|
99
|
+
# pos_file sample/sys/secure.pos
|
|
100
|
+
# read_from_head true
|
|
101
|
+
# tag syslog.secure
|
|
102
|
+
# </source>
|
|
103
|
+
|
|
104
|
+
#######################################
|
|
105
|
+
# syslog audit
|
|
106
|
+
#######################################
|
|
107
|
+
# <source>
|
|
108
|
+
# @type tail
|
|
109
|
+
# path sample/sys/audit.log
|
|
110
|
+
# pos_file sample/sys/audit.log.pos
|
|
111
|
+
# read_from_head true
|
|
112
|
+
# tag syslog.audit
|
|
113
|
+
# <parse>
|
|
114
|
+
# @type none
|
|
115
|
+
# </parse>
|
|
116
|
+
# </source>
|
|
79
117
|
|
|
80
118
|
<filter **>
|
|
81
119
|
@type light_core
|
|
@@ -83,6 +121,8 @@
|
|
|
83
121
|
sentry_dsn https://4447403a2a86437491d307ceb72898e5@sentry.zf.link/4
|
|
84
122
|
mongo_querytime 300
|
|
85
123
|
mongo_severity F,E
|
|
124
|
+
mongo_dataBytes 1073741824
|
|
125
|
+
mongo_reslen 20971520
|
|
86
126
|
lb_stream stderr
|
|
87
127
|
lb_code 400,500
|
|
88
128
|
lb_elapsed 5
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-light-core
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- LIN LI
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-
|
|
11
|
+
date: 2021-12-18 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -112,6 +112,26 @@ dependencies:
|
|
|
112
112
|
- - "<"
|
|
113
113
|
- !ruby/object:Gem::Version
|
|
114
114
|
version: '4'
|
|
115
|
+
- !ruby/object:Gem::Dependency
|
|
116
|
+
name: audit_log_parser
|
|
117
|
+
requirement: !ruby/object:Gem::Requirement
|
|
118
|
+
requirements:
|
|
119
|
+
- - ">="
|
|
120
|
+
- !ruby/object:Gem::Version
|
|
121
|
+
version: 0.1.3
|
|
122
|
+
- - "<"
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: '1'
|
|
125
|
+
type: :runtime
|
|
126
|
+
prerelease: false
|
|
127
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
128
|
+
requirements:
|
|
129
|
+
- - ">="
|
|
130
|
+
- !ruby/object:Gem::Version
|
|
131
|
+
version: 0.1.3
|
|
132
|
+
- - "<"
|
|
133
|
+
- !ruby/object:Gem::Version
|
|
134
|
+
version: '1'
|
|
115
135
|
description: light core fluent plugin. support mongodb, nginx and application
|
|
116
136
|
email:
|
|
117
137
|
- l.li@alphabets.cn
|