fluent-plugin-light-core 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +2 -0
- data/Gemfile.lock +7 -4
- data/fluent-plugin-light-core.gemspec +1 -1
- data/lib/fluent/plugin/filter_light_core.rb +33 -9
- data/sample/source.conf +110 -72
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: '08ec4ec35703b1b0621b198aadfe16db38c875a0256b36ea1f49c787c309412b'
|
|
4
|
+
data.tar.gz: 4f61fd3d5470795e626e97d3edac0dc8a64de5558338e83a52d9f0d98e0c2d4c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cd25d33ca42164a1df21a91a424af363f844b31188e2eda280a8945fd6ee2c31913e3333c5e94eeefae3d20ad9b7184f044e7955a9cdf54f92aff8c7b9ee81c8
|
|
7
|
+
data.tar.gz: 651ba35b07783375102c4fdb2026231ccc67661a8bb8a215116dbef5fb7da00094f35db45ddbc3b9f8b2df3b4fad80e59da7db7a4dcf053dcaa457d237389c1d
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
fluent-plugin-light-core (0.
|
|
4
|
+
fluent-plugin-light-core (0.3.1)
|
|
5
5
|
fluentd (>= 1.14.2, < 2)
|
|
6
|
+
oj (>= 3.13, < 4)
|
|
6
7
|
sentry-ruby (>= 4.8.0, < 5)
|
|
7
8
|
|
|
8
9
|
GEM
|
|
9
10
|
remote: https://rubygems.org/
|
|
10
11
|
specs:
|
|
12
|
+
audit_log_parser (0.1.3)
|
|
11
13
|
concurrent-ruby (1.1.9)
|
|
12
14
|
cool.io (1.7.1)
|
|
13
15
|
faraday (1.8.0)
|
|
@@ -29,10 +31,10 @@ GEM
|
|
|
29
31
|
faraday-net_http_persistent (1.2.0)
|
|
30
32
|
faraday-patron (1.0.0)
|
|
31
33
|
faraday-rack (1.0.0)
|
|
32
|
-
fluentd (1.14.
|
|
34
|
+
fluentd (1.14.3)
|
|
33
35
|
bundler
|
|
34
36
|
cool.io (>= 1.4.5, < 2.0.0)
|
|
35
|
-
http_parser.rb (>= 0.5.1, < 0.
|
|
37
|
+
http_parser.rb (>= 0.5.1, < 0.9.0)
|
|
36
38
|
msgpack (>= 1.3.1, < 2.0.0)
|
|
37
39
|
serverengine (>= 2.2.2, < 3.0.0)
|
|
38
40
|
sigdump (~> 0.2.2)
|
|
@@ -41,7 +43,7 @@ GEM
|
|
|
41
43
|
tzinfo-data (~> 1.0)
|
|
42
44
|
webrick (>= 1.4.2, < 1.8.0)
|
|
43
45
|
yajl-ruby (~> 1.0)
|
|
44
|
-
http_parser.rb (0.
|
|
46
|
+
http_parser.rb (0.8.0)
|
|
45
47
|
msgpack (1.4.2)
|
|
46
48
|
multipart-post (2.1.1)
|
|
47
49
|
oj (3.13.9)
|
|
@@ -72,6 +74,7 @@ PLATFORMS
|
|
|
72
74
|
ruby
|
|
73
75
|
|
|
74
76
|
DEPENDENCIES
|
|
77
|
+
audit_log_parser
|
|
75
78
|
bundler (~> 1.14)
|
|
76
79
|
fluent-plugin-light-core!
|
|
77
80
|
oj (~> 3.13)
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
require 'fluent/plugin/filter'
|
|
17
17
|
require 'json'
|
|
18
18
|
require 'sentry-ruby'
|
|
19
|
+
require 'audit_log_parser'
|
|
19
20
|
|
|
20
21
|
module Fluent
|
|
21
22
|
module Plugin
|
|
@@ -72,33 +73,58 @@ module Fluent
|
|
|
72
73
|
# 主处理
|
|
73
74
|
def filter(tag, time, record)
|
|
74
75
|
|
|
76
|
+
# 应用
|
|
75
77
|
if ['app', 'service'].include? tag
|
|
76
78
|
record = filter_app(tag, time, record)
|
|
77
|
-
|
|
79
|
+
notice('app', record)
|
|
78
80
|
end
|
|
79
81
|
|
|
80
|
-
|
|
82
|
+
# 负载均衡
|
|
83
|
+
if ['lb'].include? tag
|
|
81
84
|
record = filter_lb(tag, time, record)
|
|
82
|
-
|
|
85
|
+
notice('lb', record)
|
|
83
86
|
end
|
|
84
87
|
|
|
85
|
-
|
|
88
|
+
# 数据库 - TODO: 其中mongo为旧tag删除预定
|
|
89
|
+
if ['mongo', 'master', 'secondary', 'arbiter'].include? tag
|
|
86
90
|
record = filter_mongo(tag, time, record)
|
|
87
|
-
|
|
91
|
+
notice('mongo', record)
|
|
88
92
|
end
|
|
89
93
|
|
|
94
|
+
if ['syslog.messages', 'syslog.secure', 'syslog.audit'].include? tag
|
|
95
|
+
record = filter_syslog(tag, time, record)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
record['environment'] = ENV['FLUENTD_ENV']
|
|
99
|
+
record['node'] = ENV['NODE_IP']
|
|
100
|
+
|
|
101
|
+
# 其他
|
|
90
102
|
record
|
|
91
103
|
|
|
92
104
|
end
|
|
93
105
|
|
|
106
|
+
# Parse syslog
|
|
107
|
+
def filter_syslog(tag, time, record)
|
|
108
|
+
|
|
109
|
+
if (tag == 'syslog.audit')
|
|
110
|
+
line = record['message']
|
|
111
|
+
return record unless line
|
|
112
|
+
|
|
113
|
+
record = AuditLogParser.parse_line(line, flatten: false)
|
|
114
|
+
record['time'] = Time.at(record["header"]["msg"][/[0-9]+/].to_i).to_s
|
|
115
|
+
return record
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
record['time'] = Time.at(time).to_s
|
|
119
|
+
return record
|
|
120
|
+
end
|
|
121
|
+
|
|
94
122
|
# Parse the application log
|
|
95
123
|
def filter_app(tag, time, record)
|
|
96
124
|
file = record['file'].split('/').last.split('_') # Parse log file name
|
|
97
125
|
log = record['log'] # Get detailed log content
|
|
98
126
|
|
|
99
127
|
# Set common items
|
|
100
|
-
# record['environment'] = Socket.gethostname.split('-')[0] # dev | prd
|
|
101
|
-
record['environment'] = ENV['FLUENTD_ENV'] # dev | prd
|
|
102
128
|
record['cid'] = file[0] # container id
|
|
103
129
|
record['cname'] = file[0].split('-')[1] # container name
|
|
104
130
|
record['ctime'] = record['time'] # container time
|
|
@@ -152,7 +178,6 @@ module Fluent
|
|
|
152
178
|
file = record['file'].split('/').last.split('_')
|
|
153
179
|
log = record['log']
|
|
154
180
|
|
|
155
|
-
record['environment'] = ENV['FLUENTD_ENV']
|
|
156
181
|
record['cid'] = file[0]
|
|
157
182
|
record['cname'] = tag
|
|
158
183
|
record['ctime'] = record['time']
|
|
@@ -218,7 +243,6 @@ module Fluent
|
|
|
218
243
|
file = record['file'].split('/').last.split('_')
|
|
219
244
|
log = record['log']
|
|
220
245
|
|
|
221
|
-
record['environment'] = ENV['FLUENTD_ENV']
|
|
222
246
|
record['cid'] = file[0]
|
|
223
247
|
record['cname'] = tag
|
|
224
248
|
record['ctime'] = record['time']
|
data/sample/source.conf
CHANGED
|
@@ -1,81 +1,119 @@
|
|
|
1
1
|
|
|
2
|
-
<source>
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
</source>
|
|
2
|
+
# <source>
|
|
3
|
+
# @type tail
|
|
4
|
+
# path sample/app*.log
|
|
5
|
+
# pos_file sample/source.app.pos
|
|
6
|
+
# tag app
|
|
7
|
+
# format json
|
|
8
|
+
# read_from_head true
|
|
9
|
+
# path_key file
|
|
10
|
+
# time_key time
|
|
11
|
+
# keep_time_key true
|
|
12
|
+
# time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
13
|
+
# </source>
|
|
14
14
|
|
|
15
|
-
<source>
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
</source>
|
|
15
|
+
# <source>
|
|
16
|
+
# @type tail
|
|
17
|
+
# path sample/ingress-nginx*.log
|
|
18
|
+
# pos_file sample/source.ingress-nginx.pos
|
|
19
|
+
# tag lb
|
|
20
|
+
# format json
|
|
21
|
+
# read_from_head true
|
|
22
|
+
# path_key file
|
|
23
|
+
# time_key time
|
|
24
|
+
# keep_time_key true
|
|
25
|
+
# time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
26
|
+
# </source>
|
|
27
27
|
|
|
28
|
-
<source>
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
</source>
|
|
28
|
+
# <source>
|
|
29
|
+
# @type tail
|
|
30
|
+
# path sample/hub*.log
|
|
31
|
+
# pos_file sample/source.hub.pos
|
|
32
|
+
# tag hub
|
|
33
|
+
# format json
|
|
34
|
+
# read_from_head true
|
|
35
|
+
# path_key file
|
|
36
|
+
# time_key time
|
|
37
|
+
# keep_time_key true
|
|
38
|
+
# time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
39
|
+
# </source>
|
|
40
40
|
|
|
41
|
-
<source>
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
</source>
|
|
41
|
+
# <source>
|
|
42
|
+
# @type tail
|
|
43
|
+
# path sample/db*.log
|
|
44
|
+
# pos_file sample/source.mongo.pos
|
|
45
|
+
# tag mongo
|
|
46
|
+
# format json
|
|
47
|
+
# read_from_head true
|
|
48
|
+
# path_key file
|
|
49
|
+
# time_key time
|
|
50
|
+
# keep_time_key true
|
|
51
|
+
# time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
52
|
+
# </source>
|
|
53
53
|
|
|
54
|
-
<source>
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
</source>
|
|
54
|
+
# <source>
|
|
55
|
+
# @type tail
|
|
56
|
+
# path sample/secondary*.log
|
|
57
|
+
# pos_file sample/source.secondary.pos
|
|
58
|
+
# tag secondary
|
|
59
|
+
# format json
|
|
60
|
+
# read_from_head true
|
|
61
|
+
# path_key file
|
|
62
|
+
# time_key time
|
|
63
|
+
# keep_time_key true
|
|
64
|
+
# time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
65
|
+
# </source>
|
|
66
66
|
|
|
67
|
-
<source>
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
</source>
|
|
67
|
+
# <source>
|
|
68
|
+
# @type tail
|
|
69
|
+
# path sample/arbiter*.log
|
|
70
|
+
# pos_file sample/source.arbiter.pos
|
|
71
|
+
# tag arbiter
|
|
72
|
+
# format json
|
|
73
|
+
# read_from_head true
|
|
74
|
+
# path_key file
|
|
75
|
+
# time_key time
|
|
76
|
+
# keep_time_key true
|
|
77
|
+
# time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
78
|
+
# </source>
|
|
79
|
+
|
|
80
|
+
#######################################
|
|
81
|
+
# syslog messages
|
|
82
|
+
#######################################
|
|
83
|
+
# <source>
|
|
84
|
+
# @type tail
|
|
85
|
+
# format syslog
|
|
86
|
+
# path sample/sys/messages.log
|
|
87
|
+
# pos_file sample/sys/messages.pos
|
|
88
|
+
# read_from_head true
|
|
89
|
+
# tag syslog.messages
|
|
90
|
+
# </source>
|
|
91
|
+
|
|
92
|
+
#######################################
|
|
93
|
+
# syslog secure
|
|
94
|
+
#######################################
|
|
95
|
+
# <source>
|
|
96
|
+
# @type tail
|
|
97
|
+
# format syslog
|
|
98
|
+
# path sample/sys/secure.log
|
|
99
|
+
# pos_file sample/sys/secure.pos
|
|
100
|
+
# read_from_head true
|
|
101
|
+
# tag syslog.secure
|
|
102
|
+
# </source>
|
|
103
|
+
|
|
104
|
+
#######################################
|
|
105
|
+
# syslog audit
|
|
106
|
+
#######################################
|
|
107
|
+
# <source>
|
|
108
|
+
# @type tail
|
|
109
|
+
# path sample/sys/audit.log
|
|
110
|
+
# pos_file sample/sys/audit.log.pos
|
|
111
|
+
# read_from_head true
|
|
112
|
+
# tag syslog.audit
|
|
113
|
+
# <parse>
|
|
114
|
+
# @type none
|
|
115
|
+
# </parse>
|
|
116
|
+
# </source>
|
|
79
117
|
|
|
80
118
|
<filter **>
|
|
81
119
|
@type light_core
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-light-core
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- LIN LI
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-12-
|
|
11
|
+
date: 2021-12-18 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|