fluent-plugin-path2tag 0.0.3 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +4 -5
- data/lib/fluent/plugin/out_path2tag.rb +50 -13
- data/lib/fluent/plugin/path2tag/version.rb +1 -1
- data/test/plugin/test_path2tag.rb +66 -15
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ae392e5bb11da4f51e7b6009341ad9212a6e1dda
|
4
|
+
data.tar.gz: b5d9ffd349a0f27c42d3cc6d7718d219d0c288de
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9e19a37f51ab1d925e6bb09d2149e98c60abe750e56ab52638d2a9b5d4de2cd0cfa5f6e3d5031f98859bbbe22e001b8028d0e6a5bfbeccf5ad5c15bc28627fe8
|
7
|
+
data.tar.gz: 967be47a6ab16b03746463b28820d6d8582ed7581acf70161c071555cfe9e157477ed05ea6d60a8a96828754aa1252b6d81a08b3e525480382fd1083bb2c89fe
|
data/README.md
CHANGED
@@ -1,18 +1,19 @@
|
|
1
1
|
# fluent-plugin-path2tag
|
2
2
|
|
3
3
|
## Overview
|
4
|
+
A fluent output filter plugin for Nginx's logging.
|
5
|
+
Convert uri to tag. And reformat json data.
|
4
6
|
|
5
7
|
## Installation
|
6
|
-
|
7
8
|
```
|
8
9
|
# for system installed fluentd
|
9
10
|
$ gem install fluent-plugin-path2tag
|
10
11
|
|
11
12
|
# for td-agent (Legacy)
|
12
|
-
$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install ffluent-plugin-path2tag -v 0.0
|
13
|
+
$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install ffluent-plugin-path2tag -v 1.0.0
|
13
14
|
|
14
15
|
# for td-agent2 (with fluentd v0.12)
|
15
|
-
$ sudo td-agent-gem install fluent-plugin-path2tag -v 0.0
|
16
|
+
$ sudo td-agent-gem install fluent-plugin-path2tag -v 1.0.0
|
16
17
|
```
|
17
18
|
|
18
19
|
## Configuration
|
@@ -20,7 +21,6 @@ $ sudo td-agent-gem install fluent-plugin-path2tag -v 0.0.1
|
|
20
21
|
### Syntax
|
21
22
|
```
|
22
23
|
path2tag <path_key_name> <data_key_name>
|
23
|
-
|
24
24
|
```
|
25
25
|
|
26
26
|
### Usage
|
@@ -48,4 +48,3 @@ path2tag <path_key_name> <data_key_name>
|
|
48
48
|
Copyright (c) 2017- Shota Kuwahara
|
49
49
|
[MIT License](http://opensource.org/licenses/MIT)
|
50
50
|
|
51
|
-
|
@@ -1,6 +1,11 @@
|
|
1
1
|
class Fluent::Path2tagOutput < Fluent::Output
|
2
2
|
Fluent::Plugin.register_output('path2tag', self)
|
3
3
|
|
4
|
+
# Define `router` method of v0.12 to support v0.10 or earlier
|
5
|
+
unless method_defined?(:router)
|
6
|
+
define_method("router") { Fluent::Engine }
|
7
|
+
end
|
8
|
+
|
4
9
|
def configure(conf)
|
5
10
|
super
|
6
11
|
|
@@ -8,49 +13,81 @@ class Fluent::Path2tagOutput < Fluent::Output
|
|
8
13
|
conf.keys.select{|k| k =~ /^path2tag$/}.each do |key|
|
9
14
|
path_key, data_key = conf[key].split(" ")
|
10
15
|
if path_key.nil? || data_key.nil?
|
11
|
-
raise Fluent::ConfigError, "failed to parse path2tag at #{key} #{conf[key]}"
|
16
|
+
raise Fluent::ConfigError, "[path2tag] failed to parse path2tag at #{key} #{conf[key]}"
|
12
17
|
end
|
13
18
|
|
14
19
|
@rewriterules.push([path_key, data_key])
|
15
|
-
log.info "adding path2tag rule: #{key} #{@rewriterules.last}"
|
20
|
+
log.info "[path2tag] adding path2tag rule: #{key} #{@rewriterules.last}"
|
16
21
|
end
|
17
22
|
|
18
23
|
unless @rewriterules.length > 0
|
19
|
-
raise Fluent::ConfigError, "missing path2tag rule"
|
24
|
+
raise Fluent::ConfigError, "[path2tag] missing path2tag rule"
|
20
25
|
end
|
21
26
|
end
|
22
27
|
|
23
28
|
def emit(tag, es, chain)
|
24
29
|
es.each do |time,record|
|
25
|
-
rewrited_tag,
|
26
|
-
if
|
27
|
-
|
30
|
+
rewrited_tag, newrecords = rewrite_tag_record(tag, record)
|
31
|
+
next if newrecords.nil? || tag == rewrited_tag
|
32
|
+
|
33
|
+
log.info "[path2tag] change tag to #{rewrited_tag}"
|
34
|
+
|
35
|
+
if newrecords.class == "Array"
|
36
|
+
newrecords.each do |r|
|
37
|
+
router.emit(rewrited_tag, time, r)
|
38
|
+
end
|
28
39
|
else
|
29
|
-
router.emit(rewrited_tag, time,
|
40
|
+
router.emit(rewrited_tag, time, newrecords)
|
30
41
|
end
|
31
42
|
end
|
32
43
|
|
33
44
|
chain.next
|
34
45
|
end
|
35
46
|
|
47
|
+
def parse(str = '')
|
48
|
+
array = str.scan(/(?:\\x..)+/).uniq.sort{ |a, b| b.length <=> a.length }
|
49
|
+
array.each do |reg|
|
50
|
+
s = [reg.gsub('\\x', 'x').split('x').slice(1..-1).join('')].pack('H*').force_encoding('utf-8')
|
51
|
+
str = str.gsub(reg, s)
|
52
|
+
end
|
53
|
+
return JSON.parse(str)
|
54
|
+
end
|
55
|
+
|
36
56
|
def rewrite_tag_record(tag, record)
|
37
57
|
path_key, data_key = @rewriterules[0]
|
38
|
-
|
58
|
+
|
39
59
|
unless record.has_key?(path_key)
|
40
|
-
log.warn "record has no path_key <#{path_key}>"
|
41
|
-
return
|
60
|
+
log.warn "[path2tag] record has no path_key <#{path_key}>"
|
61
|
+
return tag, nil
|
42
62
|
end
|
43
63
|
|
44
64
|
unless record.has_key?(data_key)
|
45
|
-
log.warn "record has no data_key <#{data_key}>"
|
46
|
-
return
|
65
|
+
log.warn "[path2tag] record has no data_key <#{data_key}>"
|
66
|
+
return tag, nil
|
67
|
+
end
|
68
|
+
|
69
|
+
if record[data_key].empty?
|
70
|
+
log.warn "[path2tag] record is empty <#{data_key}>"
|
71
|
+
return tag, nil
|
72
|
+
end
|
73
|
+
|
74
|
+
begin
|
75
|
+
newrecords = parse(record[data_key])
|
76
|
+
rescue => e
|
77
|
+
log.warn "[path2tag] JSON parse error!"
|
78
|
+
return tag, nil
|
47
79
|
end
|
48
80
|
|
49
81
|
newtag = record[path_key]
|
50
82
|
if newtag.start_with?("/")
|
51
83
|
newtag = newtag.sub("/", "")
|
52
84
|
end
|
85
|
+
|
86
|
+
if newtag.end_with?("/")
|
87
|
+
newtag = newtag.chop
|
88
|
+
end
|
53
89
|
newtag = newtag.gsub("/", ".")
|
54
|
-
|
90
|
+
|
91
|
+
return newtag, newrecords
|
55
92
|
end
|
56
93
|
end
|
@@ -50,55 +50,106 @@ class Path2tagTest < Test::Unit::TestCase
|
|
50
50
|
|
51
51
|
def test_emit_ok
|
52
52
|
d1 = create_driver(CONFIG_OK, 'nginx.access')
|
53
|
+
body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
|
53
54
|
d1.run do
|
54
|
-
d1.emit({'request_uri' => '/
|
55
|
+
d1.emit({'request_uri' => '/path/foo/bar', 'request_body' => body})
|
55
56
|
end
|
56
57
|
emits = d1.emits
|
57
58
|
assert_equal 1, emits.length
|
58
|
-
assert_equal '
|
59
|
-
assert_equal
|
59
|
+
assert_equal 'path.foo.bar', emits[0][0]
|
60
|
+
assert_equal JSON.parse(body), emits[0][2]
|
60
61
|
end
|
61
62
|
|
62
63
|
def test_emit_no_path_key
|
63
64
|
d1 = create_driver(CONFIG_NO_PATH_KEY, 'nginx.access')
|
65
|
+
body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
|
64
66
|
d1.run do
|
65
|
-
d1.emit({'request_uri' => '/
|
67
|
+
d1.emit({'request_uri' => '/path/foo/bar', 'request_body' => body})
|
66
68
|
end
|
67
69
|
emits = d1.emits
|
68
|
-
assert_equal
|
69
|
-
assert_equal 'path2tag.clear', emits[0][0]
|
70
|
+
assert_equal 0, emits.length
|
70
71
|
end
|
71
72
|
|
72
73
|
def test_emit_no_data_key
|
73
74
|
d1 = create_driver(CONFIG_NO_DATA_KEY, 'nginx.access')
|
75
|
+
body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
|
74
76
|
d1.run do
|
75
|
-
d1.emit({'request_uri' => '/
|
77
|
+
d1.emit({'request_uri' => '/path/foo/bar', 'request_body' => body})
|
76
78
|
end
|
77
79
|
emits = d1.emits
|
78
|
-
assert_equal
|
79
|
-
assert_equal 'path2tag.clear', emits[0][0]
|
80
|
+
assert_equal 0, emits.length
|
80
81
|
end
|
81
82
|
|
82
83
|
def test_emit_path_format_1
|
83
84
|
d1 = create_driver(CONFIG_OK, 'nginx.access')
|
85
|
+
body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
|
84
86
|
d1.run do
|
85
|
-
d1.emit({'request_uri' => '/
|
87
|
+
d1.emit({'request_uri' => '/path/foo/bar', 'request_body' => body})
|
86
88
|
end
|
87
89
|
emits = d1.emits
|
88
90
|
assert_equal 1, emits.length
|
89
|
-
assert_equal '
|
90
|
-
assert_equal
|
91
|
+
assert_equal 'path.foo.bar', emits[0][0]
|
92
|
+
assert_equal JSON.parse(body), emits[0][2]
|
91
93
|
end
|
92
94
|
|
93
95
|
def test_emit_path_format_2
|
94
96
|
d1 = create_driver(CONFIG_OK, 'nginx.access')
|
97
|
+
body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
|
95
98
|
d1.run do
|
96
|
-
d1.emit({'request_uri' => '
|
99
|
+
d1.emit({'request_uri' => 'path/foo/bar', 'request_body' => body})
|
97
100
|
end
|
98
101
|
emits = d1.emits
|
99
102
|
assert_equal 1, emits.length
|
100
|
-
assert_equal '
|
101
|
-
assert_equal
|
103
|
+
assert_equal 'path.foo.bar', emits[0][0]
|
104
|
+
assert_equal JSON.parse(body), emits[0][2]
|
102
105
|
end
|
106
|
+
|
107
|
+
def test_emit_path_format_3
|
108
|
+
d1 = create_driver(CONFIG_OK, 'nginx.access')
|
109
|
+
body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
|
110
|
+
d1.run do
|
111
|
+
d1.emit({'request_uri' => '/path/foo/bar/', 'request_body' => body})
|
112
|
+
end
|
113
|
+
emits = d1.emits
|
114
|
+
assert_equal 1, emits.length
|
115
|
+
assert_equal 'path.foo.bar', emits[0][0]
|
116
|
+
assert_equal JSON.parse(body), emits[0][2]
|
117
|
+
end
|
118
|
+
|
119
|
+
def test_emit_path_format_4
|
120
|
+
d1 = create_driver(CONFIG_OK, 'nginx.access')
|
121
|
+
body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
|
122
|
+
d1.run do
|
123
|
+
d1.emit({'request_uri' => 'path', 'request_body' => body})
|
124
|
+
end
|
125
|
+
emits = d1.emits
|
126
|
+
assert_equal 1, emits.length
|
127
|
+
assert_equal 'path', emits[0][0]
|
128
|
+
assert_equal JSON.parse(body), emits[0][2]
|
129
|
+
end
|
130
|
+
|
131
|
+
def test_emit_json_parse
|
132
|
+
d1 = create_driver(CONFIG_OK, 'nginx.access')
|
133
|
+
body = '{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}'
|
134
|
+
d1.run do
|
135
|
+
d1.emit({'request_uri' => 'path/foo/bar', 'request_body' => body})
|
136
|
+
end
|
137
|
+
emits = d1.emits
|
138
|
+
assert_equal 1, emits.length
|
139
|
+
assert_equal 'path.foo.bar', emits[0][0]
|
140
|
+
assert_equal JSON.parse(body), emits[0][2]
|
141
|
+
end
|
142
|
+
|
143
|
+
def test_emit_json_parse_2
|
144
|
+
d1 = create_driver(CONFIG_OK, 'nginx.access')
|
145
|
+
body = '-'
|
146
|
+
d1.run do
|
147
|
+
d1.emit({'request_uri' => 'path/foo/bar', 'request_body' => body})
|
148
|
+
end
|
149
|
+
emits = d1.emits
|
150
|
+
assert_equal 0, emits.length
|
151
|
+
end
|
152
|
+
|
153
|
+
|
103
154
|
end
|
104
155
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-path2tag
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shota Kuwahara
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-03-
|
11
|
+
date: 2017-03-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: test-unit
|