fluent-plugin-rewrite-tag-filter 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
data/README.rdoc
CHANGED
@@ -4,7 +4,9 @@
|
|
4
4
|
|
5
5
|
=== RewriteTagFilterOutput
|
6
6
|
|
7
|
-
|
7
|
+
It has designed to rewrite tags like mod_rewrite.
|
8
|
+
Also you can change a tag from apache log by domain, status-code(ex. 500 error),
|
9
|
+
user-agent, and request-uri and so on with regular expression.
|
8
10
|
|
9
11
|
== Installation
|
10
12
|
|
@@ -22,7 +24,7 @@ Format
|
|
22
24
|
|
23
25
|
Usage
|
24
26
|
|
25
|
-
# It's a sample to exclude some
|
27
|
+
# It's a sample to exclude some static file log before split tag by domain.
|
26
28
|
<source>
|
27
29
|
type tail
|
28
30
|
path /var/log/httpd/access_log
|
@@ -38,6 +40,7 @@ Usage
|
|
38
40
|
rewriterule2 domain ^maps\.google\.com$ site.GoogleMap
|
39
41
|
rewriterule3 domain ^news\.google\.com$ site.GoogleNews
|
40
42
|
rewriterule4 domain ^mail\.google\.com$ site.GoogleMail
|
43
|
+
rewriterule5 domain .+ site.unmatched
|
41
44
|
</match>
|
42
45
|
|
43
46
|
<match site.*>
|
@@ -46,7 +49,6 @@ Usage
|
|
46
49
|
database apache_access
|
47
50
|
remove_tag_prefix site
|
48
51
|
tag_mapped
|
49
|
-
collection misc
|
50
52
|
capped
|
51
53
|
capped_size 100m
|
52
54
|
</match>
|
@@ -65,6 +67,7 @@ Result
|
|
65
67
|
GoogleMap
|
66
68
|
GoogleNews
|
67
69
|
GoogleMail
|
70
|
+
unmatched
|
68
71
|
|
69
72
|
== Example
|
70
73
|
|
@@ -74,12 +77,17 @@ Result
|
|
74
77
|
- Example2: how to exclude specified patterns before analyze response_time for each virtual domain websites.
|
75
78
|
- https://github.com/y-ken/fluent-plugin-rewrite-tag-filter/blob/master/example2.conf
|
76
79
|
|
80
|
+
== Review
|
81
|
+
|
82
|
+
- 自在にタグを書き換える fluent-plugin-rewrite-tag-filter でログ解析が捗るお話 #fluentd
|
83
|
+
- http://d.hatena.ne.jp/yoshi-ken/20120701/1341137269
|
84
|
+
|
77
85
|
== TODO
|
78
86
|
|
79
87
|
- patches welcome!
|
80
88
|
|
81
89
|
== Copyright
|
82
90
|
|
83
|
-
Copyright:: Copyright (c) 2012- Kentaro Yoshida
|
91
|
+
Copyright:: Copyright (c) 2012- Kentaro Yoshida (@yoshi_ken)
|
84
92
|
License:: Apache License, Version 2.0
|
85
93
|
|
data/example2.conf
CHANGED
@@ -29,9 +29,10 @@
|
|
29
29
|
rewriterule1 path ^/(img|css|js|static|assets)/ clear
|
30
30
|
rewriterule2 status ^(?!200)$ clear
|
31
31
|
rewriterule3 method ^(?!GET)$ clear
|
32
|
-
rewriterule4 agent
|
33
|
-
rewriterule5
|
34
|
-
rewriterule6 path
|
32
|
+
rewriterule4 agent "Mac OS X" clear
|
33
|
+
rewriterule5 agent (spider|bot|crawler|\+http\:) clear
|
34
|
+
rewriterule6 path ^/(admin|api|backend) site.GoogleMap.backend
|
35
|
+
rewriterule7 path .+ site.GoogleMap.front
|
35
36
|
</match>
|
36
37
|
|
37
38
|
|
@@ -3,8 +3,8 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-rewrite-tag-filter"
|
6
|
-
s.version = "0.0.
|
7
|
-
s.authors = ["Kentaro
|
6
|
+
s.version = "0.0.5"
|
7
|
+
s.authors = ["Kentaro YOSHIDA"]
|
8
8
|
s.email = ["y.ken.studio@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/y-ken/fluent-plugin-rewrite-tag-filter"
|
10
10
|
s.summary = %q{Fluentd Output filter plugin to rewrite tags that matches specified attribute.}
|
@@ -20,7 +20,7 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
|
|
20
20
|
end
|
21
21
|
(1..PATTERN_MAX_NUM).each do |i|
|
22
22
|
next unless conf["rewriterule#{i}"]
|
23
|
-
rewritekey,regexp,rewritetag = conf["rewriterule#{i}"].split(
|
23
|
+
rewritekey,regexp,rewritetag = conf["rewriterule#{i}"].split(/^(\S+) "?([\S\s]+?)"? (\S+)$/).reject(&:empty?)
|
24
24
|
unless regexp != nil && rewritetag != nil
|
25
25
|
raise Fluent::ConfigError, "missing values at " + "rewriterule#{i} " + conf["rewriterule#{i}"].inspect
|
26
26
|
end
|
@@ -8,6 +8,7 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
|
|
8
8
|
CONFIG = %[
|
9
9
|
rewriterule1 domain ^www\.google\.com$ site.Google
|
10
10
|
rewriterule2 domain ^news\.google\.com$ site.GoogleNews
|
11
|
+
rewriterule3 agent "Mac OS X" agent.MacOSX
|
11
12
|
]
|
12
13
|
|
13
14
|
def create_driver(conf=CONFIG,tag='test')
|
@@ -31,14 +32,16 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
|
|
31
32
|
d1 = create_driver(CONFIG, 'input.access')
|
32
33
|
time = Time.parse("2012-01-02 13:14:15").to_i
|
33
34
|
d1.run do
|
34
|
-
d1.emit({'domain' => 'www.google.com', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
35
|
-
d1.emit({'domain' => 'news.google.com', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
|
35
|
+
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
36
|
+
d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
|
37
|
+
d1.emit({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
|
36
38
|
end
|
37
39
|
emits = d1.emits
|
38
|
-
assert_equal
|
40
|
+
assert_equal 3, emits.length
|
39
41
|
assert_equal 'site.Google', emits[0][0] # tag
|
40
42
|
assert_equal 'site.GoogleNews', emits[1][0] # tag
|
41
43
|
assert_equal 'news.google.com', emits[1][2]['domain']
|
44
|
+
assert_equal 'agent.MacOSX', emits[2][0] #tag
|
42
45
|
end
|
43
46
|
end
|
44
47
|
|
metadata
CHANGED
@@ -1,19 +1,19 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-rewrite-tag-filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
|
-
- Kentaro
|
8
|
+
- Kentaro YOSHIDA
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-07-
|
12
|
+
date: 2012-07-27 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
16
|
-
requirement: &
|
16
|
+
requirement: &21462300 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :development
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *21462300
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: fluentd
|
27
|
-
requirement: &
|
27
|
+
requirement: &21461000 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,7 +32,7 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *21461000
|
36
36
|
description: Rewrite Tags with data matches any of specified regexp patterns in specified
|
37
37
|
attribute.
|
38
38
|
email:
|