fluent-plugin-rewrite-tag-filter 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (8) hide show
  1. data/README.rdoc +33 -6
  2. data/Rakefile +9 -0
  3. data/example.conf +113 -0
  4. data/fluent-plugin-rewrite-tag-filter.gemspec +3 -3
  5. data/lib/fluent/plugin/out_rewrite_tag_filter.rb +1 -1
  6. data/test/helper.rb +28 -0
  7. data/test/plugin/test_out_rewrite_tag_filter.rb +44 -0
  8. metadata +12 -8
data/README.rdoc CHANGED
@@ -1,26 +1,53 @@
1
1
  = fluent-plugin-rewrite-tag-filter
2
2
 
3
- == Component
3
+ == Overview
4
4
 
5
5
  === RewriteTagFilterOutput
6
6
 
7
7
  Modify Tags with data matches any of specified regexp patterns in specified attribute.
8
8
 
9
+ == Installation
10
+
11
+ # native fluentd user
12
+ gem install fluent-plugin-rewrite-tag-filter
13
+
14
+ # td-agent user
15
+ /usr/lib64/fluent/ruby/bin/gem install fluent-plugin-rewrite-tag-filter
16
+
9
17
  == Configuration
10
18
 
19
+ Format
20
+
21
+ rewruterule<num:1-200> <attribute> <regex_pattern> <rewrite_tag>
22
+
23
+ Usage
24
+
11
25
  <match td.apache.access>
12
26
  type rewrite_tag_filter
13
- rewriterule1 domain ^maps\.google\.com$ site.Google.Map
14
- rewriterule2 domain ^news.google\.com$ site.Google.News
15
- rewriterule3 agent (Googlebot|Crawler|Spyder) agent.Robot
16
- rewriterule4 referer headlines\.yahoo\.co\.jp referer.YahooHeadlines
27
+ rewriterule1 domain ^maps\.google\.com$ site.GoogleMap
28
+ rewriterule2 domain ^news\.google\.com$ site.GoogleNews
29
+ rewriterule3 domain ^mail\.google\.com$ site.GoogleMail
17
30
  </match>
18
31
 
32
+ <match site.*>
33
+ type mongo
34
+ host 10.10.0.36
35
+ database apache_access
36
+ remove_tag_prefix site
37
+ capped
38
+ capped_size 100m
39
+ </match>
40
+
41
+ Example
42
+
43
+ - Example1: how to analyze response_time, response_code and user_agent for each virtual domain websites.
44
+ - https://github.com/y-ken/fluent-plugin-rewrite-tag-filter/blob/master/example.conf
45
+
19
46
  == TODO
20
47
 
21
48
  - patches welcome!
22
49
 
23
50
  == Copyright
24
51
 
25
- Copyright:: Copyright (c) 2012- Kentaro Yoshida (yoshi_ken)
52
+ Copyright:: Copyright (c) 2012- Kentaro Yoshida
26
53
  License:: Apache License, Version 2.0
data/Rakefile CHANGED
@@ -1 +1,10 @@
1
1
  require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+ Rake::TestTask.new(:test) do |test|
4
+ test.libs << 'lib' << 'test'
5
+ test.pattern = 'test/**/test_*.rb'
6
+ test.verbose = true
7
+ end
8
+
9
+ task :default => :test
10
+
data/example.conf ADDED
@@ -0,0 +1,113 @@
1
+ # it is a sample how to analyze response_time, response_code and user_agent for each virtual domain websites.
2
+
3
+ <source>
4
+ type tail
5
+ path /var/log/httpd/access_log
6
+ format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
7
+ time_format %d/%b/%Y:%H:%M:%S %z
8
+ tag td.apache.access
9
+ pos_file /var/log/td-agent/apache_access.pos
10
+ </source>
11
+
12
+ <match td.apache.access>
13
+ type rewrite_tag_filter
14
+ rewriterule1 domain ^maps\.google\.com$ site.GoogleMap
15
+ rewriterule2 domain ^news\.google\.com$ site.GoogleNews
16
+ rewriterule3 domain ^mail\.google\.com$ site.GoogleMail
17
+ </match>
18
+
19
+ <match site.**>
20
+ type copy
21
+ <store>
22
+ type forest
23
+ subtype datacounter
24
+ <template>
25
+ unit minute
26
+ count_key response_time
27
+ outcast_unmatched false
28
+ aggregate all
29
+ tag gf.responsetime.__TAG__
30
+ pattern1 0-100msec ^\d{1,5}$
31
+ pattern2 100-300msec ^[1-2]\d{5}$
32
+ pattern3 300-600msec ^[3-5]\d{5}$
33
+ pattern4 600msec-1sec ^[6-9]\d{5}$
34
+ pattern5 1-2sec ^1\d{6}$
35
+ pattern6 2-10sec ^[2-9]\d{6}$
36
+ pattern7 10sec_over ^\d{8,}$
37
+ </template>
38
+ </store>
39
+ <store>
40
+ type forest
41
+ subtype datacounter
42
+ <template>
43
+ unit minute
44
+ outcast_unmatched false
45
+ aggregate all
46
+ tag gf.responsecode.__TAG__
47
+ count_key status
48
+ pattern1 200 ^200$
49
+ pattern2 2xx ^2\d\d$
50
+ pattern3 301 ^301$
51
+ pattern4 302 ^302$
52
+ pattern5 3xx ^3\d\d$
53
+ pattern6 403 ^403$
54
+ pattern7 404 ^404$
55
+ pattern8 410 ^410$
56
+ pattern9 4xx ^4\d\d$
57
+ pattern10 500 ^5\d\d$
58
+ </template>
59
+ </store>
60
+ <store>
61
+ type forest
62
+ subtype datacounter
63
+ <template>
64
+ unit minute
65
+ count_key agent
66
+ outcast_unmatched false
67
+ aggregate all
68
+ tag gf.useragent.__TAG__
69
+ pattern1 api HttpRequest
70
+ pattern2 robot (spider|bot|crawler|\+http\:)
71
+ pattern3 smartphone (iPhone|iPod|Android|dream|CUPCAKE|blackberry|webOS|incognito|webmate|IEMobile)
72
+ pattern4 mobile (^KDDI|UP.Browser|DoCoMo|Vodafone|SoftBank|WILLCOM)
73
+ pattern5 pc .+
74
+ </template>
75
+ </store>
76
+ </match>
77
+
78
+ <match gf.responsecode.**>
79
+ type forest
80
+ subtype growthforecast
81
+ remove_prefix gf.responsecode.site
82
+ <template>
83
+ gfapi_url http://10.10.0.55:5125/api/
84
+ service __TAG__
85
+ section response_code
86
+ name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
87
+ </template>
88
+ </match>
89
+
90
+ <match gf.responsetime.**>
91
+ type forest
92
+ subtype growthforecast
93
+ remove_prefix gf.responsetime.site
94
+ <template>
95
+ gfapi_url http://10.10.0.55:5125/api/
96
+ service __TAG__
97
+ section response_time
98
+ name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
99
+ </template>
100
+ </match>
101
+
102
+ <match gf.useragent.**>
103
+ type forest
104
+ subtype growthforecast
105
+ remove_prefix gf.useragent.site
106
+ <template>
107
+ gfapi_url http://10.10.0.55:5125/api/
108
+ service __TAG__
109
+ section useragent
110
+ name_keys pc_count,mobile_count,smartphone_count,robot_count,api_count
111
+ </template>
112
+ </match>
113
+
data/fluent-plugin-rewrite-tag-filter.gemspec CHANGED
@@ -3,12 +3,12 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-rewrite-tag-filter"
6
- s.version = "0.0.1"
6
+ s.version = "0.0.2"
7
7
  s.authors = ["Kentaro Yoshida"]
8
- s.email = [""]
8
+ s.email = ["y.ken.studio@gmail.com"]
9
9
  s.homepage = "https://github.com/y-ken/fluent-plugin-rewrite-tag-filter"
10
10
  s.summary = %q{Output filter plugin to rewrite tags that matches specified attribute}
11
- s.description = %q{Output filter plugin to rewrite tags that matches specified attribute}
11
+ s.description = %q{Modify Tags with data matches any of specified regexp patterns in specified attribute.}
12
12
 
13
13
  s.rubyforge_project = "fluent-plugin-rewrite-tag-filter"
14
14
 
data/lib/fluent/plugin/out_rewrite_tag_filter.rb CHANGED
@@ -1,7 +1,7 @@
1
1
  class Fluent::RewriteTagFilterOutput < Fluent::Output
2
2
  Fluent::Plugin.register_output('rewrite_tag_filter', self)
3
3
 
4
- PATTERN_MAX_NUM = 50
4
+ PATTERN_MAX_NUM = 200
5
5
 
6
6
  config_param :rewriterule1, :string # string: NAME REGEXP
7
7
  (2..PATTERN_MAX_NUM).each do |i|
data/test/helper.rb ADDED
@@ -0,0 +1,28 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'fluent/test'
15
+ unless ENV.has_key?('VERBOSE')
16
+ nulllogger = Object.new
17
+ nulllogger.instance_eval {|obj|
18
+ def method_missing(method, *args)
19
+ # pass
20
+ end
21
+ }
22
+ $log = nulllogger
23
+ end
24
+
25
+ require 'fluent/plugin/out_rewrite_tag_filter'
26
+
27
+ class Test::Unit::TestCase
28
+ end
data/test/plugin/test_out_rewrite_tag_filter.rb CHANGED
@@ -0,0 +1,44 @@
1
+ require 'helper'
2
+
3
+ class RewriteTagFilterOutputTest < Test::Unit::TestCase
4
+ def setup
5
+ Fluent::Test.setup
6
+ end
7
+
8
+ CONFIG = %[
9
+ rewriterule1 domain ^www\.google\.com$ site.Google
10
+ rewriterule2 domain ^news\.google\.com$ site.GoogleNews
11
+ ]
12
+
13
+ def create_driver(conf=CONFIG,tag='test')
14
+ Fluent::Test::OutputTestDriver.new(Fluent::RewriteTagFilterOutput, tag).configure(conf)
15
+ end
16
+
17
+ def test_configure
18
+ assert_raise(Fluent::ConfigError) {
19
+ d = create_driver('')
20
+ }
21
+ d = create_driver %[
22
+ rewriterule1 domain ^www.google.com$ site.Google
23
+ rewriterule2 domain ^news.google.com$ site.GoogleNews
24
+ ]
25
+ d.instance.inspect
26
+ assert_equal 'domain ^www.google.com$ site.Google', d.instance.rewriterule1
27
+ assert_equal 'domain ^news.google.com$ site.GoogleNews', d.instance.rewriterule2
28
+ end
29
+
30
+ def test_emit
31
+ d1 = create_driver(CONFIG, 'input.access')
32
+ time = Time.parse("2012-01-02 13:14:15").to_i
33
+ d1.run do
34
+ d1.emit({'domain' => 'www.google.com', 'agent' => 'Googlebot', 'response_time' => 1000000})
35
+ d1.emit({'domain' => 'news.google.com', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
36
+ end
37
+ emits = d1.emits
38
+ assert_equal 2, emits.length
39
+ assert_equal 'site.Google', emits[0][0] # tag
40
+ assert_equal 'site.GoogleNews', emits[1][0] # tag
41
+ assert_equal 'news.google.com', emits[1][2]['domain']
42
+ end
43
+ end
44
+
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-rewrite-tag-filter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-06-04 00:00:00.000000000 Z
12
+ date: 2012-06-30 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
16
- requirement: &16391840 !ruby/object:Gem::Requirement
16
+ requirement: &11789920 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: '0'
22
22
  type: :development
23
23
  prerelease: false
24
- version_requirements: *16391840
24
+ version_requirements: *11789920
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: fluentd
27
- requirement: &16390600 !ruby/object:Gem::Requirement
27
+ requirement: &11788560 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,10 +32,11 @@ dependencies:
32
32
  version: '0'
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *16390600
36
- description: Output filter plugin to rewrite tags that matches specified attribute
35
+ version_requirements: *11788560
36
+ description: Modify Tags with data matches any of specified regexp patterns in specified
37
+ attribute.
37
38
  email:
38
- - ''
39
+ - y.ken.studio@gmail.com
39
40
  executables: []
40
41
  extensions: []
41
42
  extra_rdoc_files: []
@@ -45,8 +46,10 @@ files:
45
46
  - LICENSE.txt
46
47
  - README.rdoc
47
48
  - Rakefile
49
+ - example.conf
48
50
  - fluent-plugin-rewrite-tag-filter.gemspec
49
51
  - lib/fluent/plugin/out_rewrite_tag_filter.rb
52
+ - test/helper.rb
50
53
  - test/plugin/test_out_rewrite_tag_filter.rb
51
54
  homepage: https://github.com/y-ken/fluent-plugin-rewrite-tag-filter
52
55
  licenses: []
@@ -73,4 +76,5 @@ signing_key:
73
76
  specification_version: 3
74
77
  summary: Output filter plugin to rewrite tags that matches specified attribute
75
78
  test_files:
79
+ - test/helper.rb
76
80
  - test/plugin/test_out_rewrite_tag_filter.rb