fluent-plugin-rewrite-tag-filter 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. data/README.rdoc +33 -6
  2. data/Rakefile +9 -0
  3. data/example.conf +113 -0
  4. data/fluent-plugin-rewrite-tag-filter.gemspec +3 -3
  5. data/lib/fluent/plugin/out_rewrite_tag_filter.rb +1 -1
  6. data/test/helper.rb +28 -0
  7. data/test/plugin/test_out_rewrite_tag_filter.rb +44 -0
  8. metadata +12 -8
data/README.rdoc CHANGED
@@ -1,26 +1,53 @@
1
1
  = fluent-plugin-rewrite-tag-filter
2
2
 
3
- == Component
3
+ == Overview
4
4
 
5
5
  === RewriteTagFilterOutput
6
6
 
7
7
  Modify Tags with data matches any of specified regexp patterns in specified attribute.
8
8
 
9
+ == Installation
10
+
11
+ # native fluentd user
12
+ gem install fluent-plugin-rewrite-tag-filter
13
+
14
+ # td-agent user
15
+ /usr/lib64/fluent/ruby/bin/gem install fluent-plugin-rewrite-tag-filter
16
+
9
17
  == Configuration
10
18
 
19
+ Format
20
+
21
+ rewruterule<num:1-200> <attribute> <regex_pattern> <rewrite_tag>
22
+
23
+ Usage
24
+
11
25
  <match td.apache.access>
12
26
  type rewrite_tag_filter
13
- rewriterule1 domain ^maps\.google\.com$ site.Google.Map
14
- rewriterule2 domain ^news.google\.com$ site.Google.News
15
- rewriterule3 agent (Googlebot|Crawler|Spyder) agent.Robot
16
- rewriterule4 referer headlines\.yahoo\.co\.jp referer.YahooHeadlines
27
+ rewriterule1 domain ^maps\.google\.com$ site.GoogleMap
28
+ rewriterule2 domain ^news\.google\.com$ site.GoogleNews
29
+ rewriterule3 domain ^mail\.google\.com$ site.GoogleMail
17
30
  </match>
18
31
 
32
+ <match site.*>
33
+ type mongo
34
+ host 10.10.0.36
35
+ database apache_access
36
+ remove_tag_prefix site
37
+ capped
38
+ capped_size 100m
39
+ </match>
40
+
41
+ Example
42
+
43
+ - Example1: how to analyze response_time, response_code and user_agent for each virtual domain websites.
44
+ - https://github.com/y-ken/fluent-plugin-rewrite-tag-filter/blob/master/example.conf
45
+
19
46
  == TODO
20
47
 
21
48
  - patches welcome!
22
49
 
23
50
  == Copyright
24
51
 
25
- Copyright:: Copyright (c) 2012- Kentaro Yoshida (yoshi_ken)
52
+ Copyright:: Copyright (c) 2012- Kentaro Yoshida
26
53
  License:: Apache License, Version 2.0
data/Rakefile CHANGED
@@ -1 +1,10 @@
1
1
  require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+ Rake::TestTask.new(:test) do |test|
4
+ test.libs << 'lib' << 'test'
5
+ test.pattern = 'test/**/test_*.rb'
6
+ test.verbose = true
7
+ end
8
+
9
+ task :default => :test
10
+
data/example.conf ADDED
@@ -0,0 +1,113 @@
1
+ # it is a sample how to analyze response_time, response_code and user_agent for each virtual domain websites.
2
+
3
+ <source>
4
+ type tail
5
+ path /var/log/httpd/access_log
6
+ format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
7
+ time_format %d/%b/%Y:%H:%M:%S %z
8
+ tag td.apache.access
9
+ pos_file /var/log/td-agent/apache_access.pos
10
+ </source>
11
+
12
+ <match td.apache.access>
13
+ type rewrite_tag_filter
14
+ rewriterule1 domain ^maps\.google\.com$ site.GoogleMap
15
+ rewriterule2 domain ^news\.google\.com$ site.GoogleNews
16
+ rewriterule3 domain ^mail\.google\.com$ site.GoogleMail
17
+ </match>
18
+
19
+ <match site.**>
20
+ type copy
21
+ <store>
22
+ type forest
23
+ subtype datacounter
24
+ <template>
25
+ unit minute
26
+ count_key response_time
27
+ outcast_unmatched false
28
+ aggregate all
29
+ tag gf.responsetime.__TAG__
30
+ pattern1 0-100msec ^\d{1,5}$
31
+ pattern2 100-300msec ^[1-2]\d{5}$
32
+ pattern3 300-600msec ^[3-5]\d{5}$
33
+ pattern4 600msec-1sec ^[6-9]\d{5}$
34
+ pattern5 1-2sec ^1\d{6}$
35
+ pattern6 2-10sec ^[2-9]\d{6}$
36
+ pattern7 10sec_over ^\d{8,}$
37
+ </template>
38
+ </store>
39
+ <store>
40
+ type forest
41
+ subtype datacounter
42
+ <template>
43
+ unit minute
44
+ outcast_unmatched false
45
+ aggregate all
46
+ tag gf.responsecode.__TAG__
47
+ count_key status
48
+ pattern1 200 ^200$
49
+ pattern2 2xx ^2\d\d$
50
+ pattern3 301 ^301$
51
+ pattern4 302 ^302$
52
+ pattern5 3xx ^3\d\d$
53
+ pattern6 403 ^403$
54
+ pattern7 404 ^404$
55
+ pattern8 410 ^410$
56
+ pattern9 4xx ^4\d\d$
57
+ pattern10 500 ^5\d\d$
58
+ </template>
59
+ </store>
60
+ <store>
61
+ type forest
62
+ subtype datacounter
63
+ <template>
64
+ unit minute
65
+ count_key agent
66
+ outcast_unmatched false
67
+ aggregate all
68
+ tag gf.useragent.__TAG__
69
+ pattern1 api HttpRequest
70
+ pattern2 robot (spider|bot|crawler|\+http\:)
71
+ pattern3 smartphone (iPhone|iPod|Android|dream|CUPCAKE|blackberry|webOS|incognito|webmate|IEMobile)
72
+ pattern4 mobile (^KDDI|UP.Browser|DoCoMo|Vodafone|SoftBank|WILLCOM)
73
+ pattern5 pc .+
74
+ </template>
75
+ </store>
76
+ </match>
77
+
78
+ <match gf.responsecode.**>
79
+ type forest
80
+ subtype growthforecast
81
+ remove_prefix gf.responsecode.site
82
+ <template>
83
+ gfapi_url http://10.10.0.55:5125/api/
84
+ service __TAG__
85
+ section response_code
86
+ name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
87
+ </template>
88
+ </match>
89
+
90
+ <match gf.responsetime.**>
91
+ type forest
92
+ subtype growthforecast
93
+ remove_prefix gf.responsetime.site
94
+ <template>
95
+ gfapi_url http://10.10.0.55:5125/api/
96
+ service __TAG__
97
+ section response_time
98
+ name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
99
+ </template>
100
+ </match>
101
+
102
+ <match gf.useragent.**>
103
+ type forest
104
+ subtype growthforecast
105
+ remove_prefix gf.useragent.site
106
+ <template>
107
+ gfapi_url http://10.10.0.55:5125/api/
108
+ service __TAG__
109
+ section useragent
110
+ name_keys pc_count,mobile_count,smartphone_count,robot_count,api_count
111
+ </template>
112
+ </match>
113
+
data/fluent-plugin-rewrite-tag-filter.gemspec CHANGED
@@ -3,12 +3,12 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-rewrite-tag-filter"
6
- s.version = "0.0.1"
6
+ s.version = "0.0.2"
7
7
  s.authors = ["Kentaro Yoshida"]
8
- s.email = [""]
8
+ s.email = ["y.ken.studio@gmail.com"]
9
9
  s.homepage = "https://github.com/y-ken/fluent-plugin-rewrite-tag-filter"
10
10
  s.summary = %q{Output filter plugin to rewrite tags that matches specified attribute}
11
- s.description = %q{Output filter plugin to rewrite tags that matches specified attribute}
11
+ s.description = %q{Modify Tags with data matches any of specified regexp patterns in specified attribute.}
12
12
 
13
13
  s.rubyforge_project = "fluent-plugin-rewrite-tag-filter"
14
14
 
data/lib/fluent/plugin/out_rewrite_tag_filter.rb CHANGED
@@ -1,7 +1,7 @@
1
1
  class Fluent::RewriteTagFilterOutput < Fluent::Output
2
2
  Fluent::Plugin.register_output('rewrite_tag_filter', self)
3
3
 
4
- PATTERN_MAX_NUM = 50
4
+ PATTERN_MAX_NUM = 200
5
5
 
6
6
  config_param :rewriterule1, :string # string: NAME REGEXP
7
7
  (2..PATTERN_MAX_NUM).each do |i|
data/test/helper.rb ADDED
@@ -0,0 +1,28 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'fluent/test'
15
+ unless ENV.has_key?('VERBOSE')
16
+ nulllogger = Object.new
17
+ nulllogger.instance_eval {|obj|
18
+ def method_missing(method, *args)
19
+ # pass
20
+ end
21
+ }
22
+ $log = nulllogger
23
+ end
24
+
25
+ require 'fluent/plugin/out_rewrite_tag_filter'
26
+
27
+ class Test::Unit::TestCase
28
+ end
data/test/plugin/test_out_rewrite_tag_filter.rb CHANGED
@@ -0,0 +1,44 @@
1
+ require 'helper'
2
+
3
+ class RewriteTagFilterOutputTest < Test::Unit::TestCase
4
+ def setup
5
+ Fluent::Test.setup
6
+ end
7
+
8
+ CONFIG = %[
9
+ rewriterule1 domain ^www\.google\.com$ site.Google
10
+ rewriterule2 domain ^news\.google\.com$ site.GoogleNews
11
+ ]
12
+
13
+ def create_driver(conf=CONFIG,tag='test')
14
+ Fluent::Test::OutputTestDriver.new(Fluent::RewriteTagFilterOutput, tag).configure(conf)
15
+ end
16
+
17
+ def test_configure
18
+ assert_raise(Fluent::ConfigError) {
19
+ d = create_driver('')
20
+ }
21
+ d = create_driver %[
22
+ rewriterule1 domain ^www.google.com$ site.Google
23
+ rewriterule2 domain ^news.google.com$ site.GoogleNews
24
+ ]
25
+ d.instance.inspect
26
+ assert_equal 'domain ^www.google.com$ site.Google', d.instance.rewriterule1
27
+ assert_equal 'domain ^news.google.com$ site.GoogleNews', d.instance.rewriterule2
28
+ end
29
+
30
+ def test_emit
31
+ d1 = create_driver(CONFIG, 'input.access')
32
+ time = Time.parse("2012-01-02 13:14:15").to_i
33
+ d1.run do
34
+ d1.emit({'domain' => 'www.google.com', 'agent' => 'Googlebot', 'response_time' => 1000000})
35
+ d1.emit({'domain' => 'news.google.com', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
36
+ end
37
+ emits = d1.emits
38
+ assert_equal 2, emits.length
39
+ assert_equal 'site.Google', emits[0][0] # tag
40
+ assert_equal 'site.GoogleNews', emits[1][0] # tag
41
+ assert_equal 'news.google.com', emits[1][2]['domain']
42
+ end
43
+ end
44
+
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-rewrite-tag-filter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-06-04 00:00:00.000000000 Z
12
+ date: 2012-06-30 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
16
- requirement: &16391840 !ruby/object:Gem::Requirement
16
+ requirement: &11789920 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: '0'
22
22
  type: :development
23
23
  prerelease: false
24
- version_requirements: *16391840
24
+ version_requirements: *11789920
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: fluentd
27
- requirement: &16390600 !ruby/object:Gem::Requirement
27
+ requirement: &11788560 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,10 +32,11 @@ dependencies:
32
32
  version: '0'
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *16390600
36
- description: Output filter plugin to rewrite tags that matches specified attribute
35
+ version_requirements: *11788560
36
+ description: Modify Tags with data matches any of specified regexp patterns in specified
37
+ attribute.
37
38
  email:
38
- - ''
39
+ - y.ken.studio@gmail.com
39
40
  executables: []
40
41
  extensions: []
41
42
  extra_rdoc_files: []
@@ -45,8 +46,10 @@ files:
45
46
  - LICENSE.txt
46
47
  - README.rdoc
47
48
  - Rakefile
49
+ - example.conf
48
50
  - fluent-plugin-rewrite-tag-filter.gemspec
49
51
  - lib/fluent/plugin/out_rewrite_tag_filter.rb
52
+ - test/helper.rb
50
53
  - test/plugin/test_out_rewrite_tag_filter.rb
51
54
  homepage: https://github.com/y-ken/fluent-plugin-rewrite-tag-filter
52
55
  licenses: []
@@ -73,4 +76,5 @@ signing_key:
73
76
  specification_version: 3
74
77
  summary: Output filter plugin to rewrite tags that matches specified attribute
75
78
  test_files:
79
+ - test/helper.rb
76
80
  - test/plugin/test_out_rewrite_tag_filter.rb