fluent-plugin-grepcounter 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: e62f5b7c59d5196eb5a38630bb8a1292883f1d07
4
+ data.tar.gz: 64a2bede2b197b7f693207bc63b6a71101df8c50
5
+ SHA512:
6
+ metadata.gz: 49ae4eca6395c2b381631ced8bebdb30d3451cd4f504a817b2ec858a45c9acbd198385c465062f93968e547cf49c52ec6f124d6e198b9a72457c1048a37e9cd7
7
+ data.tar.gz: f9d7d8b9c50c6d45e22af946edef16d440e147c76f1e2507bf48dd272c837dd52282db09fa9d3411aa6cf44971c35f8c410a40622ac746b23118ec041286be79
@@ -0,0 +1,11 @@
1
+ /*.gem
2
+ ~*
3
+ #*
4
+ *~
5
+ .bundle
6
+ Gemfile.lock
7
+ .rbenv-version
8
+ vendor
9
+ doc/*
10
+ tmp/*
11
+ .yardoc
@@ -0,0 +1,4 @@
1
+ set autolist
2
+ set autoeval
3
+ set autoreload
4
+
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --colour
2
+ --format documentation
@@ -0,0 +1,6 @@
1
+ rvm:
2
+ - 1.9.2
3
+ - 1.9.3
4
+ - 2.0.0
5
+ gemfile:
6
+ - Gemfile
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "http://rubygems.org"
2
+
3
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013 Naotoshi SEO
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,64 @@
1
+ # fluent-plugin-grepcounter [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-grepcounter.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-grepcounter) [![Dependency Status](https://gemnasium.com/sonots/fluent-plugin-grepcounter.png)](https://gemnasium.com/sonots/fluent-plugin-grepcounter)
2
+
3
+ ## Component
4
+
5
+ ### GrepCounterOutput
6
+
7
+ Fluentd plugin to count the number of matched messages
8
+
9
+ ## Configuration
10
+
11
+ ## GrepCounterOutput
12
+
13
+ Assume inputs from another plugin are as belows:
14
+
15
+ syslog.host1: {"message":"2013/01/13T07:02:11.124202 INFO GET /ping" }
16
+ syslog.host1: {"message":"2013/01/13T07:02:13.232645 WARN POST /auth" }
17
+ syslog.host1: {"message":"2013/01/13T07:02:21.542145 WARN GET /favicon.ico" }
18
+ syslog.host1: {"message":"2013/01/13T07:02:43.632145 WARN POST /login" }
19
+
20
+ An example of grepcounter configuration:
21
+
22
+ <match syslog.**>
23
+ type grepcounter
24
+ count_interval 60
25
+ input_key message
26
+ regexp WARN
27
+ exclude favicon.ico
28
+ threshold 1
29
+ add_tag_prefix warn.count
30
+ </source>
31
+
32
+ Outputs as belows:
33
+
34
+ warn.count.syslog.host1: {"count":2,"input_tag":"syslog.host1","input_tag_last":"host1"}
35
+
36
+ Another example of grepcounter configuration:
37
+
38
+ <match syslog.**>
39
+ type grepcounter
40
+ count_interval 60
41
+ input_key message
42
+ regexp WARN
43
+ exclude favicon.ico
44
+ threshold 1
45
+ add_tag_prefix warn.count
46
+ output_matched_message true
47
+ output_with_joined_delimiter \n
48
+ </source>
49
+
50
+ Outputs as belows:
51
+
52
+ warn.count.syslog.host1: {"count":2,"input_tag":"syslog.host1","input_tag_last":"host1","message":"2013/01/13T07:02:13.232645 WARN POST /auth\n2013/01/13T07:02:43.632145 WARN POST /login"}
53
+
54
+ ## Contributing
55
+
56
+ 1. Fork it
57
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
58
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
59
+ 4. Push to the branch (`git push origin my-new-feature`)
60
+ 5. Create new [Pull Request](../../pull/new/master)
61
+
62
+ ## Copyright
63
+
64
+ Copyright (c) 2013 Naotoshi SEO. See [LICENSE](LICENSE) for details.
@@ -0,0 +1,15 @@
1
+ # encoding: utf-8
2
+ require "bundler/gem_tasks"
3
+
4
+ require 'rspec/core'
5
+ require 'rspec/core/rake_task'
6
+ RSpec::Core::RakeTask.new(:spec) do |spec|
7
+ spec.pattern = FileList['spec/**/*_spec.rb']
8
+ end
9
+ task :default => :spec
10
+
11
+ desc 'Open an irb session preloaded with the gem library'
12
+ task :console do
13
+ sh 'irb -rubygems -I lib'
14
+ end
15
+ task :c => :console
@@ -0,0 +1,25 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+
4
+ Gem::Specification.new do |s|
5
+ s.name = "fluent-plugin-grepcounter"
6
+ s.version = "0.0.1"
7
+ s.authors = ["Naotoshi SEO"]
8
+ s.email = ["sonots@gmail.com"]
9
+ s.homepage = "https://github.com/sonots/fluent-plugin-grepcounter"
10
+ s.summary = "Count the number of matched messages"
11
+ s.description = s.summary
12
+
13
+ s.rubyforge_project = "fluent-plugin-grepcounter"
14
+
15
+ s.files = `git ls-files`.split("\n")
16
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
17
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
18
+ s.require_paths = ["lib"]
19
+
20
+ s.add_runtime_dependency "fluentd"
21
+ s.add_development_dependency "rake"
22
+ s.add_development_dependency "rspec"
23
+ s.add_development_dependency "pry"
24
+ # s.add_development_dependency "delorean"
25
+ end
@@ -0,0 +1,107 @@
1
+ class Fluent::GrepCounterOutput < Fluent::Output
2
+ Fluent::Plugin.register_output('grepcounter', self)
3
+
4
+ config_param :input_key, :string
5
+ config_param :regexp, :string
6
+ config_param :count_interval, :time, :default => 5
7
+ config_param :exclude, :string, :default => nil
8
+ config_param :threshold, :integer, :default => 1
9
+ config_param :output_tag, :string, :default => 'count'
10
+ config_param :add_tag_prefix, :string, :default => nil
11
+ config_param :output_matched_message, :bool, :default => false
12
+ config_param :output_with_joined_delimiter, :string, :default => nil
13
+
14
+ attr_accessor :matches
15
+ attr_accessor :last_checked
16
+
17
+ def configure(conf)
18
+ super
19
+
20
+ @count_interval = @count_interval.to_i
21
+ @input_key = @input_key.to_s
22
+ @regexp = Regexp.compile(@regexp) if @regexp
23
+ @exclude = Regexp.compile(@exclude) if @exclude
24
+ @threshold = @threshold.to_i
25
+
26
+ if @output_with_joined_delimiter and @output_matched_message == false
27
+ raise Fluent::ConfigError, "'output_matched_message' must be true to use 'output_with_joined_delimiter'"
28
+ end
29
+
30
+ @matches = {}
31
+ @counts = {}
32
+ @mutex = Mutex.new
33
+ end
34
+
35
+ def start
36
+ super
37
+ @watcher = Thread.new(&method(:watcher))
38
+ end
39
+
40
+ def shutdown
41
+ super
42
+ @watcher.terminate
43
+ @watcher.join
44
+ end
45
+
46
+ # Called when new line comes. This method actually does not emit
47
+ def emit(tag, es, chain)
48
+ count = 0; matches = []
49
+ # filter out and insert
50
+ es.each do |time,record|
51
+ value = record[@input_key]
52
+ next unless @regexp and @regexp.match(value)
53
+ next if @exclude and @exclude.match(value)
54
+ matches << value if @output_matched_message
55
+ count += 1
56
+ end
57
+ # thread safe merge
58
+ @counts[tag] ||= 0
59
+ @matches[tag] ||= []
60
+ @mutex.synchronize do
61
+ @counts[tag] += count
62
+ @matches[tag] += matches
63
+ end
64
+
65
+ chain.next
66
+ end
67
+
68
+ # thread callback
69
+ def watcher
70
+ # instance variable, and public accessable, for test
71
+ @last_checked = Fluent::Engine.now
72
+ while true
73
+ sleep 0.5
74
+ if Fluent::Engine.now - @last_checked >= @count_interval
75
+ now = Fluent::Engine.now
76
+ flush_emit(now - @last_checked)
77
+ @last_checked = now
78
+ end
79
+ end
80
+ end
81
+
82
+ # This method is the real one to emit
83
+ def flush_emit(step)
84
+ time = Fluent::Engine.now
85
+ flushed_counts, flushed_matches, @counts, @matches = @counts, @matches, {}, {}
86
+ flushed_counts.keys.each do |tag|
87
+ count = flushed_counts[tag]
88
+ matches = flushed_matches[tag]
89
+ output = generate_output(tag, count, matches)
90
+ tag = @add_tag_prefix ? "#{@add_tag_prefix}.#{tag}" : @output_tag
91
+ Fluent::Engine.emit(tag, time, output) if output
92
+ end
93
+ end
94
+
95
+ def generate_output(input_tag, count, matches)
96
+ return nil if count < @threshold
97
+ output = {}
98
+ output['count'] = count
99
+ output['input_tag'] = input_tag
100
+ output['input_tag_last'] = input_tag.split(".").last
101
+ if @output_matched_message
102
+ output['message'] = @output_with_joined_delimiter.nil? ? matches : matches.join(@output_with_joined_delimiter)
103
+ end
104
+ output
105
+ end
106
+
107
+ end
@@ -0,0 +1,174 @@
1
+ # encoding: UTF-8
2
+ require_relative 'spec_helper'
3
+
4
+ describe Fluent::GrepCounterOutput do
5
+ before { Fluent::Test.setup }
6
+ CONFIG = %[
7
+ input_key message
8
+ regexp WARN
9
+ ]
10
+ let(:tag) { 'syslog.host1' }
11
+ let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::GrepCounterOutput, tag).configure(config) }
12
+
13
+ describe 'test configure' do
14
+ describe 'bad configuration' do
15
+ context "lack of requirements" do
16
+ let(:config) { '' }
17
+ it { expect { driver }.to raise_error(Fluent::ConfigError) }
18
+ end
19
+
20
+ context "check output_with_joined_delimiter" do
21
+ let(:config) { CONFIG + %[ output_with_joined_delimiter \\n ] }
22
+ it { expect { driver }.to raise_error(Fluent::ConfigError) }
23
+ end
24
+ end
25
+
26
+ describe 'good configuration' do
27
+ subject { driver.instance }
28
+
29
+ context "check default" do
30
+ let(:config) { CONFIG }
31
+ its(:input_key) { should == "message" }
32
+ its(:count_interval) { should == 5 }
33
+ its(:regexp) { should == /WARN/ }
34
+ its(:exclude) { should be_nil }
35
+ its(:threshold) { should == 1 }
36
+ its(:output_tag) { should == 'count' }
37
+ its(:add_tag_prefix) { should be_nil }
38
+ its(:output_matched_message) { should be_false }
39
+ its(:output_with_joined_delimiter) { should be_nil }
40
+ end
41
+ end
42
+ end
43
+
44
+ describe 'test emit' do
45
+ let(:time) { Time.now.to_i }
46
+ let(:messages) do
47
+ [
48
+ "2013/01/13T07:02:11.124202 INFO GET /ping",
49
+ "2013/01/13T07:02:13.232645 WARN POST /auth",
50
+ "2013/01/13T07:02:21.542145 WARN GET /favicon.ico",
51
+ "2013/01/13T07:02:43.632145 WARN POST /login",
52
+ ]
53
+ end
54
+ let(:emit) do
55
+ driver.run { messages.each {|message| driver.emit({'message' => message}, time) } }
56
+ driver.instance.flush_emit(0)
57
+ end
58
+
59
+ context 'count_interval' do
60
+ pending
61
+ end
62
+
63
+ context 'regexp' do
64
+ let(:config) { CONFIG + %[ regexp WARN ] }
65
+ before do
66
+ Fluent::Engine.stub(:now).and_return(time)
67
+ Fluent::Engine.should_receive(:emit).with("count", time, {"count"=>3, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last})
68
+ end
69
+ it { emit }
70
+ end
71
+
72
+ context 'exclude' do
73
+ let(:config) do
74
+ CONFIG + %[
75
+ exclude favicon
76
+ ]
77
+ end
78
+ before do
79
+ Fluent::Engine.stub(:now).and_return(time)
80
+ Fluent::Engine.should_receive(:emit).with("count", time, {"count"=>2, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last})
81
+ end
82
+ it { emit }
83
+ end
84
+
85
+ context 'threshold (less than or equal to)' do
86
+ let(:config) do
87
+ CONFIG + %[
88
+ threshold 3
89
+ ]
90
+ end
91
+ before do
92
+ Fluent::Engine.stub(:now).and_return(time)
93
+ Fluent::Engine.should_receive(:emit).with("count", time, {"count"=>3, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last})
94
+ end
95
+ it { emit }
96
+ end
97
+
98
+ context 'threshold (greater)' do
99
+ let(:config) do
100
+ CONFIG + %[
101
+ threshold 4
102
+ ]
103
+ end
104
+ before do
105
+ Fluent::Engine.stub(:now).and_return(time)
106
+ Fluent::Engine.should_not_receive(:emit)
107
+ end
108
+ it { emit }
109
+ end
110
+
111
+ context 'output_tag' do
112
+ let(:config) do
113
+ CONFIG + %[
114
+ output_tag foo
115
+ ]
116
+ end
117
+ before do
118
+ Fluent::Engine.stub(:now).and_return(time)
119
+ Fluent::Engine.should_receive(:emit).with("foo", time, {"count"=>3, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last})
120
+ end
121
+ it { emit }
122
+ end
123
+
124
+ context 'add_tag_prefix' do
125
+ let(:config) do
126
+ CONFIG + %[
127
+ add_tag_prefix foo
128
+ ]
129
+ end
130
+ before do
131
+ Fluent::Engine.stub(:now).and_return(time)
132
+ Fluent::Engine.should_receive(:emit).with("foo.#{tag}", time, {"count"=>3, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last})
133
+ end
134
+ it { emit }
135
+ end
136
+
137
+ context 'output_matched_message' do
138
+ let(:config) do
139
+ CONFIG + %[
140
+ output_matched_message true
141
+ ]
142
+ end
143
+ before do
144
+ Fluent::Engine.stub(:now).and_return(time)
145
+ Fluent::Engine.should_receive(:emit).with("count", time, {
146
+ "count"=>3, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last,
147
+ "message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"]
148
+ })
149
+ end
150
+ it { emit }
151
+ end
152
+
153
+ context 'output_with_joined_delimiter' do
154
+ let(:config) do
155
+ # \\n shall be \n in config file
156
+ CONFIG + %[
157
+ output_matched_message true
158
+ output_with_joined_delimiter \\n
159
+ ]
160
+ end
161
+ before do
162
+ Fluent::Engine.stub(:now).and_return(time)
163
+ Fluent::Engine.should_receive(:emit).with("count", time, {
164
+ "count"=>3, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last,
165
+ "message"=>"2013/01/13T07:02:13.232645 WARN POST /auth\\n2013/01/13T07:02:21.542145 WARN GET /favicon.ico\\n2013/01/13T07:02:43.632145 WARN POST /login"
166
+ })
167
+ end
168
+ it { emit }
169
+ end
170
+ end
171
+ end
172
+
173
+
174
+
@@ -0,0 +1,14 @@
1
+ # encoding: UTF-8
2
+ require 'rubygems'
3
+ require 'bundler'
4
+ Bundler.setup(:default, :test)
5
+ Bundler.require(:default, :test)
6
+
7
+ require 'fluent/test'
8
+ require 'rspec'
9
+ require 'pry'
10
+ # require 'delorean'
11
+
12
+ $TESTING=true
13
+ $:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
14
+ require 'fluent/plugin/out_grepcounter'
metadata ADDED
@@ -0,0 +1,113 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-grepcounter
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Naotoshi SEO
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2013-03-11 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '>='
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - '>='
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - '>='
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - '>='
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: pry
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '>='
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ description: Count the number of matched messages
70
+ email:
71
+ - sonots@gmail.com
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - .gitignore
77
+ - .rdebugrc
78
+ - .rspec
79
+ - .travis.yml
80
+ - Gemfile
81
+ - LICENSE
82
+ - README.md
83
+ - Rakefile
84
+ - fluent-plugin-grepcounter.gemspec
85
+ - lib/fluent/plugin/out_grepcounter.rb
86
+ - spec/out_grepcounter_spec.rb
87
+ - spec/spec_helper.rb
88
+ homepage: https://github.com/sonots/fluent-plugin-grepcounter
89
+ licenses: []
90
+ metadata: {}
91
+ post_install_message:
92
+ rdoc_options: []
93
+ require_paths:
94
+ - lib
95
+ required_ruby_version: !ruby/object:Gem::Requirement
96
+ requirements:
97
+ - - '>='
98
+ - !ruby/object:Gem::Version
99
+ version: '0'
100
+ required_rubygems_version: !ruby/object:Gem::Requirement
101
+ requirements:
102
+ - - '>='
103
+ - !ruby/object:Gem::Version
104
+ version: '0'
105
+ requirements: []
106
+ rubyforge_project: fluent-plugin-grepcounter
107
+ rubygems_version: 2.0.0
108
+ signing_key:
109
+ specification_version: 4
110
+ summary: Count the number of matched messages
111
+ test_files:
112
+ - spec/out_grepcounter_spec.rb
113
+ - spec/spec_helper.rb