fluent-plugin-calc 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 3ef459d57b1a570eb06400f3eb8e6a4b54ed0323
4
+ data.tar.gz: 3f075999d044512d5504897e237dd7e8a07f57e6
5
+ SHA512:
6
+ metadata.gz: 94465d66d28c3ec8596b94c19e9da14c6ecd2624262dc9ce27038b19ca8ee50c529242bba14be09aba95ba7545627da935dd5e4fd6d07f7af44af3f8fefac1c1
7
+ data.tar.gz: 76c2f2d778128c2287111b086290c14f7cf05df08eac3f7d7aa39717061b0c9408749ddcf6103a58df9b6c3a641898135cb57d87c757a8ce4f568d2114364d5d
data/.coveralls.yml ADDED
@@ -0,0 +1 @@
1
+ repo_token: i4DJCtdksuIwhBck1tukIjzKoMCxWIIvQ
data/.gitignore ADDED
@@ -0,0 +1,13 @@
1
+ /*.gem
2
+ ~*
3
+ #*
4
+ *~
5
+ .bundle
6
+ Gemfile.lock
7
+ .rbenv-version
8
+ vendor
9
+ doc/*
10
+ tmp/*
11
+ coverage
12
+ .yardoc
13
+ pkg/*
data/.rdebugrc ADDED
@@ -0,0 +1,4 @@
1
+ set autolist
2
+ set autoeval
3
+ set autoreload
4
+
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --colour
2
+ --format documentation
data/.travis.yml ADDED
@@ -0,0 +1,6 @@
1
+ rvm:
2
+ - 1.9.2
3
+ - 1.9.3
4
+ - 2.0.0
5
+ gemfile:
6
+ - Gemfile
data/CHANGELOG.md ADDED
@@ -0,0 +1,3 @@
1
+ ## 0.0.1 (2013/05/05)
2
+
3
+ First version
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ gem 'simplecov', git: 'https://github.com/colszowka/simplecov.git'
4
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013 Naotoshi SEO
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,69 @@
1
+ # fluent-plugin-calc [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-calc.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-calc) [![Dependency Status](https://gemnasium.com/sonots/fluent-plugin-calc.png)](https://gemnasium.com/sonots/fluent-plugin-calc) [![Coverage Status](https://coveralls.io/repos/sonots/fluent-plugin-calc/badge.png?branch=master)](https://coveralls.io/r/sonots/fluent-plugin-calc)
2
+
3
+ Fluentd plugin to calc messages.
4
+
5
+ ## Configuration
6
+
7
+ <match foo.**>
8
+ type calc
9
+ interval 5s
10
+ aggragate tag
11
+ add_tag_prefix calc
12
+
13
+ sum .*_count$
14
+ max .*_max$
15
+ min .*_min$
16
+ avg .*_avg$
17
+ </match>
18
+
19
+ Assuming following inputs are coming:
20
+
21
+ foo.bar: {"4xx_count":1,"5xx_count":2","reqtime_max":12083,"reqtime_min":10,reqtime_avg":240.46}
22
+ foo.bar: {"4xx_count":4,"5xx_count":2","reqtime_max":24831,"reqtime_min":82,reqtime_avg":300.46}
23
+
24
+ then output bocomes as belows:
25
+
26
+ foo.bar: {"4xx_count":5,"5xx_count":4","reqtime_max":24831,"reqtime_min":82,reqtime_avg":270.46}
27
+
28
+ ## Parameters
29
+
30
+ - aggragate
31
+
32
+ Calculate by each `tag` or `all`. The default value is `tag`.
33
+
34
+ - interval
35
+
36
+ The interval to calculate in seconds. Default is 5s.
37
+
38
+ - tag
39
+
40
+ The output tag name
41
+
42
+ - add_tag_prefix
43
+
44
+ Add tag prefix for output message
45
+
46
+ - sum, min, max, avg
47
+
48
+ Calculation. Specify input keys by a regular expression
49
+
50
+ ## ChangeLog
51
+
52
+ See [CHANGELOG.md](CHANGELOG.md) for details.
53
+
54
+ ## ToDo
55
+
56
+ Get the number of denominator to calculate `avg` from input json field.
57
+
58
+ ## Contributing
59
+
60
+ 1. Fork it
61
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
62
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
63
+ 4. Push to the branch (`git push origin my-new-feature`)
64
+ 5. Create new [Pull Request](../../pull/new/master)
65
+
66
+ ## Copyright
67
+
68
+ Copyright (c) 2013 Naotoshi SEO. See [LICENSE](LICENSE) for details.
69
+
data/Rakefile ADDED
@@ -0,0 +1,15 @@
1
+ # encoding: utf-8
2
+ require "bundler/gem_tasks"
3
+
4
+ require 'rspec/core'
5
+ require 'rspec/core/rake_task'
6
+ RSpec::Core::RakeTask.new(:spec) do |spec|
7
+ spec.pattern = FileList['spec/**/*_spec.rb']
8
+ end
9
+ task :default => :spec
10
+
11
+ desc 'Open an irb session preloaded with the gem library'
12
+ task :console do
13
+ sh 'irb -rubygems -I lib'
14
+ end
15
+ task :c => :console
@@ -0,0 +1,25 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+
4
+ Gem::Specification.new do |s|
5
+ s.name = "fluent-plugin-calc"
6
+ s.version = "0.0.1"
7
+ s.authors = ["Naotoshi SEO"]
8
+ s.email = ["sonots@gmail.com"]
9
+ s.homepage = "https://github.com/sonots/fluent-plugin-calc"
10
+ s.summary = "fluentd plugin to calc messages"
11
+ s.description = s.summary
12
+
13
+ s.rubyforge_project = "fluent-plugin-calc"
14
+
15
+ s.files = `git ls-files`.split("\n")
16
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
17
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
18
+ s.require_paths = ["lib"]
19
+
20
+ s.add_runtime_dependency "fluentd"
21
+ s.add_development_dependency "rake"
22
+ s.add_development_dependency "rspec"
23
+ s.add_development_dependency "pry"
24
+ s.add_development_dependency 'coveralls'
25
+ end
@@ -0,0 +1,131 @@
1
+ # encoding: UTF-8
2
+ class Fluent::CalcOutput < Fluent::Output
3
+ Fluent::Plugin.register_output('calc', self)
4
+
5
+ config_param :sum, :string, :default => nil
6
+ config_param :max, :string, :default => nil
7
+ config_param :min, :string, :default => nil
8
+ config_param :avg, :string, :default => nil
9
+ config_param :interval, :time, :default => 5
10
+ config_param :tag, :string, :default => nil
11
+ config_param :add_tag_prefix, :string, :default => 'calc'
12
+ config_param :aggregate, :string, :default => 'tag'
13
+
14
+ attr_accessor :matches
15
+ attr_accessor :last_checked
16
+
17
+ def configure(conf)
18
+ super
19
+
20
+ @interval = @interval.to_i
21
+ @sum = Regexp.new(@sum) if @sum
22
+ @max = Regexp.new(@max) if @max
23
+ @min = Regexp.new(@min) if @min
24
+ @avg = Regexp.new(@avg) if @avg
25
+
26
+ unless ['tag', 'all'].include?(@aggregate)
27
+ raise Fluent::ConfigError, "aggregate allows tag/all"
28
+ end
29
+
30
+ case @aggregate
31
+ when 'all'
32
+ raise Fluent::ConfigError, "tag must be specified for aggregate all" if @tag.nil?
33
+ end
34
+
35
+ @counts = {}
36
+ @matches = {}
37
+ @mutex = Mutex.new
38
+ end
39
+
40
+ def start
41
+ super
42
+ @watcher = Thread.new(&method(:watcher))
43
+ end
44
+
45
+ def shutdown
46
+ super
47
+ @watcher.terminate
48
+ @watcher.join
49
+ end
50
+
51
+ # Called when new line comes. This method actually does not emit
52
+ def emit(tag, es, chain)
53
+ tag = 'all' if @aggregate == 'all'
54
+
55
+ # calc
56
+ count = 0; matches = {}
57
+ es.each do |time,record|
58
+ record.keys.each do |key|
59
+ if @sum and @sum.match(key)
60
+ matches[key] = (matches[key] ? matches[key] + record[key] : record[key])
61
+ elsif @max and @max.match(key)
62
+ matches[key] = (matches[key] ? [matches[key], record[key]].max : record[key])
63
+ elsif @min and @min.match(key)
64
+ matches[key] = (matches[key] ? [matches[key], record[key]].min : record[key])
65
+ elsif @avg and @avg.match(key)
66
+ matches[key] = (matches[key] ? matches[key] + record[key] : record[key]) # sum yet
67
+ end
68
+ end
69
+ count += 1
70
+ end
71
+
72
+ # thread safe merge
73
+ @counts[tag] ||= 0
74
+ @matches[tag] ||= {}
75
+ @mutex.synchronize do
76
+ matches.keys.each do |key|
77
+ if @sum and @sum.match(key)
78
+ @matches[tag][key] = (@matches[tag][key] ? @matches[tag][key] + matches[key] : matches[key])
79
+ elsif @max and @max.match(key)
80
+ @matches[tag][key] = (@matches[tag][key] ? [@matches[tag][key], matches[key]].max : matches[key])
81
+ elsif @min and @min.match(key)
82
+ @matches[tag][key] = (@matches[tag][key] ? [@matches[tag][key], matches[key]].min : matches[key])
83
+ elsif @avg and @avg.match(key)
84
+ @matches[tag][key] = (@matches[tag][key] ? @matches[tag][key] + matches[key] : matches[key]) # sum yet
85
+ end
86
+ end
87
+ @counts[tag] += count
88
+ end
89
+
90
+ chain.next
91
+ end
92
+
93
+ # thread callback
94
+ def watcher
95
+ # instance variable, and public accessable, for test
96
+ @last_checked = Fluent::Engine.now
97
+ while true
98
+ sleep 0.5
99
+ if Fluent::Engine.now - @last_checked >= @interval
100
+ now = Fluent::Engine.now
101
+ flush_emit(now - @last_checked)
102
+ @last_checked = now
103
+ end
104
+ end
105
+ end
106
+
107
+ # This method is the real one to emit
108
+ def flush_emit(step)
109
+ time = Fluent::Engine.now
110
+ flushed_counts, flushed_matches, @counts, @matches = @counts, @matches, {}, {}
111
+
112
+ flushed_counts.keys.each do |tag|
113
+ count = flushed_counts[tag]
114
+ matches = flushed_matches[tag]
115
+ output = generate_output(count, matches)
116
+ tag = @tag ? @tag : "#{@add_tag_prefix}.#{tag}"
117
+ Fluent::Engine.emit(tag, time, output) if output
118
+ end
119
+ end
120
+
121
+ def generate_output(count, matches)
122
+ output = matches.dup
123
+ output.keys.each do |key|
124
+ if @avg.match(key)
125
+ output[key] = matches[key] / count.to_f # compute avg
126
+ end
127
+ end
128
+ output
129
+ end
130
+
131
+ end
@@ -0,0 +1,145 @@
1
+ # encoding: UTF-8
2
+ require_relative 'spec_helper'
3
+
4
+ describe Fluent::CalcOutput do
5
+ before { Fluent::Test.setup }
6
+ CONFIG = %[
7
+ input_key message
8
+ ]
9
+ let(:tag) { 'foo.bar' }
10
+ let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::CalcOutput, tag).configure(config) }
11
+
12
+ describe 'test configure' do
13
+ describe 'bad configuration' do
14
+ context 'invalid aggregate' do
15
+ let(:config) do
16
+ CONFIG + %[
17
+ aggregate foo
18
+ ]
19
+ end
20
+ it { expect { driver }.to raise_error(Fluent::ConfigError) }
21
+ end
22
+
23
+ context 'no tag for aggregate all' do
24
+ let(:config) do
25
+ CONFIG + %[
26
+ aggregate all
27
+ ]
28
+ end
29
+ it { expect { driver }.to raise_error(Fluent::ConfigError) }
30
+ end
31
+ end
32
+
33
+ describe 'good configuration' do
34
+ context "nothing" do
35
+ let(:config) { '' }
36
+ it { expect { driver }.to_not raise_error(Fluent::ConfigError) }
37
+ end
38
+
39
+ context 'sum/max/min/avg' do
40
+ let(:config) do
41
+ CONFIG + %[
42
+ sum _count$
43
+ max _max$
44
+ min _min$
45
+ avg _avg$
46
+ ]
47
+ end
48
+ it { expect { driver }.to_not raise_error(Fluent::ConfigError) }
49
+ end
50
+
51
+ context "check default" do
52
+ subject { driver.instance }
53
+ let(:config) { CONFIG }
54
+ its(:interval) { should == 5 }
55
+ its(:tag) { should be_nil }
56
+ its(:add_tag_prefix) { should == 'calc' }
57
+ its(:aggregate) { should == 'tag' }
58
+ end
59
+ end
60
+ end
61
+
62
+ describe 'test emit' do
63
+ let(:time) { Time.now.to_i }
64
+ let(:messages) do
65
+ [
66
+ {"4xx_count"=>1,"5xx_count"=>2,"reqtime_max"=>6,"reqtime_min"=>1,"reqtime_avg"=>3},
67
+ {"4xx_count"=>2,"5xx_count"=>2,"reqtime_max"=>5,"reqtime_min"=>2,"reqtime_avg"=>2},
68
+ {"4xx_count"=>3,"5xx_count"=>2,"reqtime_max"=>1,"reqtime_min"=>3,"reqtime_avg"=>4},
69
+ ]
70
+ end
71
+ let(:emit) do
72
+ driver.run { messages.each {|message| driver.emit(message, time) } }
73
+ driver.instance.flush_emit(0)
74
+ end
75
+
76
+ context 'interval' do
77
+ pending
78
+ end
79
+
80
+ context 'sum/max/min/avg' do
81
+ let(:config) do
82
+ CONFIG + %[
83
+ sum _count$
84
+ max _max$
85
+ min _min$
86
+ avg _avg$
87
+ ]
88
+ end
89
+ before do
90
+ Fluent::Engine.stub(:now).and_return(time)
91
+ Fluent::Engine.should_receive(:emit).with("calc.#{tag}", time, {
92
+ "4xx_count"=>6,"5xx_count"=>6,"reqtime_max"=>6,"reqtime_min"=>1,"reqtime_avg"=>3.0
93
+ })
94
+ end
95
+ it { emit }
96
+ end
97
+
98
+ context 'tag' do
99
+ let(:config) do
100
+ CONFIG + %[
101
+ tag foo
102
+ ]
103
+ end
104
+ before do
105
+ Fluent::Engine.stub(:now).and_return(time)
106
+ Fluent::Engine.should_receive(:emit).with("foo", time, {})
107
+ end
108
+ it { emit }
109
+ end
110
+
111
+ context 'add_tag_prefix' do
112
+ let(:config) do
113
+ CONFIG + %[
114
+ add_tag_prefix foo
115
+ ]
116
+ end
117
+ before do
118
+ Fluent::Engine.stub(:now).and_return(time)
119
+ Fluent::Engine.should_receive(:emit).with("foo.#{tag}", time, {})
120
+ end
121
+ it { emit }
122
+ end
123
+
124
+ context 'aggregate all' do
125
+ let(:config) do
126
+ CONFIG + %[
127
+ aggregate all
128
+ tag foo
129
+ sum _count$
130
+ max _max$
131
+ min _min$
132
+ avg _avg$
133
+ ]
134
+ end
135
+ before do
136
+ Fluent::Engine.stub(:now).and_return(time)
137
+ Fluent::Engine.should_receive(:emit).with("foo", time, {
138
+ "4xx_count"=>6,"5xx_count"=>6,"reqtime_max"=>6,"reqtime_min"=>1,"reqtime_avg"=>3.0
139
+ })
140
+ end
141
+ it { emit }
142
+ end
143
+ end
144
+ end
145
+
@@ -0,0 +1,16 @@
1
+ # encoding: UTF-8
2
+ require 'rubygems'
3
+ require 'bundler'
4
+ Bundler.setup(:default, :test)
5
+ Bundler.require(:default, :test)
6
+
7
+ require 'coveralls'
8
+ Coveralls.wear!
9
+
10
+ require 'fluent/test'
11
+ require 'rspec'
12
+ require 'pry'
13
+
14
+ $TESTING=true
15
+ $:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
16
+ require 'fluent/plugin/out_calc'
metadata ADDED
@@ -0,0 +1,129 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-calc
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Naotoshi SEO
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2013-05-05 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '>='
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - '>='
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - '>='
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - '>='
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: pry
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '>='
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: coveralls
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - '>='
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - '>='
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ description: fluentd plugin to calc messages
84
+ email:
85
+ - sonots@gmail.com
86
+ executables: []
87
+ extensions: []
88
+ extra_rdoc_files: []
89
+ files:
90
+ - .coveralls.yml
91
+ - .gitignore
92
+ - .rdebugrc
93
+ - .rspec
94
+ - .travis.yml
95
+ - CHANGELOG.md
96
+ - Gemfile
97
+ - LICENSE
98
+ - README.md
99
+ - Rakefile
100
+ - fluent-plugin-calc.gemspec
101
+ - lib/fluent/plugin/out_calc.rb
102
+ - spec/out_calc_spec.rb
103
+ - spec/spec_helper.rb
104
+ homepage: https://github.com/sonots/fluent-plugin-calc
105
+ licenses: []
106
+ metadata: {}
107
+ post_install_message:
108
+ rdoc_options: []
109
+ require_paths:
110
+ - lib
111
+ required_ruby_version: !ruby/object:Gem::Requirement
112
+ requirements:
113
+ - - '>='
114
+ - !ruby/object:Gem::Version
115
+ version: '0'
116
+ required_rubygems_version: !ruby/object:Gem::Requirement
117
+ requirements:
118
+ - - '>='
119
+ - !ruby/object:Gem::Version
120
+ version: '0'
121
+ requirements: []
122
+ rubyforge_project: fluent-plugin-calc
123
+ rubygems_version: 2.0.0
124
+ signing_key:
125
+ specification_version: 4
126
+ summary: fluentd plugin to calc messages
127
+ test_files:
128
+ - spec/out_calc_spec.rb
129
+ - spec/spec_helper.rb