fluent-plugin-lossycount 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/.gitignore +3 -0
- data/AUTHORS +1 -0
- data/Gemfile +3 -0
- data/README.rdoc +30 -0
- data/Rakefile +11 -0
- data/fluent-plugin-lossycount.gemspec +19 -0
- data/lib/fluent/plugin/out_lossycount.rb +195 -0
- data/test/counter/test_lossy_counter.rb +73 -0
- data/test/helper.rb +20 -0
- metadata +96 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
MjA2OWZjMWIwY2QxZTE0OTI2NGMzODkxMjJiYjg4OGQ2Mzg0YjZhYQ==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
MmFiMTk3Nzk0ODZlZTY3YTViMmQ4NjU1MWQ0ZTk1ZjhiYzEwYjRkZA==
|
7
|
+
!binary "U0hBNTEy":
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
ZGZiM2M2Y2MzZjk4NDk2NTU4ZjMwZmQyYzMxMzhiNWEwMTViNGNkZTAzZTYx
|
10
|
+
ZjViNjRkZTExNDY2YzMzNmI1Y2EzODIyMGIwNTFjNTFiZmZhYTQzZjE1YTE4
|
11
|
+
ZWY3MDk0ZmYyZDUzNjI4MTEyMTc2MDRiZWM1OWJhYTVlYzYwYTA=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
Y2EyMzg0ZjcwYzQ3ZWMzNmZlNTZhMTZjMDI1MTEyZWVlZGJlMjYyZTY0MWU1
|
14
|
+
OTMzNzQ1NTE2MDk3YWU2ZmViMzlmNThmYzk0MjQzYjNhZTdkZDg3ODAyMWM1
|
15
|
+
NzM5YmE3OGFmY2U2YWFhNjFiMmYzZTVhNTExNmEwOWM0MjRiN2M=
|
data/.gitignore
ADDED
data/AUTHORS
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
moaikids
|
data/Gemfile
ADDED
data/README.rdoc
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
= Lossy count output plugin for Fluentd
|
2
|
+
|
3
|
+
== Overview
|
4
|
+
|
5
|
+
*lossycount* output plugin is a counter logic.
|
6
|
+
|
7
|
+
== Installation
|
8
|
+
|
9
|
+
== Configuration
|
10
|
+
|
11
|
+
<match pattern>
|
12
|
+
type lossycount
|
13
|
+
gamma LOSSY_COUNT_GAMMA
|
14
|
+
epsilon LOSSY_COUNT_EPSILON
|
15
|
+
key_name INPUT_KEY_NAME
|
16
|
+
time_windows SECONDS
|
17
|
+
output_tag OUTPUT_TAG_NAME
|
18
|
+
output_key_name OUTPUT_KEY_NAME
|
19
|
+
output_timestamp_name OUTPUT_TIMESTAMP_NAME
|
20
|
+
output_value_name OUTPUT_VALUE_NAME
|
21
|
+
enable_metrics true
|
22
|
+
metrics_tag METRICS_OUTPUT_TAG_NAME
|
23
|
+
verbose false
|
24
|
+
</match>
|
25
|
+
|
26
|
+
== Copyright
|
27
|
+
|
28
|
+
Copyright:: Copyright (c) 2013 moaikids
|
29
|
+
License:: Apache License, Version 2.0
|
30
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
Gem::Specification.new do |gem|
|
3
|
+
gem.name = "fluent-plugin-lossycount"
|
4
|
+
gem.version = "0.0.1"
|
5
|
+
gem.authors = ["moaikids"]
|
6
|
+
gem.licenses = ["Apache License Version 2.0"]
|
7
|
+
gem.summary = %q{Lossy count output plugin for Fluentd}
|
8
|
+
gem.description = %q{Lossy count output plugin for Fluentd...}
|
9
|
+
gem.homepage = "https://github.com/moaikids/fluent-plugin-lossycount"
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.require_paths = ["lib"]
|
15
|
+
|
16
|
+
gem.add_development_dependency "rake"
|
17
|
+
gem.add_development_dependency "shoulda"
|
18
|
+
gem.add_runtime_dependency "fluentd"
|
19
|
+
end
|
@@ -0,0 +1,195 @@
|
|
1
|
+
class Fluent::LossyCountOutput < Fluent::Output
|
2
|
+
Fluent::Plugin.register_output('lossycount', self)
|
3
|
+
|
4
|
+
config_param :key_name, :string, :default => nil
|
5
|
+
config_param :time_windows, :time, :default => 60
|
6
|
+
config_param :output_tag, :string, :default => nil
|
7
|
+
config_param :output_key_name, :string, :default => 'key'
|
8
|
+
config_param :output_timestamp_name, :string, :default => 'timestamp'
|
9
|
+
config_param :output_value_name, :string, :default => 'value'
|
10
|
+
config_param :gamma, :float, :default => 0.005
|
11
|
+
config_param :epsilon, :float, :default => 0.004
|
12
|
+
config_param :enable_metrics, :bool, :default => false
|
13
|
+
config_param :metrics_tag, :string, :default => 'counter.metrics'
|
14
|
+
config_param :verbose, :bool, :default => false
|
15
|
+
|
16
|
+
def configure(config)
|
17
|
+
super
|
18
|
+
unless config.has_key?('key_name')
|
19
|
+
raise Fluent::ConfigError, "you must set 'key_name'"
|
20
|
+
end
|
21
|
+
unless config.has_key?('output_tag')
|
22
|
+
raise Fluent::ConfigError, "you must set 'output_tag'"
|
23
|
+
end
|
24
|
+
|
25
|
+
@mutex = Mutex.new
|
26
|
+
@sleep_wait = 0.4
|
27
|
+
init_counter()
|
28
|
+
end
|
29
|
+
|
30
|
+
def start
|
31
|
+
super
|
32
|
+
start_watch
|
33
|
+
end
|
34
|
+
|
35
|
+
def shutdown
|
36
|
+
super
|
37
|
+
@watcher.terminate
|
38
|
+
@watcher.join
|
39
|
+
end
|
40
|
+
|
41
|
+
def init_counter()
|
42
|
+
@counter = Counter::LossyCounter.new({:gamma => @gamma, :epsilon => @epsilon})
|
43
|
+
end
|
44
|
+
|
45
|
+
def flush_counter(step)
|
46
|
+
freq_counter = {}
|
47
|
+
metrics = {}
|
48
|
+
@mutex.synchronize {
|
49
|
+
freq_counter = @counter.get()
|
50
|
+
if @enable_metrics
|
51
|
+
metrics = @counter.get_metrics()
|
52
|
+
end
|
53
|
+
init_counter()
|
54
|
+
}
|
55
|
+
flush_time = Fluent::Engine.now.to_i
|
56
|
+
if @verbose
|
57
|
+
$log.info "flushtime : " + flush_time.to_s
|
58
|
+
$log.info "{ "
|
59
|
+
end
|
60
|
+
freq_counter.each_pair { |key, value|
|
61
|
+
map = {@output_key_name => key, @output_timestamp_name => flush_time, @output_value_name => value}
|
62
|
+
if @verbose
|
63
|
+
$log.info map
|
64
|
+
end
|
65
|
+
Fluent::Engine.emit(@output_tag, Fluent::Engine.now, map)
|
66
|
+
}
|
67
|
+
if @verbose
|
68
|
+
$log.info "}"
|
69
|
+
end
|
70
|
+
|
71
|
+
if @enable_metrics
|
72
|
+
if @verbose
|
73
|
+
$log.info "metrics : " + metrics.to_s
|
74
|
+
end
|
75
|
+
Fluent::Engine.emit(@metrics_tag, Fluent::Engine.now, metrics)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def start_watch
|
80
|
+
@watcher = Thread.new{
|
81
|
+
@last_checked = Fluent::Engine.now.to_i
|
82
|
+
while true
|
83
|
+
sleep @sleep_wait
|
84
|
+
now = Fluent::Engine.now.to_i
|
85
|
+
if now - @last_checked >= @time_windows
|
86
|
+
flush_counter(now - @last_checked)
|
87
|
+
@last_checked = now
|
88
|
+
end
|
89
|
+
end
|
90
|
+
}
|
91
|
+
end
|
92
|
+
|
93
|
+
def emit(tag, es, chain)
|
94
|
+
es.each {|time,record|
|
95
|
+
k = traverse(record, @key_name)
|
96
|
+
if k
|
97
|
+
if k.is_a?(Array)
|
98
|
+
k.each{ |v|
|
99
|
+
@counter.add(v.to_s)
|
100
|
+
}
|
101
|
+
else
|
102
|
+
@counter.add(k.to_s)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
}
|
106
|
+
chain.next
|
107
|
+
end
|
108
|
+
|
109
|
+
def traverse(data, key)
|
110
|
+
val = data
|
111
|
+
key.split('.').each{ |k|
|
112
|
+
if val.has_key?(k)
|
113
|
+
val = val[k]
|
114
|
+
else
|
115
|
+
return nil
|
116
|
+
end
|
117
|
+
}
|
118
|
+
return val
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
|
123
|
+
module Counter
|
124
|
+
class LossyCounter
|
125
|
+
def initialize(config)
|
126
|
+
@gamma = config.has_key?(:gamma) ? config[:gamma].to_f : 0.005
|
127
|
+
@epsilon = config.has_key?(:epsilon) ? config[:epsilon].to_f : 0.001
|
128
|
+
@current = 1
|
129
|
+
@freq_counter = {}
|
130
|
+
@delta_counter = {}
|
131
|
+
@num = 0
|
132
|
+
@max_size = -1
|
133
|
+
end
|
134
|
+
|
135
|
+
def add(key)
|
136
|
+
if @freq_counter.has_key?(key)
|
137
|
+
@freq_counter[key] += 1
|
138
|
+
else
|
139
|
+
@freq_counter[key] = 1
|
140
|
+
@delta_counter[key] = @current - 1
|
141
|
+
end
|
142
|
+
@num += 1
|
143
|
+
if @num % (1 / @epsilon).to_i == 0
|
144
|
+
sweep()
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
def sweep()
|
149
|
+
length = @freq_counter.length
|
150
|
+
if @max_size < length
|
151
|
+
@max_size = length
|
152
|
+
end
|
153
|
+
|
154
|
+
@freq_counter.each_pair { |key, value|
|
155
|
+
if value <= (@current - @delta_counter[key])
|
156
|
+
@freq_counter.delete(key)
|
157
|
+
@delta_counter.delete(key)
|
158
|
+
end
|
159
|
+
}
|
160
|
+
@current += 1
|
161
|
+
end
|
162
|
+
|
163
|
+
def get()
|
164
|
+
buf = {}
|
165
|
+
@freq_counter.each_pair { |key, value|
|
166
|
+
if value > (@num * (@gamma - @epsilon) ).to_i
|
167
|
+
buf[key] = value
|
168
|
+
end
|
169
|
+
}
|
170
|
+
return buf
|
171
|
+
end
|
172
|
+
|
173
|
+
def get_num()
|
174
|
+
return @num
|
175
|
+
end
|
176
|
+
|
177
|
+
def get_current_max_size()
|
178
|
+
return @max_size
|
179
|
+
end
|
180
|
+
|
181
|
+
def get_num_x_gamma()
|
182
|
+
return @num.to_f * @gamma
|
183
|
+
end
|
184
|
+
|
185
|
+
def get_num_x_gamma_d_epsilon()
|
186
|
+
return @num.to_f * (@gamma - @epsilon)
|
187
|
+
end
|
188
|
+
|
189
|
+
def get_metrics()
|
190
|
+
return {'num' => get_num(), 'max_size' => get_current_max_size(), 'current_size' => @freq_counter.size() , 'reduced_size' => get().size(), "gamma" => @gamma , "epsilon" => @epsilon , "n_x_gamma" => get_num_x_gamma(), "n_x_gamma-epsilon" => get_num_x_gamma_d_epsilon() }
|
191
|
+
end
|
192
|
+
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
@@ -0,0 +1,73 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class LossyCounterTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
@gamma = 0.005
|
6
|
+
@epsilon = 0.0045
|
7
|
+
end
|
8
|
+
|
9
|
+
def test_add_naive()
|
10
|
+
counter = Counter::LossyCounter.new({:gamma => @gamma, :epsilon => @epsilon})
|
11
|
+
counter.add('a')
|
12
|
+
counter.add('b')
|
13
|
+
counter.add('c')
|
14
|
+
counter.add('b')
|
15
|
+
counter.add('a')
|
16
|
+
counter.add('b')
|
17
|
+
|
18
|
+
freq_counter = counter.get()
|
19
|
+
|
20
|
+
p freq_counter
|
21
|
+
assert_equal freq_counter.has_key?('a') , true
|
22
|
+
assert_equal freq_counter.has_key?('b') , true
|
23
|
+
assert_equal freq_counter.has_key?('c') , true
|
24
|
+
assert_equal freq_counter.has_key?('d') , false
|
25
|
+
assert_equal freq_counter['a'] , 2
|
26
|
+
assert_equal freq_counter['b'] , 3
|
27
|
+
assert_equal freq_counter['c'] , 1
|
28
|
+
assert_equal freq_counter['d'] , nil
|
29
|
+
|
30
|
+
p counter.get_metrics()
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_add_and_sweep()
|
34
|
+
counter = Counter::LossyCounter.new({:gamma => @gamma, :epsilon => @epsilon})
|
35
|
+
buf = {}
|
36
|
+
radix = [10000000, 1000000, 100000, 10000, 1000, 100, 10]
|
37
|
+
l = 100000
|
38
|
+
|
39
|
+
(0..(l - 1)).step(1) do |i|
|
40
|
+
now = ((Time.now.to_f * 10000000) % 10000000).to_i
|
41
|
+
for r in radix
|
42
|
+
key = now & r
|
43
|
+
counter.add(key.to_s)
|
44
|
+
if buf.has_key?(key.to_s)
|
45
|
+
buf[key] += 1
|
46
|
+
else
|
47
|
+
buf[key] = 1
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
n = counter.get_num()
|
53
|
+
g = counter.get_num_x_gamma().to_i
|
54
|
+
e = (n * @epsilon).to_i
|
55
|
+
ge = counter.get_num_x_gamma_d_epsilon().to_i
|
56
|
+
freq_count = counter.get()
|
57
|
+
|
58
|
+
p counter.get_metrics()
|
59
|
+
|
60
|
+
buf.each_pair { |key, value|
|
61
|
+
if freq_count.has_key?(key)
|
62
|
+
if freq_count[key] >= g.to_i
|
63
|
+
assert_equal value , freq_count[key]
|
64
|
+
else
|
65
|
+
assert_operator e.to_i , :>= , (freq_count[key] - value).abs
|
66
|
+
end
|
67
|
+
else
|
68
|
+
assert_operator e.to_i , :>= , value
|
69
|
+
end
|
70
|
+
}
|
71
|
+
|
72
|
+
end
|
73
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
|
11
|
+
require 'test/unit'
|
12
|
+
require 'shoulda'
|
13
|
+
|
14
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
15
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
16
|
+
require 'fluent/test'
|
17
|
+
require 'fluent/plugin/out_lossycount'
|
18
|
+
|
19
|
+
class Test::Unit::TestCase
|
20
|
+
end
|
metadata
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-lossycount
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- moaikids
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-08-31 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ! '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ! '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: shoulda
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ! '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ! '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: fluentd
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ! '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description: Lossy count output plugin for Fluentd...
|
56
|
+
email:
|
57
|
+
executables: []
|
58
|
+
extensions: []
|
59
|
+
extra_rdoc_files: []
|
60
|
+
files:
|
61
|
+
- .gitignore
|
62
|
+
- AUTHORS
|
63
|
+
- Gemfile
|
64
|
+
- README.rdoc
|
65
|
+
- Rakefile
|
66
|
+
- fluent-plugin-lossycount.gemspec
|
67
|
+
- lib/fluent/plugin/out_lossycount.rb
|
68
|
+
- test/counter/test_lossy_counter.rb
|
69
|
+
- test/helper.rb
|
70
|
+
homepage: https://github.com/moaikids/fluent-plugin-lossycount
|
71
|
+
licenses:
|
72
|
+
- Apache License Version 2.0
|
73
|
+
metadata: {}
|
74
|
+
post_install_message:
|
75
|
+
rdoc_options: []
|
76
|
+
require_paths:
|
77
|
+
- lib
|
78
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ! '>='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
84
|
+
requirements:
|
85
|
+
- - ! '>='
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
version: '0'
|
88
|
+
requirements: []
|
89
|
+
rubyforge_project:
|
90
|
+
rubygems_version: 2.0.7
|
91
|
+
signing_key:
|
92
|
+
specification_version: 4
|
93
|
+
summary: Lossy count output plugin for Fluentd
|
94
|
+
test_files:
|
95
|
+
- test/counter/test_lossy_counter.rb
|
96
|
+
- test/helper.rb
|