fluent-plugin-sampling-filter 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.2
|
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "fluent-plugin-sampling-filter"
|
8
|
-
s.version = "0.1.
|
8
|
+
s.version = "0.1.2"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["TAGOMORI Satoshi"]
|
12
|
-
s.date = "2012-
|
12
|
+
s.date = "2012-08-08"
|
13
13
|
s.description = "fluentd plugin to pickup sample data from matched massages"
|
14
14
|
s.email = "tagomoris@gmail.com"
|
15
15
|
s.extra_rdoc_files = [
|
@@ -31,7 +31,7 @@ Gem::Specification.new do |s|
|
|
31
31
|
]
|
32
32
|
s.homepage = "http://github.com/tagomoris/fluent-plugin-sampling-filter"
|
33
33
|
s.require_paths = ["lib"]
|
34
|
-
s.rubygems_version = "1.8.
|
34
|
+
s.rubygems_version = "1.8.21"
|
35
35
|
s.summary = "fluentd plugin to pickup sample data from matched massages"
|
36
36
|
|
37
37
|
if s.respond_to? :specification_version then
|
@@ -5,6 +5,7 @@ class Fluent::SamplingFilterOutput < Fluent::Output
|
|
5
5
|
config_param :sample_unit, :string, :default => 'tag'
|
6
6
|
config_param :remove_prefix, :string, :default => nil
|
7
7
|
config_param :add_prefix, :string, :default => 'sampled'
|
8
|
+
config_param :minimum_rate_per_min, :integer, :default => nil
|
8
9
|
|
9
10
|
def configure(conf)
|
10
11
|
super
|
@@ -24,6 +25,7 @@ class Fluent::SamplingFilterOutput < Fluent::Output
|
|
24
25
|
raise Fluent::ConfigError, "sample_unit allows only 'tag' or 'all'"
|
25
26
|
end
|
26
27
|
@counts = {}
|
28
|
+
@resets = {} if @minimum_rate_per_min
|
27
29
|
end
|
28
30
|
|
29
31
|
def emit_sampled(tag, time_record_pairs)
|
@@ -48,19 +50,38 @@ class Fluent::SamplingFilterOutput < Fluent::Output
|
|
48
50
|
else
|
49
51
|
tag
|
50
52
|
end
|
53
|
+
|
54
|
+
pairs = []
|
55
|
+
|
51
56
|
# Access to @counts SHOULD be protected by mutex, with a heavy penalty.
|
52
57
|
# Code below is not thread safe, but @counts (counter for sampling rate) is not
|
53
|
-
# so serious value (and probably will not be
|
58
|
+
# so serious value (and probably will not be broken...),
|
54
59
|
# then i let here as it is now.
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
if c % @interval == 0
|
59
|
-
pairs.push [time, record]
|
60
|
-
# reset only just before @counts[t] is to be bignum from fixnum
|
61
|
-
@counts[t] = 0 if c > 0x6fffffff
|
60
|
+
if @minimum_rate_per_min
|
61
|
+
unless @resets[t]
|
62
|
+
@resets[t] = Fluent::Engine.now + (60 - rand(30))
|
62
63
|
end
|
63
|
-
|
64
|
+
if Fluent::Engine.now > @resets[t]
|
65
|
+
@resets[t] = Fluent::Engine.now + 60
|
66
|
+
@counts[t] = 0
|
67
|
+
end
|
68
|
+
es.each do |time,record|
|
69
|
+
c = (@counts[t] = @counts.fetch(t, 0) + 1)
|
70
|
+
if c < @minimum_rate_per_min or c % @interval == 0
|
71
|
+
pairs.push [time, record]
|
72
|
+
end
|
73
|
+
end
|
74
|
+
else
|
75
|
+
es.each do |time,record|
|
76
|
+
c = (@counts[t] = @counts.fetch(t, 0) + 1)
|
77
|
+
if c % @interval == 0
|
78
|
+
pairs.push [time, record]
|
79
|
+
# reset only just before @counts[t] is to be bignum from fixnum
|
80
|
+
@counts[t] = 0 if c > 0x6fffffff
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
64
85
|
emit_sampled(tag, pairs)
|
65
86
|
|
66
87
|
chain.next
|
@@ -41,12 +41,6 @@ class SamplingFilterOutputTest < Test::Unit::TestCase
|
|
41
41
|
assert_equal 'output', d.instance.add_prefix
|
42
42
|
end
|
43
43
|
|
44
|
-
# CONFIG = %[
|
45
|
-
# interval 10
|
46
|
-
# sample_unit tag
|
47
|
-
# remove_prefix input
|
48
|
-
# add_prefix sampled
|
49
|
-
# ]
|
50
44
|
def test_emit
|
51
45
|
d1 = create_driver(CONFIG, 'input.hoge1')
|
52
46
|
time = Time.parse("2012-01-02 13:14:15").to_i
|
@@ -97,4 +91,57 @@ class SamplingFilterOutputTest < Test::Unit::TestCase
|
|
97
91
|
assert_equal 'record9', emits[2][2]['field1']
|
98
92
|
assert_equal 'record12', emits[3][2]['field1']
|
99
93
|
end
|
94
|
+
|
95
|
+
def test_minimum_rate
|
96
|
+
config = %[
|
97
|
+
interval 10
|
98
|
+
sample_unit tag
|
99
|
+
remove_prefix input
|
100
|
+
minimum_rate_per_min 100
|
101
|
+
]
|
102
|
+
d = create_driver(config, 'input.hoge3')
|
103
|
+
time = Time.parse("2012-01-02 13:14:15").to_i
|
104
|
+
d.run do
|
105
|
+
(1..100).each do |t|
|
106
|
+
d.emit({'times' => t, 'data' => 'x'})
|
107
|
+
end
|
108
|
+
(101..130).each do |t|
|
109
|
+
d.emit({'times' => t, 'data' => 'y'})
|
110
|
+
end
|
111
|
+
end
|
112
|
+
emits = d.emits
|
113
|
+
assert_equal 103, emits.length
|
114
|
+
assert_equal 'sampled.hoge3', emits[0][0]
|
115
|
+
assert_equal ((1..100).map(&:to_i) + [110, 120, 130]), emits.map{|t,time,r| r['times']}
|
116
|
+
assert_equal (['x']*100 + ['y']*3), emits.map{|t,time,r| r['data']}
|
117
|
+
|
118
|
+
end
|
119
|
+
def test_minimum_rate_expire
|
120
|
+
# hey, this test needs 60 seconds....
|
121
|
+
assert_equal 1, 1
|
122
|
+
return
|
123
|
+
|
124
|
+
config = %[
|
125
|
+
interval 10
|
126
|
+
sample_unit tag
|
127
|
+
remove_prefix input
|
128
|
+
minimum_rate_per_min 10
|
129
|
+
]
|
130
|
+
d = create_driver(config, 'input.hoge4')
|
131
|
+
time = Time.parse("2012-01-02 13:14:15").to_i
|
132
|
+
d.run do
|
133
|
+
(1..100).each do |t|
|
134
|
+
d.emit({'times' => t, 'data' => 'x'})
|
135
|
+
end
|
136
|
+
sleep 60
|
137
|
+
(101..130).each do |t|
|
138
|
+
d.emit({'times' => t, 'data' => 'y'})
|
139
|
+
end
|
140
|
+
end
|
141
|
+
emits = d.emits
|
142
|
+
# assert_equal (19 + 12), emits.length
|
143
|
+
assert_equal 'sampled.hoge4', emits[0][0]
|
144
|
+
assert_equal ((1..10).map(&:to_i)+[20,30,40,50,60,70,80,90,100]+(101..110).map(&:to_i)+[120,130]), emits.map{|t,time,r| r['times']}
|
145
|
+
assert_equal (['x']*19 + ['y']*12), emits.map{|t,time,r| r['data']}
|
146
|
+
end
|
100
147
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sampling-filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-08-08 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -140,7 +140,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
140
140
|
version: '0'
|
141
141
|
segments:
|
142
142
|
- 0
|
143
|
-
hash:
|
143
|
+
hash: 2658451737493219281
|
144
144
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
145
145
|
none: false
|
146
146
|
requirements:
|
@@ -149,7 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
149
149
|
version: '0'
|
150
150
|
requirements: []
|
151
151
|
rubyforge_project:
|
152
|
-
rubygems_version: 1.8.
|
152
|
+
rubygems_version: 1.8.21
|
153
153
|
signing_key:
|
154
154
|
specification_version: 3
|
155
155
|
summary: fluentd plugin to pickup sample data from matched massages
|