fluent-plugin-sampling-filter 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.1
1
+ 0.1.2
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = "fluent-plugin-sampling-filter"
8
- s.version = "0.1.1"
8
+ s.version = "0.1.2"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["TAGOMORI Satoshi"]
12
- s.date = "2012-03-29"
12
+ s.date = "2012-08-08"
13
13
  s.description = "fluentd plugin to pickup sample data from matched massages"
14
14
  s.email = "tagomoris@gmail.com"
15
15
  s.extra_rdoc_files = [
@@ -31,7 +31,7 @@ Gem::Specification.new do |s|
31
31
  ]
32
32
  s.homepage = "http://github.com/tagomoris/fluent-plugin-sampling-filter"
33
33
  s.require_paths = ["lib"]
34
- s.rubygems_version = "1.8.19"
34
+ s.rubygems_version = "1.8.21"
35
35
  s.summary = "fluentd plugin to pickup sample data from matched massages"
36
36
 
37
37
  if s.respond_to? :specification_version then
@@ -5,6 +5,7 @@ class Fluent::SamplingFilterOutput < Fluent::Output
5
5
  config_param :sample_unit, :string, :default => 'tag'
6
6
  config_param :remove_prefix, :string, :default => nil
7
7
  config_param :add_prefix, :string, :default => 'sampled'
8
+ config_param :minimum_rate_per_min, :integer, :default => nil
8
9
 
9
10
  def configure(conf)
10
11
  super
@@ -24,6 +25,7 @@ class Fluent::SamplingFilterOutput < Fluent::Output
24
25
  raise Fluent::ConfigError, "sample_unit allows only 'tag' or 'all'"
25
26
  end
26
27
  @counts = {}
28
+ @resets = {} if @minimum_rate_per_min
27
29
  end
28
30
 
29
31
  def emit_sampled(tag, time_record_pairs)
@@ -48,19 +50,38 @@ class Fluent::SamplingFilterOutput < Fluent::Output
48
50
  else
49
51
  tag
50
52
  end
53
+
54
+ pairs = []
55
+
51
56
  # Access to @counts SHOULD be protected by mutex, with a heavy penalty.
52
57
  # Code below is not thread safe, but @counts (counter for sampling rate) is not
53
- # so serious value (and probably will not be broke...),
58
+ # so serious value (and probably will not be broken...),
54
59
  # then i let here as it is now.
55
- pairs = []
56
- es.each {|time,record|
57
- c = (@counts[t] = @counts.fetch(t, 0) + 1)
58
- if c % @interval == 0
59
- pairs.push [time, record]
60
- # reset only just before @counts[t] is to be bignum from fixnum
61
- @counts[t] = 0 if c > 0x6fffffff
60
+ if @minimum_rate_per_min
61
+ unless @resets[t]
62
+ @resets[t] = Fluent::Engine.now + (60 - rand(30))
62
63
  end
63
- }
64
+ if Fluent::Engine.now > @resets[t]
65
+ @resets[t] = Fluent::Engine.now + 60
66
+ @counts[t] = 0
67
+ end
68
+ es.each do |time,record|
69
+ c = (@counts[t] = @counts.fetch(t, 0) + 1)
70
+ if c < @minimum_rate_per_min or c % @interval == 0
71
+ pairs.push [time, record]
72
+ end
73
+ end
74
+ else
75
+ es.each do |time,record|
76
+ c = (@counts[t] = @counts.fetch(t, 0) + 1)
77
+ if c % @interval == 0
78
+ pairs.push [time, record]
79
+ # reset only just before @counts[t] is to be bignum from fixnum
80
+ @counts[t] = 0 if c > 0x6fffffff
81
+ end
82
+ end
83
+ end
84
+
64
85
  emit_sampled(tag, pairs)
65
86
 
66
87
  chain.next
@@ -41,12 +41,6 @@ class SamplingFilterOutputTest < Test::Unit::TestCase
41
41
  assert_equal 'output', d.instance.add_prefix
42
42
  end
43
43
 
44
- # CONFIG = %[
45
- # interval 10
46
- # sample_unit tag
47
- # remove_prefix input
48
- # add_prefix sampled
49
- # ]
50
44
  def test_emit
51
45
  d1 = create_driver(CONFIG, 'input.hoge1')
52
46
  time = Time.parse("2012-01-02 13:14:15").to_i
@@ -97,4 +91,57 @@ class SamplingFilterOutputTest < Test::Unit::TestCase
97
91
  assert_equal 'record9', emits[2][2]['field1']
98
92
  assert_equal 'record12', emits[3][2]['field1']
99
93
  end
94
+
95
+ def test_minimum_rate
96
+ config = %[
97
+ interval 10
98
+ sample_unit tag
99
+ remove_prefix input
100
+ minimum_rate_per_min 100
101
+ ]
102
+ d = create_driver(config, 'input.hoge3')
103
+ time = Time.parse("2012-01-02 13:14:15").to_i
104
+ d.run do
105
+ (1..100).each do |t|
106
+ d.emit({'times' => t, 'data' => 'x'})
107
+ end
108
+ (101..130).each do |t|
109
+ d.emit({'times' => t, 'data' => 'y'})
110
+ end
111
+ end
112
+ emits = d.emits
113
+ assert_equal 103, emits.length
114
+ assert_equal 'sampled.hoge3', emits[0][0]
115
+ assert_equal ((1..100).map(&:to_i) + [110, 120, 130]), emits.map{|t,time,r| r['times']}
116
+ assert_equal (['x']*100 + ['y']*3), emits.map{|t,time,r| r['data']}
117
+
118
+ end
119
+ def test_minimum_rate_expire
120
+ # hey, this test needs 60 seconds....
121
+ assert_equal 1, 1
122
+ return
123
+
124
+ config = %[
125
+ interval 10
126
+ sample_unit tag
127
+ remove_prefix input
128
+ minimum_rate_per_min 10
129
+ ]
130
+ d = create_driver(config, 'input.hoge4')
131
+ time = Time.parse("2012-01-02 13:14:15").to_i
132
+ d.run do
133
+ (1..100).each do |t|
134
+ d.emit({'times' => t, 'data' => 'x'})
135
+ end
136
+ sleep 60
137
+ (101..130).each do |t|
138
+ d.emit({'times' => t, 'data' => 'y'})
139
+ end
140
+ end
141
+ emits = d.emits
142
+ # assert_equal (19 + 12), emits.length
143
+ assert_equal 'sampled.hoge4', emits[0][0]
144
+ assert_equal ((1..10).map(&:to_i)+[20,30,40,50,60,70,80,90,100]+(101..110).map(&:to_i)+[120,130]), emits.map{|t,time,r| r['times']}
145
+ assert_equal (['x']*19 + ['y']*12), emits.map{|t,time,r| r['data']}
146
+ end
100
147
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-sampling-filter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.2
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-03-29 00:00:00.000000000 Z
12
+ date: 2012-08-08 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -140,7 +140,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
140
140
  version: '0'
141
141
  segments:
142
142
  - 0
143
- hash: 2842791238956206034
143
+ hash: 2658451737493219281
144
144
  required_rubygems_version: !ruby/object:Gem::Requirement
145
145
  none: false
146
146
  requirements:
@@ -149,7 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
149
149
  version: '0'
150
150
  requirements: []
151
151
  rubyforge_project:
152
- rubygems_version: 1.8.19
152
+ rubygems_version: 1.8.21
153
153
  signing_key:
154
154
  specification_version: 3
155
155
  summary: fluentd plugin to pickup sample data from matched massages