stud 0.0.12 → 0.0.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. data/README.md +1 -0
  2. data/lib/stud/buffer.rb +260 -0
  3. metadata +8 -7
data/README.md CHANGED
@@ -18,6 +18,7 @@ each 'pattern' are quite small by code size.
18
18
  * {Stud::Pool} - generic resource pools
19
19
  * {Stud::Task} - tasks (threads that can return values, exceptions, etc)
20
20
  * {Stud.interval} - interval execution (do X every N seconds)
21
+ * {Stud::Buffer} - batch & flush behavior.
21
22
 
22
23
  ## TODO:
23
24
 
@@ -0,0 +1,260 @@
1
+ module Stud
2
+
3
+ # @author {Alex Dean}[http://github.com/alexdean]
4
+ #
5
+ # Implements a generic framework for accepting events which are later flushed
6
+ # in batches. Flushing occurrs whenever +:max_items+ or +:max_interval+ (seconds)
7
+ # has been reached.
8
+ #
9
+ # Including class must implement +flush+, which will be called with all accumulated
10
+ # items either when the output buffer fills (+:max_items+) or when a fixed amount
11
+ # of time (+:max_interval+) passes.
12
+ #
13
+ # == batch_receive and flush
14
+ # General receive/flush can be implemented in one of two ways.
15
+ #
16
+ # === batch_receive(event) / flush(events)
17
+ # +flush+ will receive an array of events which were passed to +buffer_receive+.
18
+ #
19
+ # batch_receive('one')
20
+ # batch_receive('two')
21
+ #
22
+ # will cause a flush invocation like
23
+ #
24
+ # flush(['one', 'two'])
25
+ #
26
+ # === batch_receive(event, group) / flush(events, group)
27
+ # flush() will receive an array of events, plus a grouping key.
28
+ #
29
+ # batch_receive('one', :server => 'a')
30
+ # batch_receive('two', :server => 'b')
31
+ # batch_receive('three', :server => 'a')
32
+ # batch_receive('four', :server => 'b')
33
+ #
34
+ # will result in the following flush calls
35
+ #
36
+ # flush(['one', 'three'], {:server => 'a'})
37
+ # flush(['two', 'four'], {:server => 'b'})
38
+ #
39
+ # Grouping keys can be anything which are valid Hash keys. (They don't have to
40
+ # be hashes themselves.) Strings or Fixnums work fine. Use anything which you'd
41
+ # like to receive in your +flush+ method to help enable different handling for
42
+ # various groups of events.
43
+ #
44
+ # == on_flush_error
45
+ # Including class may implement +on_flush_error+, which will be called with an
46
+ # Exception instance whenever buffer_flush encounters an error.
47
+ #
48
+ # * +buffer_flush+ will automatically re-try failed flushes, so +on_flush_error+
49
+ # should not try to implement retry behavior.
50
+ # * Exceptions occurring within +on_flush_error+ are not handled by
51
+ # +buffer_flush+.
52
+ #
53
+ # == on_full_buffer_receive
54
+ # Including class may implement +on_full_buffer_receive+, which will be called
55
+ # whenever +buffer_receive+ is called while the buffer is full.
56
+ #
57
+ # +on_full_buffer_receive+ will receive a Hash like <code>{:pending => 30,
58
+ # :outgoing => 20}</code> which describes the internal state of the module at
59
+ # the moment.
60
+ #
61
+ # == final flush
62
+ # Including class should call <code>buffer_flush(:final => true)</code> during a teardown/
63
+ # shutdown routine (after the last call to buffer_receive) to ensure that all
64
+ # accumulated messages are flushed.
65
+ module Buffer
66
+
67
+ public
68
+ # Initialize the buffer.
69
+ #
70
+ # Call directly from your constructor if you wish to set some non-default
71
+ # options. Otherwise buffer_initialize will be called automatically during the
72
+ # first buffer_receive call.
73
+ #
74
+ # Options:
75
+ # * :max_items, Max number of items to buffer before flushing. Default 50.
76
+ # * :max_interval, Max number of seconds to wait between flushes. Default 5.
77
+ # * :logger, A logger to write log messages to. No default. Optional.
78
+ #
79
+ # @param [Hash] options
80
+ def buffer_initialize(options={})
81
+ if ! self.class.method_defined?(:flush)
82
+ raise ArgumentError, "Any class including Stud::Buffer must define a flush() method."
83
+ end
84
+
85
+ @buffer_config = {
86
+ :max_items => options[:max_items] || 50,
87
+ :max_interval => options[:max_interval] || 5,
88
+ :logger => options[:logger] || nil,
89
+ :has_on_flush_error => self.class.method_defined?(:on_flush_error),
90
+ :has_on_full_buffer_receive => self.class.method_defined?(:on_full_buffer_receive)
91
+ }
92
+ @buffer_state = {
93
+ # items accepted from including class
94
+ :pending_items => {},
95
+ :pending_count => 0,
96
+
97
+ # guard access to pending_items & pending_count
98
+ :pending_mutex => Mutex.new,
99
+
100
+ # items which are currently being flushed
101
+ :outgoing_items => {},
102
+ :outgoing_count => 0,
103
+
104
+ # ensure only 1 flush is operating at once
105
+ :flush_mutex => Mutex.new,
106
+
107
+ # data for timed flushes
108
+ :last_flush => Time.now.to_i,
109
+ :timer => Thread.new do
110
+ loop do
111
+ sleep(@buffer_config[:max_interval])
112
+ buffer_flush(:force => true)
113
+ end
114
+ end
115
+ }
116
+
117
+ # events we've accumulated
118
+ buffer_clear_pending
119
+ end
120
+
121
+ # Determine if +:max_items+ has been reached.
122
+ #
123
+ # buffer_receive calls will block while <code>buffer_full? == true</code>.
124
+ #
125
+ # @return [bool] Is the buffer full?
126
+ def buffer_full?
127
+ @buffer_state[:pending_count] + @buffer_state[:outgoing_count] >= @buffer_config[:max_items]
128
+ end
129
+
130
+ # Save an event for later delivery
131
+ #
132
+ # Events are grouped by the (optional) group parameter you provide.
133
+ # Groups of events, plus the group name, are later passed to +flush+.
134
+ #
135
+ # This call will block if +:max_items+ has been reached.
136
+ #
137
+ # @see Stud::Buffer The overview has more information on grouping and flushing.
138
+ #
139
+ # @param event An item to buffer for flushing later.
140
+ # @param group Optional grouping key. All events with the same key will be
141
+ # passed to +flush+ together, along with the grouping key itself.
142
+ def buffer_receive(event, group=nil)
143
+ buffer_initialize if ! @buffer_state
144
+
145
+ # block if we've accumulated too many events
146
+ while buffer_full? do
147
+ on_full_buffer_receive(
148
+ :pending => @buffer_state[:pending_count],
149
+ :outgoing => @buffer_state[:outgoing_count]
150
+ ) if @buffer_config[:has_on_full_buffer_receive]
151
+ sleep 0.1
152
+ end
153
+
154
+ @buffer_state[:pending_mutex].synchronize do
155
+ @buffer_state[:pending_items][group] << event
156
+ @buffer_state[:pending_count] += 1
157
+ end
158
+
159
+ buffer_flush
160
+ end
161
+
162
+ # Try to flush events.
163
+ #
164
+ # Returns immediately if flushing is not necessary/possible at the moment:
165
+ # * :max_items have not been accumulated
166
+ # * :max_interval seconds have not elapased since the last flush
167
+ # * another flush is in progress
168
+ #
169
+ # <code>buffer_flush(:force => true)</code> will cause a flush to occur even
170
+ # if +:max_items+ or +:max_interval+ have not been reached. A forced flush
171
+ # will still return immediately (without flushing) if another flush is
172
+ # currently in progress.
173
+ #
174
+ # <code>buffer_flush(:final => true)</code> is identical to <code>buffer_flush(:force => true)</code>,
175
+ # except that if another flush is already in progress, <code>buffer_flush(:final => true)</code>
176
+ # will block/wait for the other flush to finish before proceeding.
177
+ #
178
+ # @param [Hash] options Optional. May be <code>{:force => true}</code> or <code>{:final => true}</code>.
179
+ # @return [Fixnum] The number of items successfully passed to +flush+.
180
+ def buffer_flush(options={})
181
+ force = options[:force] || options[:final]
182
+ final = options[:final]
183
+
184
+ # final flush will wait for lock, so we are sure to flush out all buffered events
185
+ if options[:final]
186
+ @buffer_state[:flush_mutex].lock
187
+ elsif ! @buffer_state[:flush_mutex].try_lock # failed to get lock, another flush already in progress
188
+ return 0
189
+ end
190
+
191
+ items_flushed = 0
192
+
193
+ begin
194
+ time_since_last_flush = Time.now.to_i - @buffer_state[:last_flush]
195
+
196
+ return 0 if @buffer_state[:pending_count] == 0
197
+ return 0 if (!force) &&
198
+ (@buffer_state[:pending_count] < @buffer_config[:max_items]) &&
199
+ (time_since_last_flush < @buffer_config[:max_interval])
200
+
201
+ @buffer_state[:pending_mutex].synchronize do
202
+ @buffer_state[:outgoing_items] = @buffer_state[:pending_items]
203
+ @buffer_state[:outgoing_count] = @buffer_state[:pending_count]
204
+ buffer_clear_pending
205
+ end
206
+
207
+ @buffer_config[:logger].debug("Flushing output",
208
+ :outgoing_count => @buffer_state[:outgoing_count],
209
+ :time_since_last_flush => time_since_last_flush,
210
+ :outgoing_events => @buffer_state[:outgoing_items],
211
+ :batch_timeout => @buffer_config[:max_interval],
212
+ :force => force,
213
+ :final => final
214
+ ) if @buffer_config[:logger]
215
+
216
+ @buffer_state[:outgoing_items].each do |group, events|
217
+ begin
218
+ if group.nil?
219
+ flush(events)
220
+ else
221
+ flush(events, group)
222
+ end
223
+
224
+ @buffer_state[:outgoing_items].delete(group)
225
+ events_size = events.size
226
+ @buffer_state[:outgoing_count] -= events_size
227
+ items_flushed += events_size
228
+
229
+ rescue => e
230
+
231
+ @buffer_config[:logger].warn("Failed to flush outgoing items",
232
+ :outgoing_count => @buffer_state[:outgoing_count],
233
+ :exception => e,
234
+ :backtrace => e.backtrace
235
+ ) if @buffer_config[:logger]
236
+
237
+ if @buffer_config[:has_on_flush_error]
238
+ on_flush_error e
239
+ end
240
+
241
+ sleep 1
242
+ retry
243
+ end
244
+ @buffer_state[:last_flush] = Time.now.to_i
245
+ end
246
+
247
+ ensure
248
+ @buffer_state[:flush_mutex].unlock
249
+ end
250
+
251
+ items_flushed
252
+ end
253
+
254
+ private
255
+ def buffer_clear_pending
256
+ @buffer_state[:pending_items] = Hash.new { |h, k| h[k] = [] }
257
+ @buffer_state[:pending_count] = 0
258
+ end
259
+ end
260
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: stud
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.12
4
+ version: 0.0.13
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-02-08 00:00:00.000000000 Z
12
+ date: 2013-02-15 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: metriks
@@ -82,14 +82,15 @@ executables: []
82
82
  extensions: []
83
83
  extra_rdoc_files: []
84
84
  files:
85
- - lib/stud/benchmark/rusage.rb
86
- - lib/stud/interval.rb
87
- - lib/stud/pool.rb
88
- - lib/stud/secret.rb
89
85
  - lib/stud/try.rb
90
- - lib/stud/task.rb
86
+ - lib/stud/interval.rb
91
87
  - lib/stud/benchmark.rb
88
+ - lib/stud/benchmark/rusage.rb
89
+ - lib/stud/task.rb
92
90
  - lib/stud/trap.rb
91
+ - lib/stud/buffer.rb
92
+ - lib/stud/secret.rb
93
+ - lib/stud/pool.rb
93
94
  - LICENSE
94
95
  - CHANGELIST
95
96
  - README.md