zk-eventmachine 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,229 @@
1
+ # Taken from EventMachine release candidate
2
+ module ZK
3
+ module ZKEventMachine
4
+ # A simple iterator for concurrent asynchronous work.
5
+ #
6
+ # Unlike ruby's built-in iterators, the end of the current iteration cycle is signaled manually,
7
+ # instead of happening automatically after the yielded block finishes executing. For example:
8
+ #
9
+ # (0..10).each{ |num| }
10
+ #
11
+ # becomes:
12
+ #
13
+ # EM::Iterator.new(0..10).each{ |num,iter| iter.next }
14
+ #
15
+ # This is especially useful when doing asynchronous work via reactor libraries and
16
+ # functions. For example, given a sync and async http api:
17
+ #
18
+ # response = sync_http_get(url); ...
19
+ # async_http_get(url){ |response| ... }
20
+ #
21
+ # a synchronous iterator such as:
22
+ #
23
+ # responses = urls.map{ |url| sync_http_get(url) }
24
+ # ...
25
+ # puts 'all done!'
26
+ #
27
+ # could be written as:
28
+ #
29
+ # EM::Iterator.new(urls).map(proc{ |url,iter|
30
+ # async_http_get(url){ |res|
31
+ # iter.return(res)
32
+ # }
33
+ # }, proc{ |responses|
34
+ # ...
35
+ # puts 'all done!'
36
+ # })
37
+ #
38
+ # Now, you can take advantage of the asynchronous api to issue requests in parallel. For example,
39
+ # to fetch 10 urls at a time, simply pass in a concurrency of 10:
40
+ #
41
+ # EM::Iterator.new(urls, 10).each do |url,iter|
42
+ # async_http_get(url){ iter.next }
43
+ # end
44
+ #
45
+ class Iterator
46
+ # Create a new parallel async iterator with specified concurrency.
47
+ #
48
+ # i = EM::Iterator.new(1..100, 10)
49
+ #
50
+ # will create an iterator over the range that processes 10 items at a time. Iteration
51
+ # is started via #each, #map or #inject
52
+ #
53
+ def initialize(list, concurrency = 1)
54
+ raise ArgumentError, 'argument must be an array' unless list.respond_to?(:to_a)
55
+ @list = list.to_a.dup
56
+ @concurrency = concurrency
57
+
58
+ @started = false
59
+ @ended = false
60
+ end
61
+
62
+ # Change the concurrency of this iterator. Workers will automatically be spawned or destroyed
63
+ # to accomodate the new concurrency level.
64
+ #
65
+ def concurrency=(val)
66
+ old = @concurrency
67
+ @concurrency = val
68
+
69
+ spawn_workers if val > old and @started and !@ended
70
+ end
71
+ attr_reader :concurrency
72
+
73
+ # Iterate over a set of items using the specified block or proc.
74
+ #
75
+ # EM::Iterator.new(1..100).each do |num, iter|
76
+ # puts num
77
+ # iter.next
78
+ # end
79
+ #
80
+ # An optional second proc is invoked after the iteration is complete.
81
+ #
82
+ # EM::Iterator.new(1..100).each(
83
+ # proc{ |num,iter| iter.next },
84
+ # proc{ puts 'all done' }
85
+ # )
86
+ #
87
+ def each(foreach=nil, after=nil, &blk)
88
+ raise ArgumentError, 'proc or block required for iteration' unless foreach ||= blk
89
+ raise RuntimeError, 'cannot iterate over an iterator more than once' if @started or @ended
90
+
91
+ @started = true
92
+ @pending = 0
93
+ @workers = 0
94
+
95
+ all_done = proc{
96
+ after.call if after and @ended and @pending == 0
97
+ }
98
+
99
+ @process_next = proc{
100
+ # p [:process_next, :pending=, @pending, :workers=, @workers, :ended=, @ended, :concurrency=, @concurrency, :list=, @list]
101
+ unless @ended or @workers > @concurrency
102
+ if @list.empty?
103
+ @ended = true
104
+ @workers -= 1
105
+ all_done.call
106
+ else
107
+ item = @list.shift
108
+ @pending += 1
109
+
110
+ is_done = false
111
+ on_done = proc{
112
+ raise RuntimeError, 'already completed this iteration' if is_done
113
+ is_done = true
114
+
115
+ @pending -= 1
116
+
117
+ if @ended
118
+ all_done.call
119
+ else
120
+ EM.next_tick(@process_next)
121
+ end
122
+ }
123
+ class << on_done
124
+ alias :next :call
125
+ end
126
+
127
+ foreach.call(item, on_done)
128
+ end
129
+ else
130
+ @workers -= 1
131
+ end
132
+ }
133
+
134
+ spawn_workers
135
+
136
+ self
137
+ end
138
+
139
+ # Collect the results of an asynchronous iteration into an array.
140
+ #
141
+ # EM::Iterator.new(%w[ pwd uptime uname date ], 2).map(proc{ |cmd,iter|
142
+ # EM.system(cmd){ |output,status|
143
+ # iter.return(output)
144
+ # }
145
+ # }, proc{ |results|
146
+ # p results
147
+ # })
148
+ #
149
+ def map(foreach, after)
150
+ index = 0
151
+
152
+ inject([], proc{ |results,item,iter|
153
+ i = index
154
+ index += 1
155
+
156
+ is_done = false
157
+ on_done = proc{ |res|
158
+ raise RuntimeError, 'already returned a value for this iteration' if is_done
159
+ is_done = true
160
+
161
+ results[i] = res
162
+ iter.return(results)
163
+ }
164
+ class << on_done
165
+ alias :return :call
166
+ def next
167
+ raise NoMethodError, 'must call #return on a map iterator'
168
+ end
169
+ end
170
+
171
+ foreach.call(item, on_done)
172
+ }, proc{ |results|
173
+ after.call(results)
174
+ })
175
+ end
176
+
177
+ # Inject the results of an asynchronous iteration onto a given object.
178
+ #
179
+ # EM::Iterator.new(%w[ pwd uptime uname date ], 2).inject({}, proc{ |hash,cmd,iter|
180
+ # EM.system(cmd){ |output,status|
181
+ # hash[cmd] = status.exitstatus == 0 ? output.strip : nil
182
+ # iter.return(hash)
183
+ # }
184
+ # }, proc{ |results|
185
+ # p results
186
+ # })
187
+ #
188
+ def inject(obj, foreach, after)
189
+ each(proc{ |item,iter|
190
+ is_done = false
191
+ on_done = proc{ |res|
192
+ raise RuntimeError, 'already returned a value for this iteration' if is_done
193
+ is_done = true
194
+
195
+ obj = res
196
+ iter.next
197
+ }
198
+ class << on_done
199
+ alias :return :call
200
+ def next
201
+ raise NoMethodError, 'must call #return on an inject iterator'
202
+ end
203
+ end
204
+
205
+ foreach.call(obj, item, on_done)
206
+ }, proc{
207
+ after.call(obj)
208
+ })
209
+ end
210
+
211
+ private
212
+
213
+ # Spawn workers to consume items from the iterator's enumerator based on the current concurrency level.
214
+ #
215
+ def spawn_workers
216
+ EM.next_tick(start_worker = proc{
217
+ if @workers < @concurrency and !@ended
218
+ # p [:spawning_worker, :workers=, @workers, :concurrency=, @concurrency, :ended=, @ended]
219
+ @workers += 1
220
+ @process_next.call
221
+ EM.next_tick(start_worker)
222
+ end
223
+ })
224
+ nil
225
+ end
226
+ end
227
+ end
228
+ end
229
+
@@ -0,0 +1,124 @@
1
+ module ZK
2
+ module ZKEventMachine
3
+ module Unixisms
4
+ def mkdir_p(paths, &block)
5
+ dfr = Deferred::Default.new.tap do |my_dfr|
6
+ Iterator.new(Array(paths).flatten.compact, 1).map(
7
+ lambda { |path,iter| # foreach
8
+ d = _mkdir_p_dfr(path)
9
+ d.callback { |p| iter.return(p) }
10
+ d.errback { |e| my_dfr.fail(e) }
11
+ },
12
+ lambda { |results| my_dfr.succeed(results) } # after completion
13
+ )
14
+ end
15
+
16
+ _handle_calling_convention(dfr, &block)
17
+ end
18
+
19
+ def rm_rf(paths, &blk)
20
+ dfr = Deferred::Default.new.tap do |my_dfr|
21
+ Iterator.new(Array(paths).flatten.compact, 1).each(
22
+ lambda { |path,iter| # foreach
23
+ d = _rm_rf_dfr(path)
24
+ d.callback { iter.next }
25
+ d.errback { |e| my_dfr.fail(e) }
26
+ },
27
+ lambda { my_dfr.succeed } # after completion
28
+ )
29
+ end
30
+
31
+ _handle_calling_convention(dfr, &blk)
32
+ end
33
+
34
+ def find(*paths, &block)
35
+ raise NotImplementedError, "Coming soon"
36
+ end
37
+
38
+ def block_until_node_deleted(abs_node_path)
39
+ raise NotImplementedError, "blocking does not make sense in EventMachine-land"
40
+ end
41
+
42
+ protected
43
+ def _handle_calling_convention(dfr, &blk)
44
+ return dfr unless blk
45
+ dfr.callback { |*a| blk.call(nil, *a) }
46
+ dfr.errback { |exc| blk.call(exc) }
47
+ dfr
48
+ end
49
+
50
+ def _rm_rf_dfr(path)
51
+ Deferred::Default.new.tap do |my_dfr|
52
+ delete(path) do |exc|
53
+ case exc
54
+ when nil, Exceptions::NoNode
55
+ my_dfr.succeed
56
+ when Exceptions::NotEmpty
57
+ children(path) do |exc,chldrn,_|
58
+ case exc
59
+ when Exceptions::NoNode
60
+ my_dfr.succeed
61
+ when nil
62
+ abspaths = chldrn.map { |n| [path, n].join('/') }
63
+ Iterator.new(abspaths).each(
64
+ lambda { |absp,iter|
65
+ d = _rm_rf_dfr(absp)
66
+ d.callback { |*|
67
+ logger.debug { "removed #{absp}" }
68
+ iter.next
69
+ }
70
+ d.errback { |e|
71
+ logger.debug { "got failure #{e.inspect}" }
72
+ my_dfr.fail(e) # this will stop the iteration
73
+ }
74
+ },
75
+ lambda {
76
+ my_dfr.chain_to(_rm_rf_dfr(path))
77
+ }
78
+ )
79
+ else
80
+ my_dfr.fail(exc)
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
86
+ end
87
+
88
+ def _mkdir_p_dfr(path)
89
+ Deferred::Default.new.tap do |my_dfr|
90
+ d = create(path, '')
91
+
92
+ d.callback do |new_path|
93
+ my_dfr.succeed(new_path)
94
+ end
95
+
96
+ d.errback do |exc|
97
+ case exc
98
+ when Exceptions::NodeExists
99
+ # this is the bottom of the stack, where we start bubbling back up
100
+ # or the first call, path already exists, return
101
+ my_dfr.succeed(path)
102
+ when Exceptions::NoNode
103
+ # our node didn't exist now, so we try an recreate it after our
104
+ # parent has been created
105
+
106
+ parent_d = mkdir_p(File.dirname(path)) # set up our parent to be created
107
+
108
+ parent_d.callback do |parent_path| # once our parent exists
109
+ create(path, '') do |exc,p| # create our path again
110
+ exc ? my_dfr.fail(exc) : my_dfr.succeed(p) # pass our success or failure up the chain
111
+ end
112
+ end
113
+
114
+ parent_d.errback do |e| # if creating our parent fails
115
+ my_dfr.fail(e) # pass that along too
116
+ end
117
+ end
118
+ end
119
+ end
120
+ end
121
+ end
122
+ end
123
+ end
124
+
@@ -0,0 +1,5 @@
1
+ module ZK
2
+ module ZKEventMachine
3
+ VERSION = "0.1.1"
4
+ end
5
+ end
@@ -0,0 +1,24 @@
1
+ require 'eventmachine'
2
+
3
+ require 'zookeeper'
4
+ require 'zookeeper/em_client'
5
+
6
+ require 'zk'
7
+
8
+
9
+ module ZK
10
+ module ZKEventMachine
11
+ end
12
+ end
13
+
14
+
15
+ $LOAD_PATH.unshift(File.expand_path('../..', __FILE__)).uniq!
16
+
17
+ require 'z_k/z_k_event_machine/iterator'
18
+ require 'z_k/z_k_event_machine/deferred'
19
+ require 'z_k/z_k_event_machine/callback'
20
+ require 'z_k/z_k_event_machine/event_handler_e_m'
21
+ require 'z_k/z_k_event_machine/unixisms'
22
+ require 'z_k/z_k_event_machine/client'
23
+
24
+
@@ -0,0 +1,2 @@
1
+ require File.expand_path('../z_k/z_k_event_machine', __FILE__)
2
+
@@ -0,0 +1,19 @@
1
+ require 'rubygems'
2
+ require 'bundler/setup'
3
+
4
+ $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
5
+
6
+ require 'zk-eventmachine'
7
+ require 'evented-spec'
8
+
9
+ # Requires supporting ruby files with custom matchers and macros, etc,
10
+ # in spec/support/ and its subdirectories.
11
+ Dir[File.expand_path("../support/**/*.rb", __FILE__)].each {|f| require f}
12
+
13
+ $stderr.sync = true
14
+
15
+ RSpec.configure do |config|
16
+ config.mock_with :flexmock
17
+ end
18
+
19
+
@@ -0,0 +1,44 @@
1
+ # method to wait until block passed returns true or timeout (default is 2 seconds) is reached
2
+ def wait_until(timeout=2)
3
+ time_to_stop = Time.now + timeout
4
+
5
+ until yield
6
+ break if Time.now > time_to_stop
7
+ Thread.pass
8
+ end
9
+ end
10
+
11
+ def wait_while(timeout=2)
12
+ time_to_stop = Time.now + timeout
13
+
14
+ while yield
15
+ break if Time.now > time_to_stop
16
+ Thread.pass
17
+ end
18
+ end
19
+
20
+ class ::Thread
21
+ # join with thread until given block is true, the thread joins successfully,
22
+ # or timeout seconds have passed
23
+ #
24
+ def join_until(timeout=2)
25
+ time_to_stop = Time.now + timeout
26
+
27
+ until yield
28
+ break if Time.now > time_to_stop
29
+ break if join(0.1)
30
+ end
31
+ end
32
+
33
+ def join_while(timeout=2)
34
+ time_to_stop = Time.now + timeout
35
+
36
+ while yield
37
+ break if Time.now > time_to_stop
38
+ break if join(0.1)
39
+ end
40
+ end
41
+ end
42
+
43
+
44
+
@@ -0,0 +1,9 @@
1
+ require 'logger'
2
+
3
+ ZK.logger = Logger.new(File.expand_path('../../../test.log', __FILE__)).tap {|l| l.level = Logger::DEBUG}
4
+ Zookeeper.logger = ZK.logger
5
+
6
+ def logger
7
+ ZK.logger
8
+ end
9
+
@@ -0,0 +1,26 @@
1
+ require 'rspec/core/formatters/progress_formatter'
2
+
3
+ # module Motionbox
4
+ # # essentially a monkey-patch to the ProgressBarFormatter, outputs
5
+ # # '== #{example_proxy.description} ==' in the logs before each test. makes it
6
+ # # easier to match up tests with the SQL they produce
7
+ # class LoggingProgressBarFormatter < RSpec::Core::Formatters::ProgressFormatter
8
+ # def example_started(example)
9
+ # ZK.logger.info(yellow("\n=====<([ #{example.full_description} ])>=====\n"))
10
+ # super
11
+ # end
12
+ # end
13
+ # end
14
+
15
+ module RSpec
16
+ module Core
17
+ module Formatters
18
+ class ProgressFormatter
19
+ def example_started(example)
20
+ ZK.logger.info(yellow("\n=====<([ #{example.full_description} ])>=====\n"))
21
+ super(example)
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,135 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'ZK::ZKEventMachine::Callback' do
4
+ before do
5
+ @stat_mock = flexmock(:stat)
6
+ @context_mock = flexmock(:context)
7
+
8
+ flexmock(::EM) do |em|
9
+ em.should_receive(:next_tick).with(Proc).and_return { |b| b.call }
10
+ end
11
+
12
+ end
13
+
14
+ describe 'DataCallback' do
15
+ before do
16
+ @cb = ZK::ZKEventMachine::Callback::DataCallback.new
17
+ end
18
+
19
+ describe 'call' do
20
+ describe 'with callbacks and errbacks set' do
21
+ before do
22
+ @callback_args = @errback_args = nil
23
+
24
+ @cb.callback do |*a|
25
+ @callback_args = a
26
+ end
27
+
28
+ @cb.errback do |*a|
29
+ @errback_args = a
30
+ end
31
+ end
32
+
33
+ describe 'success' do
34
+ before do
35
+ @cb.call(:rc => 0, :data => 'data', :stat => @stat_mock, :context => @context_mock)
36
+ end
37
+
38
+ it %[should have called the callback] do
39
+ @callback_args.should_not be_nil
40
+ end
41
+
42
+ it %[should have the correct number of args] do
43
+ @callback_args.length.should == 2
44
+ end
45
+
46
+ it %[should have the correct args] do
47
+ @callback_args[0].should == 'data'
48
+ @callback_args[1].should == @stat_mock
49
+ end
50
+ end
51
+
52
+ describe 'failure' do
53
+ before do
54
+ @cb.call(:rc => ::ZK::Exceptions::NONODE)
55
+ end
56
+
57
+ it %[should have called the errback] do
58
+ @errback_args.should_not be_nil
59
+ end
60
+
61
+ it %[should be called with the appropriate exception instance] do
62
+ @errback_args.first.should be_instance_of(::ZK::Exceptions::NoNode)
63
+ end
64
+ end
65
+ end
66
+
67
+ describe 'with an on_result block set' do
68
+ before do
69
+ @args = nil
70
+
71
+ @cb.on_result do |*a|
72
+ @args = a
73
+ end
74
+ end
75
+
76
+ describe 'success' do
77
+ before do
78
+ @cb.call(:rc => 0, :data => 'data', :stat => @stat_mock, :context => @context_mock)
79
+ end
80
+
81
+ it %[should have called the block] do
82
+ @args.should_not be_nil
83
+ end
84
+
85
+ it %[should have used the correct arguments] do
86
+ @args[0].should == nil
87
+ @args[1].should == 'data'
88
+ @args[2].should == @stat_mock
89
+ end
90
+ end
91
+
92
+ describe 'failure' do
93
+ before do
94
+ @cb.call(:rc => ::ZK::Exceptions::NONODE)
95
+ end
96
+
97
+ it %[should have called the block] do
98
+ @args.should_not be_nil
99
+ end
100
+
101
+ it %[should have used the correct arguments] do
102
+ @args.first.should be_instance_of(::ZK::Exceptions::NoNode)
103
+ end
104
+ end
105
+ end
106
+
107
+ describe 'on_result can be handed a block' do
108
+ before do
109
+ @args = nil
110
+
111
+ blk = lambda { |*a| @args = a }
112
+
113
+ @cb.on_result(blk)
114
+ end
115
+
116
+ describe 'success' do
117
+ before do
118
+ @cb.call(:rc => 0, :data => 'data', :stat => @stat_mock, :context => @context_mock)
119
+ end
120
+
121
+ it %[should have called the block] do
122
+ @args.should_not be_nil
123
+ end
124
+
125
+ it %[should have used the correct arguments] do
126
+ @args[0].should == nil
127
+ @args[1].should == 'data'
128
+ @args[2].should == @stat_mock
129
+ end
130
+ end
131
+ end
132
+ end
133
+ end
134
+ end
135
+