wj_eventmachine 1.3.0.dev.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (180) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +179 -0
  3. data/GNU +281 -0
  4. data/LICENSE +60 -0
  5. data/README.md +110 -0
  6. data/docs/DocumentationGuidesIndex.md +27 -0
  7. data/docs/GettingStarted.md +520 -0
  8. data/docs/old/ChangeLog +211 -0
  9. data/docs/old/DEFERRABLES +246 -0
  10. data/docs/old/EPOLL +141 -0
  11. data/docs/old/INSTALL +13 -0
  12. data/docs/old/KEYBOARD +42 -0
  13. data/docs/old/LEGAL +25 -0
  14. data/docs/old/LIGHTWEIGHT_CONCURRENCY +130 -0
  15. data/docs/old/PURE_RUBY +75 -0
  16. data/docs/old/RELEASE_NOTES +94 -0
  17. data/docs/old/SMTP +4 -0
  18. data/docs/old/SPAWNED_PROCESSES +148 -0
  19. data/docs/old/TODO +8 -0
  20. data/examples/guides/getting_started/01_eventmachine_echo_server.rb +18 -0
  21. data/examples/guides/getting_started/02_eventmachine_echo_server_that_recognizes_exit_command.rb +22 -0
  22. data/examples/guides/getting_started/03_simple_chat_server.rb +149 -0
  23. data/examples/guides/getting_started/04_simple_chat_server_step_one.rb +27 -0
  24. data/examples/guides/getting_started/05_simple_chat_server_step_two.rb +43 -0
  25. data/examples/guides/getting_started/06_simple_chat_server_step_three.rb +98 -0
  26. data/examples/guides/getting_started/07_simple_chat_server_step_four.rb +121 -0
  27. data/examples/guides/getting_started/08_simple_chat_server_step_five.rb +141 -0
  28. data/examples/old/ex_channel.rb +43 -0
  29. data/examples/old/ex_queue.rb +2 -0
  30. data/examples/old/ex_tick_loop_array.rb +15 -0
  31. data/examples/old/ex_tick_loop_counter.rb +32 -0
  32. data/examples/old/helper.rb +2 -0
  33. data/ext/binder.cpp +124 -0
  34. data/ext/binder.h +52 -0
  35. data/ext/cmain.cpp +1046 -0
  36. data/ext/ed.cpp +2238 -0
  37. data/ext/ed.h +460 -0
  38. data/ext/em.cpp +2378 -0
  39. data/ext/em.h +266 -0
  40. data/ext/eventmachine.h +152 -0
  41. data/ext/extconf.rb +285 -0
  42. data/ext/fastfilereader/extconf.rb +120 -0
  43. data/ext/fastfilereader/mapper.cpp +214 -0
  44. data/ext/fastfilereader/mapper.h +59 -0
  45. data/ext/fastfilereader/rubymain.cpp +126 -0
  46. data/ext/kb.cpp +79 -0
  47. data/ext/page.cpp +107 -0
  48. data/ext/page.h +51 -0
  49. data/ext/pipe.cpp +354 -0
  50. data/ext/project.h +174 -0
  51. data/ext/rubymain.cpp +1610 -0
  52. data/ext/ssl.cpp +627 -0
  53. data/ext/ssl.h +103 -0
  54. data/ext/wait_for_single_fd.h +36 -0
  55. data/java/.classpath +8 -0
  56. data/java/.project +17 -0
  57. data/java/src/com/rubyeventmachine/EmReactor.java +625 -0
  58. data/java/src/com/rubyeventmachine/EmReactorException.java +40 -0
  59. data/java/src/com/rubyeventmachine/EmReactorInterface.java +70 -0
  60. data/java/src/com/rubyeventmachine/EventableChannel.java +72 -0
  61. data/java/src/com/rubyeventmachine/EventableDatagramChannel.java +201 -0
  62. data/java/src/com/rubyeventmachine/EventableSocketChannel.java +415 -0
  63. data/java/src/com/rubyeventmachine/NullEmReactor.java +157 -0
  64. data/java/src/com/rubyeventmachine/NullEventableChannel.java +81 -0
  65. data/lib/em/buftok.rb +59 -0
  66. data/lib/em/callback.rb +58 -0
  67. data/lib/em/channel.rb +69 -0
  68. data/lib/em/completion.rb +307 -0
  69. data/lib/em/connection.rb +776 -0
  70. data/lib/em/deferrable.rb +210 -0
  71. data/lib/em/deferrable/pool.rb +2 -0
  72. data/lib/em/file_watch.rb +73 -0
  73. data/lib/em/future.rb +61 -0
  74. data/lib/em/io_streamer.rb +68 -0
  75. data/lib/em/iterator.rb +252 -0
  76. data/lib/em/messages.rb +66 -0
  77. data/lib/em/pool.rb +151 -0
  78. data/lib/em/process_watch.rb +45 -0
  79. data/lib/em/processes.rb +123 -0
  80. data/lib/em/protocols.rb +37 -0
  81. data/lib/em/protocols/header_and_content.rb +138 -0
  82. data/lib/em/protocols/httpclient.rb +303 -0
  83. data/lib/em/protocols/httpclient2.rb +602 -0
  84. data/lib/em/protocols/line_and_text.rb +125 -0
  85. data/lib/em/protocols/line_protocol.rb +33 -0
  86. data/lib/em/protocols/linetext2.rb +179 -0
  87. data/lib/em/protocols/memcache.rb +331 -0
  88. data/lib/em/protocols/object_protocol.rb +46 -0
  89. data/lib/em/protocols/postgres3.rb +246 -0
  90. data/lib/em/protocols/saslauth.rb +175 -0
  91. data/lib/em/protocols/smtpclient.rb +394 -0
  92. data/lib/em/protocols/smtpserver.rb +666 -0
  93. data/lib/em/protocols/socks4.rb +66 -0
  94. data/lib/em/protocols/stomp.rb +205 -0
  95. data/lib/em/protocols/tcptest.rb +54 -0
  96. data/lib/em/pure_ruby.rb +1299 -0
  97. data/lib/em/queue.rb +80 -0
  98. data/lib/em/resolver.rb +232 -0
  99. data/lib/em/spawnable.rb +84 -0
  100. data/lib/em/streamer.rb +118 -0
  101. data/lib/em/threaded_resource.rb +90 -0
  102. data/lib/em/tick_loop.rb +85 -0
  103. data/lib/em/timers.rb +61 -0
  104. data/lib/em/version.rb +3 -0
  105. data/lib/eventmachine.rb +1602 -0
  106. data/lib/jeventmachine.rb +318 -0
  107. data/rakelib/package.rake +120 -0
  108. data/rakelib/test.rake +6 -0
  109. data/rakelib/test_pure.rake +11 -0
  110. data/tests/client.crt +31 -0
  111. data/tests/client.key +51 -0
  112. data/tests/dhparam.pem +13 -0
  113. data/tests/em_ssl_handlers.rb +153 -0
  114. data/tests/em_test_helper.rb +198 -0
  115. data/tests/jruby/test_jeventmachine.rb +38 -0
  116. data/tests/test_attach.rb +199 -0
  117. data/tests/test_basic.rb +321 -0
  118. data/tests/test_channel.rb +75 -0
  119. data/tests/test_completion.rb +178 -0
  120. data/tests/test_connection_count.rb +83 -0
  121. data/tests/test_connection_write.rb +35 -0
  122. data/tests/test_defer.rb +35 -0
  123. data/tests/test_deferrable.rb +35 -0
  124. data/tests/test_epoll.rb +141 -0
  125. data/tests/test_error_handler.rb +38 -0
  126. data/tests/test_exc.rb +37 -0
  127. data/tests/test_file_watch.rb +86 -0
  128. data/tests/test_fork.rb +75 -0
  129. data/tests/test_futures.rb +170 -0
  130. data/tests/test_handler_check.rb +35 -0
  131. data/tests/test_hc.rb +155 -0
  132. data/tests/test_httpclient.rb +238 -0
  133. data/tests/test_httpclient2.rb +132 -0
  134. data/tests/test_idle_connection.rb +31 -0
  135. data/tests/test_inactivity_timeout.rb +102 -0
  136. data/tests/test_io_streamer.rb +47 -0
  137. data/tests/test_ipv4.rb +96 -0
  138. data/tests/test_ipv6.rb +107 -0
  139. data/tests/test_iterator.rb +122 -0
  140. data/tests/test_kb.rb +28 -0
  141. data/tests/test_keepalive.rb +113 -0
  142. data/tests/test_line_protocol.rb +33 -0
  143. data/tests/test_ltp.rb +155 -0
  144. data/tests/test_ltp2.rb +332 -0
  145. data/tests/test_many_fds.rb +21 -0
  146. data/tests/test_next_tick.rb +104 -0
  147. data/tests/test_object_protocol.rb +36 -0
  148. data/tests/test_pause.rb +109 -0
  149. data/tests/test_pending_connect_timeout.rb +52 -0
  150. data/tests/test_pool.rb +196 -0
  151. data/tests/test_process_watch.rb +50 -0
  152. data/tests/test_processes.rb +128 -0
  153. data/tests/test_proxy_connection.rb +180 -0
  154. data/tests/test_pure.rb +156 -0
  155. data/tests/test_queue.rb +64 -0
  156. data/tests/test_resolver.rb +129 -0
  157. data/tests/test_running.rb +14 -0
  158. data/tests/test_sasl.rb +46 -0
  159. data/tests/test_send_file.rb +217 -0
  160. data/tests/test_servers.rb +32 -0
  161. data/tests/test_shutdown_hooks.rb +23 -0
  162. data/tests/test_smtpclient.rb +75 -0
  163. data/tests/test_smtpserver.rb +90 -0
  164. data/tests/test_sock_opt.rb +53 -0
  165. data/tests/test_spawn.rb +290 -0
  166. data/tests/test_ssl_args.rb +41 -0
  167. data/tests/test_ssl_dhparam.rb +57 -0
  168. data/tests/test_ssl_ecdh_curve.rb +57 -0
  169. data/tests/test_ssl_extensions.rb +24 -0
  170. data/tests/test_ssl_methods.rb +31 -0
  171. data/tests/test_ssl_protocols.rb +190 -0
  172. data/tests/test_ssl_verify.rb +52 -0
  173. data/tests/test_stomp.rb +38 -0
  174. data/tests/test_system.rb +46 -0
  175. data/tests/test_threaded_resource.rb +68 -0
  176. data/tests/test_tick_loop.rb +58 -0
  177. data/tests/test_timers.rb +150 -0
  178. data/tests/test_ud.rb +8 -0
  179. data/tests/test_unbind_reason.rb +40 -0
  180. metadata +384 -0
@@ -0,0 +1,210 @@
1
+ #--
2
+ #
3
+ # Author:: Francis Cianfrocca (gmail: blackhedd)
4
+ # Homepage:: http://rubyeventmachine.com
5
+ # Date:: 16 Jul 2006
6
+ #
7
+ # See EventMachine and EventMachine::Connection for documentation and
8
+ # usage examples.
9
+ #
10
+ #----------------------------------------------------------------------------
11
+ #
12
+ # Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
13
+ # Gmail: blackhedd
14
+ #
15
+ # This program is free software; you can redistribute it and/or modify
16
+ # it under the terms of either: 1) the GNU General Public License
17
+ # as published by the Free Software Foundation; either version 2 of the
18
+ # License, or (at your option) any later version; or 2) Ruby's License.
19
+ #
20
+ # See the file COPYING for complete licensing information.
21
+ #
22
+ #---------------------------------------------------------------------------
23
+ #
24
+ #
25
+
26
+ module EventMachine
27
+ module Deferrable
28
+ autoload :Pool, 'em/deferrable/pool'
29
+
30
+ # Specify a block to be executed if and when the Deferrable object receives
31
+ # a status of :succeeded. See #set_deferred_status for more information.
32
+ #
33
+ # Calling this method on a Deferrable object whose status is not yet known
34
+ # will cause the callback block to be stored on an internal list.
35
+ # If you call this method on a Deferrable whose status is :succeeded, the
36
+ # block will be executed immediately, receiving the parameters given to the
37
+ # prior #set_deferred_status call.
38
+ #
39
+ #--
40
+ # If there is no status, add a callback to an internal list.
41
+ # If status is succeeded, execute the callback immediately.
42
+ # If status is failed, do nothing.
43
+ #
44
+ def callback &block
45
+ return unless block
46
+ @deferred_status ||= :unknown
47
+ if @deferred_status == :succeeded
48
+ block.call(*@deferred_args)
49
+ elsif @deferred_status != :failed
50
+ @callbacks ||= []
51
+ @callbacks.unshift block # << block
52
+ end
53
+ self
54
+ end
55
+
56
+ # Cancels an outstanding callback to &block if any. Undoes the action of #callback.
57
+ #
58
+ def cancel_callback block
59
+ @callbacks ||= []
60
+ @callbacks.delete block
61
+ end
62
+
63
+ # Specify a block to be executed if and when the Deferrable object receives
64
+ # a status of :failed. See #set_deferred_status for more information.
65
+ #--
66
+ # If there is no status, add an errback to an internal list.
67
+ # If status is failed, execute the errback immediately.
68
+ # If status is succeeded, do nothing.
69
+ #
70
+ def errback &block
71
+ return unless block
72
+ @deferred_status ||= :unknown
73
+ if @deferred_status == :failed
74
+ block.call(*@deferred_args)
75
+ elsif @deferred_status != :succeeded
76
+ @errbacks ||= []
77
+ @errbacks.unshift block # << block
78
+ end
79
+ self
80
+ end
81
+
82
+ # Cancels an outstanding errback to &block if any. Undoes the action of #errback.
83
+ #
84
+ def cancel_errback block
85
+ @errbacks ||= []
86
+ @errbacks.delete block
87
+ end
88
+
89
+ # Sets the "disposition" (status) of the Deferrable object. See also the large set of
90
+ # sugarings for this method.
91
+ # Note that if you call this method without arguments,
92
+ # no arguments will be passed to the callback/errback.
93
+ # If the user has coded these with arguments, then the
94
+ # user code will throw an argument exception.
95
+ # Implementors of deferrable classes <b>must</b>
96
+ # document the arguments they will supply to user callbacks.
97
+ #
98
+ # OBSERVE SOMETHING VERY SPECIAL here: you may call this method even
99
+ # on the INSIDE of a callback. This is very useful when a previously-registered
100
+ # callback wants to change the parameters that will be passed to subsequently-registered
101
+ # ones.
102
+ #
103
+ # You may give either :succeeded or :failed as the status argument.
104
+ #
105
+ # If you pass :succeeded, then all of the blocks passed to the object using the #callback
106
+ # method (if any) will be executed BEFORE the #set_deferred_status method returns. All of the blocks
107
+ # passed to the object using #errback will be discarded.
108
+ #
109
+ # If you pass :failed, then all of the blocks passed to the object using the #errback
110
+ # method (if any) will be executed BEFORE the #set_deferred_status method returns. All of the blocks
111
+ # passed to the object using # callback will be discarded.
112
+ #
113
+ # If you pass any arguments to #set_deferred_status in addition to the status argument,
114
+ # they will be passed as arguments to any callbacks or errbacks that are executed.
115
+ # It's your responsibility to ensure that the argument lists specified in your callbacks and
116
+ # errbacks match the arguments given in calls to #set_deferred_status, otherwise Ruby will raise
117
+ # an ArgumentError.
118
+ #
119
+ #--
120
+ # We're shifting callbacks off and discarding them as we execute them.
121
+ # This is valid because by definition callbacks are executed no more than
122
+ # once. It also has the magic effect of permitting recursive calls, which
123
+ # means that a callback can call #set_deferred_status and change the parameters
124
+ # that will be sent to subsequent callbacks down the chain.
125
+ #
126
+ # Changed @callbacks and @errbacks from push/shift to unshift/pop, per suggestion
127
+ # by Kirk Haines, to work around the memory leak bug that still exists in many Ruby
128
+ # versions.
129
+ #
130
+ # Changed 15Sep07: after processing callbacks or errbacks, CLEAR the other set of
131
+ # handlers. This gets us a little closer to the behavior of Twisted's "deferred,"
132
+ # which only allows status to be set once. Prior to making this change, it was possible
133
+ # to "succeed" a Deferrable (triggering its callbacks), and then immediately "fail" it,
134
+ # triggering its errbacks! That is clearly undesirable, but it's just as undesirable
135
+ # to raise an exception is status is set more than once on a Deferrable. The latter
136
+ # behavior would invalidate the idiom of resetting arguments by setting status from
137
+ # within a callback or errback, but more seriously it would cause spurious errors
138
+ # if a Deferrable was timed out and then an attempt was made to succeed it. See the
139
+ # comments under the new method #timeout.
140
+ #
141
+ def set_deferred_status status, *args
142
+ cancel_timeout
143
+ @errbacks ||= nil
144
+ @callbacks ||= nil
145
+ @deferred_status = status
146
+ @deferred_args = args
147
+ case @deferred_status
148
+ when :succeeded
149
+ if @callbacks
150
+ while cb = @callbacks.pop
151
+ cb.call(*@deferred_args)
152
+ end
153
+ end
154
+ @errbacks.clear if @errbacks
155
+ when :failed
156
+ if @errbacks
157
+ while eb = @errbacks.pop
158
+ eb.call(*@deferred_args)
159
+ end
160
+ end
161
+ @callbacks.clear if @callbacks
162
+ end
163
+ end
164
+
165
+
166
+ # Setting a timeout on a Deferrable causes it to go into the failed state after
167
+ # the Timeout expires (passing no arguments to the object's errbacks).
168
+ # Setting the status at any time prior to a call to the expiration of the timeout
169
+ # will cause the timer to be cancelled.
170
+ def timeout seconds, *args
171
+ cancel_timeout
172
+ me = self
173
+ @deferred_timeout = EventMachine::Timer.new(seconds) {me.fail(*args)}
174
+ self
175
+ end
176
+
177
+ # Cancels an outstanding timeout if any. Undoes the action of #timeout.
178
+ #
179
+ def cancel_timeout
180
+ @deferred_timeout ||= nil
181
+ if @deferred_timeout
182
+ @deferred_timeout.cancel
183
+ @deferred_timeout = nil
184
+ end
185
+ end
186
+
187
+
188
+ # Sugar for set_deferred_status(:succeeded, ...)
189
+ #
190
+ def succeed *args
191
+ set_deferred_status :succeeded, *args
192
+ end
193
+ alias set_deferred_success succeed
194
+
195
+ # Sugar for set_deferred_status(:failed, ...)
196
+ #
197
+ def fail *args
198
+ set_deferred_status :failed, *args
199
+ end
200
+ alias set_deferred_failure fail
201
+ end
202
+
203
+
204
+ # DefaultDeferrable is an otherwise empty class that includes Deferrable.
205
+ # This is very useful when you just need to return a Deferrable object
206
+ # as a way of communicating deferred status to some other part of a program.
207
+ class DefaultDeferrable
208
+ include Deferrable
209
+ end
210
+ end
@@ -0,0 +1,2 @@
1
+ warn "EM::Deferrable::Pool is deprecated, please use EM::Pool"
2
+ EM::Deferrable::Pool = EM::Pool
@@ -0,0 +1,73 @@
1
+ module EventMachine
2
+ # Utility class that is useful for file monitoring. Supported events are
3
+ #
4
+ # * File is modified
5
+ # * File is deleted
6
+ # * File is moved
7
+ #
8
+ # @note On Mac OS X, file watching only works when kqueue is enabled
9
+ #
10
+ # @see EventMachine.watch_file
11
+ class FileWatch < Connection
12
+ # @private
13
+ Cmodified = 'modified'.freeze
14
+ # @private
15
+ Cdeleted = 'deleted'.freeze
16
+ # @private
17
+ Cmoved = 'moved'.freeze
18
+
19
+
20
+ # @private
21
+ def receive_data(data)
22
+ case data
23
+ when Cmodified
24
+ file_modified
25
+ when Cdeleted
26
+ file_deleted
27
+ when Cmoved
28
+ file_moved
29
+ end
30
+ end
31
+
32
+ # Returns the path that is being monitored.
33
+ #
34
+ # @note Current implementation does not pick up on the new filename after a rename occurs.
35
+ #
36
+ # @return [String]
37
+ # @see EventMachine.watch_file
38
+ def path
39
+ @path
40
+ end
41
+
42
+ # Will be called when the file is modified. Supposed to be redefined by subclasses.
43
+ #
44
+ # @abstract
45
+ def file_modified
46
+ end
47
+
48
+ # Will be called when the file is deleted. Supposed to be redefined by subclasses.
49
+ # When the file is deleted, stop_watching will be called after this to make sure everything is
50
+ # cleaned up correctly.
51
+ #
52
+ # @note On Linux (with {http://en.wikipedia.org/wiki/Inotify inotify}), this method will not be called until *all* open file descriptors to
53
+ # the file have been closed.
54
+ #
55
+ # @abstract
56
+ def file_deleted
57
+ end
58
+
59
+ # Will be called when the file is moved or renamed. Supposed to be redefined by subclasses.
60
+ #
61
+ # @abstract
62
+ def file_moved
63
+ end
64
+
65
+ # Discontinue monitoring of the file.
66
+ #
67
+ # This involves cleaning up the underlying monitoring details with kqueue/inotify, and in turn firing {EventMachine::Connection#unbind}.
68
+ # This will be called automatically when a file is deleted. User code may call it as well.
69
+ def stop_watching
70
+ EventMachine::unwatch_filename(@signature)
71
+ end # stop_watching
72
+ end # FileWatch
73
+ end # EventMachine
@@ -0,0 +1,61 @@
1
+ #--
2
+ #
3
+ # Author:: Francis Cianfrocca (gmail: blackhedd)
4
+ # Homepage:: http://rubyeventmachine.com
5
+ # Date:: 16 Jul 2006
6
+ #
7
+ # See EventMachine and EventMachine::Connection for documentation and
8
+ # usage examples.
9
+ #
10
+ #----------------------------------------------------------------------------
11
+ #
12
+ # Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
13
+ # Gmail: blackhedd
14
+ #
15
+ # This program is free software; you can redistribute it and/or modify
16
+ # it under the terms of either: 1) the GNU General Public License
17
+ # as published by the Free Software Foundation; either version 2 of the
18
+ # License, or (at your option) any later version; or 2) Ruby's License.
19
+ #
20
+ # See the file COPYING for complete licensing information.
21
+ #
22
+ #---------------------------------------------------------------------------
23
+ #
24
+ #
25
+
26
+ #--
27
+ # This defines EventMachine::Deferrable#future, which requires
28
+ # that the rest of EventMachine::Deferrable has already been seen.
29
+ # (It's in deferrable.rb.)
30
+
31
+ module EventMachine
32
+ module Deferrable
33
+
34
+ # A future is a sugaring of a typical deferrable usage.
35
+ #--
36
+ # Evaluate arg (which may be an expression or a block).
37
+ # What's the class of arg?
38
+ # If arg is an ordinary expression, then return it.
39
+ # If arg is deferrable (responds to :set_deferred_status),
40
+ # then look at the arguments. If either callback or errback
41
+ # are defined, then use them. If neither are defined, then
42
+ # use the supplied block (if any) as the callback.
43
+ # Then return arg.
44
+ def self.future arg, cb=nil, eb=nil, &blk
45
+ arg = arg.call if arg.respond_to?(:call)
46
+
47
+ if arg.respond_to?(:set_deferred_status)
48
+ if cb || eb
49
+ arg.callback(&cb) if cb
50
+ arg.errback(&eb) if eb
51
+ else
52
+ arg.callback(&blk) if blk
53
+ end
54
+ end
55
+
56
+ arg
57
+ end
58
+
59
+ end
60
+ end
61
+
@@ -0,0 +1,68 @@
1
+ require 'em/streamer'
2
+
3
+ # Streams an IO object over a given connection. Streaming begins once the object is
4
+ # instantiated. Typically IOStreamer instances are not reused. IOStreamer includes
5
+ # a 16K buffer. If streaming from a file, FileStreamer is more efficient.
6
+ #
7
+ # @example
8
+ #
9
+ # module FileSender
10
+ # def post_init
11
+ # @example
12
+ #
13
+ # module SocketSender
14
+ # socket = TCPSocket.new('localhost', 2000)
15
+ # streamer = EventMachine::IOStreamer.new(self, socket)
16
+ # streamer.callback{
17
+ # # all data was sent successfully
18
+ # close_connection_after_writing
19
+ # }
20
+ # end
21
+ # end
22
+
23
+ # Stream from any IO object, similar to FileStreamer
24
+ module EventMachine
25
+ class IOStreamer
26
+ include Deferrable
27
+ CHUNK_SIZE = 16384
28
+
29
+ # @param [EventMachine::Connection] connection
30
+ # @param [IO] io Data source
31
+ #
32
+ # @option opts [Boolean] :http_chunks (false) Use HTTP 1.1 style chunked-encoding semantics.
33
+ def initialize(connection, io, opts = {})
34
+ @connection = connection
35
+ @io = io
36
+ @http_chunks = opts[:http_chunks]
37
+
38
+ @buff = String.new
39
+ @io.binmode if @io.respond_to?(:binmode)
40
+ stream_one_chunk
41
+ end
42
+
43
+ private
44
+
45
+ # Used internally to stream one chunk at a time over multiple reactor ticks
46
+ # @private
47
+ def stream_one_chunk
48
+ loop do
49
+ if @io.eof?
50
+ @connection.send_data "0\r\n\r\n" if @http_chunks
51
+ succeed
52
+ break
53
+ end
54
+
55
+ if @connection.respond_to?(:get_outbound_data_size) && (@connection.get_outbound_data_size > FileStreamer::BackpressureLevel)
56
+ EventMachine::next_tick { stream_one_chunk }
57
+ break
58
+ end
59
+
60
+ if @io.read(CHUNK_SIZE, @buff)
61
+ @connection.send_data("#{@buff.length.to_s(16)}\r\n") if @http_chunks
62
+ @connection.send_data(@buff)
63
+ @connection.send_data("\r\n") if @http_chunks
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,252 @@
1
+ module EventMachine
2
+ # A simple iterator for concurrent asynchronous work.
3
+ #
4
+ # Unlike ruby's built-in iterators, the end of the current iteration cycle is signaled manually,
5
+ # instead of happening automatically after the yielded block finishes executing. For example:
6
+ #
7
+ # (0..10).each{ |num| }
8
+ #
9
+ # becomes:
10
+ #
11
+ # EM::Iterator.new(0..10).each{ |num,iter| iter.next }
12
+ #
13
+ # This is especially useful when doing asynchronous work via reactor libraries and
14
+ # functions. For example, given a sync and async http api:
15
+ #
16
+ # response = sync_http_get(url); ...
17
+ # async_http_get(url){ |response| ... }
18
+ #
19
+ # a synchronous iterator such as:
20
+ #
21
+ # responses = urls.map{ |url| sync_http_get(url) }
22
+ # ...
23
+ # puts 'all done!'
24
+ #
25
+ # could be written as:
26
+ #
27
+ # EM::Iterator.new(urls).map(proc{ |url,iter|
28
+ # async_http_get(url){ |res|
29
+ # iter.return(res)
30
+ # }
31
+ # }, proc{ |responses|
32
+ # ...
33
+ # puts 'all done!'
34
+ # })
35
+ #
36
+ # Now, you can take advantage of the asynchronous api to issue requests in parallel. For example,
37
+ # to fetch 10 urls at a time, simply pass in a concurrency of 10:
38
+ #
39
+ # EM::Iterator.new(urls, 10).each do |url,iter|
40
+ # async_http_get(url){ iter.next }
41
+ # end
42
+ #
43
+ class Iterator
44
+ Stop = "EM::Stop"
45
+ # Create a new parallel async iterator with specified concurrency.
46
+ #
47
+ # i = EM::Iterator.new(1..100, 10)
48
+ #
49
+ # will create an iterator over the range that processes 10 items at a time. Iteration
50
+ # is started via #each, #map or #inject
51
+ #
52
+ # The list may either be an array-like object, or a proc that returns a new object
53
+ # to be processed each time it is called. If a proc is used, it must return
54
+ # EventMachine::Iterator::Stop to signal the end of the iterations.
55
+ #
56
+ def initialize(list, concurrency = 1)
57
+ raise ArgumentError, 'concurrency must be bigger than zero' unless (concurrency > 0)
58
+ if list.respond_to?(:call)
59
+ @list = nil
60
+ @list_proc = list
61
+ elsif list.respond_to?(:to_a)
62
+ @list = list.to_a.dup
63
+ @list_proc = nil
64
+ else
65
+ raise ArgumentError, 'argument must be a proc or an array'
66
+ end
67
+ @concurrency = concurrency
68
+
69
+ @started = false
70
+ @ended = false
71
+ end
72
+
73
+ # Change the concurrency of this iterator. Workers will automatically be spawned or destroyed
74
+ # to accomodate the new concurrency level.
75
+ #
76
+ def concurrency=(val)
77
+ old = @concurrency
78
+ @concurrency = val
79
+
80
+ spawn_workers if val > old and @started and !@ended
81
+ end
82
+ attr_reader :concurrency
83
+
84
+ # Iterate over a set of items using the specified block or proc.
85
+ #
86
+ # EM::Iterator.new(1..100).each do |num, iter|
87
+ # puts num
88
+ # iter.next
89
+ # end
90
+ #
91
+ # An optional second proc is invoked after the iteration is complete.
92
+ #
93
+ # EM::Iterator.new(1..100).each(
94
+ # proc{ |num,iter| iter.next },
95
+ # proc{ puts 'all done' }
96
+ # )
97
+ #
98
+ def each(foreach=nil, after=nil, &blk)
99
+ raise ArgumentError, 'proc or block required for iteration' unless foreach ||= blk
100
+ raise RuntimeError, 'cannot iterate over an iterator more than once' if @started or @ended
101
+
102
+ @started = true
103
+ @pending = 0
104
+ @workers = 0
105
+
106
+ all_done = proc{
107
+ after.call if after and @ended and @pending == 0
108
+ }
109
+
110
+ @process_next = proc{
111
+ # p [:process_next, :pending=, @pending, :workers=, @workers, :ended=, @ended, :concurrency=, @concurrency, :list=, @list]
112
+ unless @ended or @workers > @concurrency
113
+ item = next_item()
114
+ if item.equal?(Stop)
115
+ @ended = true
116
+ @workers -= 1
117
+ all_done.call
118
+ else
119
+ @pending += 1
120
+
121
+ is_done = false
122
+ on_done = proc{
123
+ raise RuntimeError, 'already completed this iteration' if is_done
124
+ is_done = true
125
+
126
+ @pending -= 1
127
+
128
+ if @ended
129
+ all_done.call
130
+ else
131
+ EM.next_tick(@process_next)
132
+ end
133
+ }
134
+ class << on_done
135
+ alias :next :call
136
+ end
137
+
138
+ foreach.call(item, on_done)
139
+ end
140
+ else
141
+ @workers -= 1
142
+ end
143
+ }
144
+
145
+ spawn_workers
146
+
147
+ self
148
+ end
149
+
150
+ # Collect the results of an asynchronous iteration into an array.
151
+ #
152
+ # EM::Iterator.new(%w[ pwd uptime uname date ], 2).map(proc{ |cmd,iter|
153
+ # EM.system(cmd){ |output,status|
154
+ # iter.return(output)
155
+ # }
156
+ # }, proc{ |results|
157
+ # p results
158
+ # })
159
+ #
160
+ def map(foreach, after)
161
+ index = 0
162
+
163
+ inject([], proc{ |results,item,iter|
164
+ i = index
165
+ index += 1
166
+
167
+ is_done = false
168
+ on_done = proc{ |res|
169
+ raise RuntimeError, 'already returned a value for this iteration' if is_done
170
+ is_done = true
171
+
172
+ results[i] = res
173
+ iter.return(results)
174
+ }
175
+ class << on_done
176
+ alias :return :call
177
+ def next
178
+ raise NoMethodError, 'must call #return on a map iterator'
179
+ end
180
+ end
181
+
182
+ foreach.call(item, on_done)
183
+ }, proc{ |results|
184
+ after.call(results)
185
+ })
186
+ end
187
+
188
+ # Inject the results of an asynchronous iteration onto a given object.
189
+ #
190
+ # EM::Iterator.new(%w[ pwd uptime uname date ], 2).inject({}, proc{ |hash,cmd,iter|
191
+ # EM.system(cmd){ |output,status|
192
+ # hash[cmd] = status.exitstatus == 0 ? output.strip : nil
193
+ # iter.return(hash)
194
+ # }
195
+ # }, proc{ |results|
196
+ # p results
197
+ # })
198
+ #
199
+ def inject(obj, foreach, after)
200
+ each(proc{ |item,iter|
201
+ is_done = false
202
+ on_done = proc{ |res|
203
+ raise RuntimeError, 'already returned a value for this iteration' if is_done
204
+ is_done = true
205
+
206
+ obj = res
207
+ iter.next
208
+ }
209
+ class << on_done
210
+ alias :return :call
211
+ def next
212
+ raise NoMethodError, 'must call #return on an inject iterator'
213
+ end
214
+ end
215
+
216
+ foreach.call(obj, item, on_done)
217
+ }, proc{
218
+ after.call(obj)
219
+ })
220
+ end
221
+
222
+ private
223
+
224
+ # Spawn workers to consume items from the iterator's enumerator based on the current concurrency level.
225
+ #
226
+ def spawn_workers
227
+ EM.next_tick(start_worker = proc{
228
+ if @workers < @concurrency and !@ended
229
+ # p [:spawning_worker, :workers=, @workers, :concurrency=, @concurrency, :ended=, @ended]
230
+ @workers += 1
231
+ @process_next.call
232
+ EM.next_tick(start_worker)
233
+ end
234
+ })
235
+ nil
236
+ end
237
+
238
+ # Return the next item from @list or @list_proc.
239
+ # Once items have run out, will return EM::Iterator::Stop. Procs must supply this themselves
240
+ def next_item
241
+ if @list_proc
242
+ @list_proc.call
243
+ else
244
+ @list.empty? ? Stop : @list.shift
245
+ end
246
+ end
247
+ end
248
+ end
249
+
250
+ # TODO: pass in one object instead of two? .each{ |iter| puts iter.current; iter.next }
251
+ # TODO: support iter.pause/resume/stop/break/continue?
252
+ # TODO: create some exceptions instead of using RuntimeError