asir 0.2.0 → 1.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (82) hide show
  1. data/Gemfile +1 -2
  2. data/README.textile +4 -2
  3. data/VERSION +1 -1
  4. data/asir.gemspec +1 -4
  5. data/asir.riterate.yml +1 -0
  6. data/bin/asir +2 -1
  7. data/example/asir_control.sh +63 -1
  8. data/example/asir_control_client_http.rb +2 -2
  9. data/example/asir_control_client_resque.rb +16 -0
  10. data/example/asir_control_client_zmq.rb +3 -3
  11. data/example/config/asir_config.rb +20 -8
  12. data/example/ex02.rb +1 -1
  13. data/example/ex03.rb +2 -2
  14. data/example/ex04.rb +2 -2
  15. data/example/ex05.rb +1 -1
  16. data/example/ex06.rb +6 -5
  17. data/example/ex07.rb +2 -2
  18. data/example/ex08.rb +2 -2
  19. data/example/ex09.rb +2 -2
  20. data/example/ex10.rb +2 -2
  21. data/example/ex11.rb +5 -5
  22. data/example/ex12.rb +6 -6
  23. data/example/ex13.rb +4 -4
  24. data/example/ex14.rb +4 -4
  25. data/example/ex15.rb +2 -2
  26. data/example/ex16.rb +8 -8
  27. data/example/ex17.rb +12 -11
  28. data/example/ex18.rb +5 -5
  29. data/example/ex19.rb +3 -3
  30. data/example/ex20.rb +3 -3
  31. data/example/ex21.rb +3 -3
  32. data/example/ex22.rb +1 -1
  33. data/example/ex23.rb +2 -2
  34. data/example/ex24.rb +4 -4
  35. data/example/ex25.rb +41 -0
  36. data/example/example_helper.rb +38 -3
  37. data/example/sample_service.rb +4 -4
  38. data/hack_night/exercise/prob-3.rb +3 -3
  39. data/hack_night/exercise/prob-6.rb +2 -2
  40. data/hack_night/exercise/prob-7.rb +2 -2
  41. data/hack_night/solution/prob-2.rb +2 -2
  42. data/hack_night/solution/prob-3.rb +3 -3
  43. data/hack_night/solution/prob-6.rb +7 -6
  44. data/hack_night/solution/prob-7.rb +6 -6
  45. data/{spec → lab}/const_get_speed_spec.rb +0 -0
  46. data/lib/asir.rb +29 -7
  47. data/lib/asir/additional_data.rb +25 -0
  48. data/lib/asir/channel.rb +4 -5
  49. data/lib/asir/client.rb +29 -13
  50. data/lib/asir/config.rb +8 -0
  51. data/lib/asir/description.rb +34 -0
  52. data/lib/asir/environment.rb +96 -0
  53. data/lib/asir/error.rb +4 -1
  54. data/lib/asir/invoker.rb +14 -0
  55. data/lib/asir/main.rb +84 -103
  56. data/lib/asir/message.rb +1 -1
  57. data/lib/asir/poll_throttle.rb +53 -0
  58. data/lib/asir/retry_behavior.rb +1 -1
  59. data/lib/asir/thread_variable.rb +183 -0
  60. data/lib/asir/transport.rb +36 -23
  61. data/lib/asir/transport/beanstalk.rb +18 -52
  62. data/lib/asir/transport/conduit.rb +42 -0
  63. data/lib/asir/transport/connection_oriented.rb +32 -56
  64. data/lib/asir/transport/delegation.rb +5 -5
  65. data/lib/asir/transport/demux.rb +33 -0
  66. data/lib/asir/transport/file.rb +5 -3
  67. data/lib/asir/transport/payload_io.rb +8 -4
  68. data/lib/asir/transport/resque.rb +212 -0
  69. data/lib/asir/transport/stream.rb +19 -9
  70. data/lib/asir/transport/tcp_socket.rb +3 -2
  71. data/lib/asir/transport/zmq.rb +14 -17
  72. data/lib/asir/uri_config.rb +51 -0
  73. data/lib/asir/version.rb +1 -1
  74. data/spec/client_spec.rb +48 -0
  75. data/spec/demux_spec.rb +38 -0
  76. data/spec/json_spec.rb +0 -2
  77. data/spec/message_spec.rb +68 -0
  78. data/spec/performance_spec.rb +66 -0
  79. data/spec/spec_helper.rb +34 -0
  80. data/spec/thread_variable_spec.rb +135 -0
  81. data/spec/transport_spec.rb +82 -0
  82. metadata +28 -4
@@ -1,5 +1,7 @@
1
1
  require 'time'
2
+ require 'asir/thread_variable'
2
3
  require 'asir/message/delay'
4
+ require 'asir/transport/conduit'
3
5
 
4
6
  module ASIR
5
7
  # !SLIDE
@@ -11,7 +13,7 @@ module ASIR
11
13
  # Service: Send the Result to the Client.
12
14
  # Client: Receive the Result from the Service.
13
15
  class Transport
14
- include Log, Initialization, AdditionalData, Message::Delay
16
+ include Log, Initialization, AdditionalData, Message::Delay, ThreadVariable, Conduit
15
17
 
16
18
  attr_accessor :encoder, :decoder, :one_way
17
19
 
@@ -22,8 +24,8 @@ module ASIR
22
24
  # * Receive decoded Result.
23
25
  def send_message message
24
26
  @message_count ||= 0; @message_count += 1
25
- message.create_timestamp! if needs_message_timestamp?
26
- message.create_identifier! if needs_message_identifier?
27
+ message.create_timestamp! if needs_message_timestamp? message
28
+ message.create_identifier! if needs_message_identifier? message
27
29
  @before_send_message.call(self, message) if @before_send_message
28
30
  relative_message_delay! message
29
31
  message_payload = encoder.dup.encode(message)
@@ -57,13 +59,6 @@ module ASIR
57
59
  # Send Result to stream.
58
60
  def send_result result, stream, message_state
59
61
  message = result.message
60
- if @on_result_exception && result.exception
61
- begin
62
- @on_result_exception.call(self, result)
63
- rescue ::Exception => exc
64
- _log { [ :send_result, :result, result, :on_result_exception, exc ] }
65
- end
66
- end
67
62
  if @one_way && message.block
68
63
  message.block.call(result)
69
64
  else
@@ -85,7 +80,7 @@ module ASIR
85
80
  result = decoder.dup.decode(result_payload)
86
81
  if result && ! message.one_way
87
82
  if exc = result.exception
88
- exc.invoke!
83
+ invoker.invoke!(exc, self)
89
84
  else
90
85
  if ! @one_way && message.block
91
86
  message.block.call(result)
@@ -105,27 +100,27 @@ module ASIR
105
100
  attr_accessor :message_count
106
101
 
107
102
  # A Proc to call within #receive_message, after #_receive_message.
108
- # trans.after_receiver_message(trans, message)
103
+ # trans.after_receive_message(trans, message)
109
104
  attr_accessor :after_receive_message
110
105
 
111
106
  # A Proc to call within #send_message, before #_send_message.
112
107
  # trans.before_send_message(trans, message)
113
108
  attr_accessor :before_send_message
114
109
 
115
- # Proc to call after #_send_result if result.exception.
110
+ # Proc to call with #invoke_message! if result.exception.
116
111
  # trans.on_result_exception.call(trans, result)
117
112
  attr_accessor :on_result_exception
118
113
 
119
114
  # Proc to call with exception, if exception occurs within #serve_message!, but outside
120
115
  # Message#invoke!.
121
116
  #
122
- # trans.on_exception.call(trans, exception, :message, Message_instance)
117
+ # trans.on_exception.call(trans, exception, :message, Message_instance, nil)
123
118
  # trans.on_exception.call(trans, exception, :result, Message_instance, Result_instance)
124
119
  attr_accessor :on_exception
125
120
 
126
121
  attr_accessor :needs_message_identifier, :needs_message_timestamp
127
- alias :needs_message_identifier? :needs_message_identifier
128
- alias :needs_message_timestamp? :needs_message_timestamp
122
+ def needs_message_identifier? m; @needs_message_identifier; end
123
+ def needs_message_timestamp? m; @needs_message_timestamp; end
129
124
 
130
125
  attr_accessor :verbose
131
126
 
@@ -154,7 +149,7 @@ module ASIR
154
149
  rescue ::Exception => exc
155
150
  exception = original_exception = exc
156
151
  _log [ :message_error, exc ]
157
- @on_exception.call(self, exc, :message, message) if @on_exception
152
+ @on_exception.call(self, exc, :message, message, nil) if @on_exception
158
153
  ensure
159
154
  begin
160
155
  if message_ok
@@ -180,6 +175,7 @@ module ASIR
180
175
 
181
176
  # !SLIDE
182
177
  # Transport Server Support
178
+ attr_accessor :running
183
179
 
184
180
  def stop! force = false
185
181
  @running = false
@@ -225,12 +221,29 @@ module ASIR
225
221
 
226
222
  # Invokes the Message object, returns a Result object.
227
223
  def invoke_message! message
228
- _processing_message = @processing_message
229
- @processing_message = true
230
- wait_for_delay! message
231
- message.invoke!
232
- ensure
233
- @processing_message = _processing_message
224
+ result = nil
225
+ Transport.with_attr! :current, self do
226
+ with_attr! :message, message do
227
+ wait_for_delay! message
228
+ result = invoker.invoke!(message, self)
229
+ # Hook for Exceptions.
230
+ if @on_result_exception && result.exception
231
+ @on_result_exception.call(self, result)
232
+ end
233
+ end
234
+ end
235
+ result
236
+ end
237
+ # The current Message being handled.
238
+ attr_accessor_thread :message
239
+
240
+ # The current active Transport.
241
+ cattr_accessor_thread :current
242
+
243
+ # The Invoker responsible for invoking the Message.
244
+ attr_accessor :invoker
245
+ def invoker
246
+ @invoker ||= Invoker.new
234
247
  end
235
248
 
236
249
  # !SLIDE END
@@ -140,9 +140,8 @@ module ASIR
140
140
 
141
141
  # !SLIDE
142
142
  # Beanstalk Server
143
-
144
- def prepare_beanstalk_server!
145
- _log { "prepare_beanstalk_server! #{uri}" } if @verbose >= 1
143
+ def _server!
144
+ _log { "_server! #{uri}" } if @verbose >= 1
146
145
  @server = connect!(:try_max => nil,
147
146
  :try_sleep => 1,
148
147
  :try_sleep_increment => 0.1,
@@ -155,61 +154,28 @@ module ASIR
155
154
  end
156
155
  self
157
156
  end
158
- alias :prepare_server! :prepare_beanstalk_server!
159
-
160
- def run_beanstalk_server!
161
- _log :run_beanstalk_server! if @verbose >= 1
162
- with_server_signals! do
163
- @running = true
164
- while @running
165
- prepare_beanstalk_server! unless @server
166
- # Same socket for both in and out stream.
167
- serve_stream! @server, @server
168
- end
169
- end
170
- self
171
- ensure
172
- _server_close!
173
- end
174
- alias :run_server! :run_beanstalk_server!
175
-
176
- def serve_stream! in_stream, out_stream
177
- while @running
178
- begin
179
- serve_stream_message! in_stream, out_stream
180
- rescue ::Exception => exc
181
- _log [ :serve_stream_error, exc ]
182
- @running = false
183
- end
184
- end
185
- self
186
- ensure
187
- _server_close!
157
+
158
+ def _server_accept_connection! server
159
+ prepare_server! unless @server
160
+ [ @server, @server ]
188
161
  end
189
162
 
190
- def start_beanstalkd!
191
- _log { "run_beanstalkd! #{uri}" } if @verbose >= 1
192
- raise "already running #{@beanstalkd_pid}" if @beanstalkd_pid
193
- addr = @address ? "-l #{@address} " : ""
194
- cmd = "beanstalkd #{addr}-p #{port}"
195
- @beanstalkd_pid = Process.fork do
196
- _log { "Start beanstalkd: #{cmd} ..." } if @verbose >= 1
197
- exec(cmd)
198
- raise "exec #{cmd.inspect} failed"
199
- end
200
- _log { "Start beanstalkd: #{cmd} pid=#{@beanstalkd_pid.inspect}" } if @verbose >= 2
201
- self
163
+ def _server_close_connection! in_stream, out_stream
164
+ # NOTHING
202
165
  end
203
166
 
204
- def stop_beanstalkd!
205
- _log { "stop_beanstalkd! #{uri} pid=#{@beanstalkd_pid.inspect}" } if @verbose >= 1
206
- Process.kill 'TERM', @beanstalkd_pid
207
- Process.waitpid @beanstalkd_pid
208
- self
209
- ensure
210
- @beanstalkd_pid = nil
167
+ def stream_eof? stream
168
+ # Note: stream.eof? on a beanstalkd connection,
169
+ # will cause blocking read *forever* because
170
+ # beanstalk connections are long lived.
171
+ false
211
172
  end
212
173
 
174
+ def _start_conduit!
175
+ addr = address ? "-l #{address} " : ""
176
+ cmd = "beanstalkd #{addr}-p #{port}"
177
+ exec(cmd)
178
+ end
213
179
  end
214
180
  # !SLIDE END
215
181
  end # class
@@ -0,0 +1,42 @@
1
+ require 'asir/transport'
2
+
3
+ module Asir
4
+ class Transport
5
+ # Conduit service support.
6
+ module Conduit
7
+ def start_conduit! options = nil
8
+ opts = { :fork => true }
9
+ opts.update(options) if options
10
+ _log { "start_conduit! #{self}" } if @verbose >= 1
11
+ in_fork = opts[:fork]
12
+ raise "already running #{@conduit_pid} #{@conduit_cmd}" if @conduit_pid
13
+ if in_fork
14
+ @conduit_pid = ::Process.fork do
15
+ _log { "start_conduit! #{self} starting pid=#{$$.inspect}" } if @verbose >= 2
16
+ _start_conduit!
17
+ raise "Could not exec"
18
+ end
19
+ _log { "start_conduit! #{self} started pid=#{@conduit_pid.inspect}" } if @verbose >= 2
20
+ else
21
+ _start_conduit!
22
+ end
23
+ self
24
+ end
25
+
26
+ def stop_conduit!
27
+ if @conduit_pid
28
+ ::Process.kill 'TERM', @conduit_pid
29
+ ::Process.waitpid @conduit_pid
30
+ # File.unlink @redis_conf
31
+ end
32
+ self
33
+ ensure
34
+ @conduit_pid = nil
35
+ end
36
+
37
+ def _start_conduit!
38
+ raise Error::SubclassResponsibility
39
+ end
40
+ end
41
+ end
42
+ end
@@ -1,51 +1,13 @@
1
1
  require 'asir/transport/stream'
2
2
  require 'asir/transport/payload_io'
3
- require 'uri'
3
+ require 'asir/uri_config'
4
4
 
5
5
  module ASIR
6
6
  class Transport
7
7
  # !SLIDE
8
8
  # Connection-Oriented Transport
9
9
  class ConnectionOriented < Stream
10
- include PayloadIO
11
-
12
- attr_accessor :uri, :scheme, :port, :address
13
- alias :protocol :scheme
14
- alias :protocol= :scheme=
15
-
16
- def uri
17
- @uri ||= "#{scheme}://#{address}:#{port}"
18
- end
19
-
20
- def scheme
21
- @scheme ||=
22
- case
23
- when @uri
24
- URI.parse(@uri).scheme
25
- else
26
- 'tcp'.freeze
27
- end
28
- end
29
-
30
- def address
31
- @address ||=
32
- case
33
- when @uri
34
- URI.parse(@uri).host
35
- else
36
- '127.0.0.1'.freeze
37
- end
38
- end
39
-
40
- def port
41
- @port ||=
42
- case
43
- when @uri
44
- URI.parse(@uri).port
45
- else
46
- raise Error, "#{self.class}: port not set."
47
- end
48
- end
10
+ include PayloadIO, UriConfig
49
11
 
50
12
  # !SLIDE
51
13
  # Returns a connected Channel.
@@ -70,12 +32,12 @@ module ASIR
70
32
  # Returns raw client stream.
71
33
  def _connect!
72
34
  _log { "_connect! #{uri}" } if @verbose >= 1
73
- sock = _client_connect!
74
- _log { "_connect! socket=#{sock}" } if @verbose >= 1
75
- _after_connect! sock
76
- sock
35
+ stream = _client_connect!
36
+ _log { "_connect! stream=#{stream}" } if @verbose >= 1
37
+ _after_connect! stream
38
+ stream
77
39
  rescue ::Exception => err
78
- raise Error, "Cannot connect to #{self.class} #{uri}: #{err.inspect}", err.backtrace
40
+ raise err.class, "Cannot connect to #{self.class} #{uri}: #{err.inspect}", err.backtrace
79
41
  end
80
42
 
81
43
  # Subclasses can override.
@@ -128,30 +90,43 @@ module ASIR
128
90
  _server!
129
91
  rescue ::Exception => err
130
92
  _log [ "prepare_server! #{uri}", :exception, err ]
131
- raise Error, "Cannot prepare server on #{self.class} #{uri}: #{err.inspect}", err.backtrace
93
+ raise err.class, "Cannot prepare server on #{self.class} #{uri}: #{err.inspect}", err.backtrace
132
94
  end
133
95
 
134
96
  def run_server!
135
97
  _log { "run_server! #{uri}" } if @verbose >= 1
136
98
  with_server_signals! do
137
99
  @running = true
100
+ server_on_start!
138
101
  while @running
139
- stream = _server_accept_connection! @server
140
- _log { "run_server!: connected" } if @verbose >= 1
141
- begin
142
- # Same socket for both in and out stream.
143
- serve_stream! stream, stream
144
- ensure
145
- _server_close_connection!(stream)
146
- end
147
- _log { "run_server!: disconnected" } if @verbose >= 1
102
+ serve_connection!
148
103
  end
149
104
  end
150
105
  self
151
106
  ensure
107
+ server_on_stop!
152
108
  _server_close!
153
109
  end
154
110
 
111
+ def server_on_start!
112
+ end
113
+
114
+ def server_on_stop!
115
+ end
116
+
117
+ def serve_connection!
118
+ in_stream, out_stream = _server_accept_connection! @server
119
+ _log { "serve_connection!: connected #{in_stream} #{out_stream}" } if @verbose >= 1
120
+ _server_serve_stream! in_stream, out_stream
121
+ rescue Error::Terminate => err
122
+ @running = false
123
+ _log [ :serve_connection_terminate, err ]
124
+ ensure
125
+ _server_close_connection!(in_stream, out_stream)
126
+ _log { "serve_connection!: disconnected" } if @verbose >= 1
127
+ end
128
+ alias :_server_serve_stream! :_serve_stream!
129
+
155
130
  def _server!
156
131
  raise Error::SubclassResponsibility, "_server!"
157
132
  end
@@ -165,12 +140,13 @@ module ASIR
165
140
  end
166
141
 
167
142
  # Accept a client connection.
143
+ # Returns [ in_stream, out_stream ].
168
144
  def _server_accept_connection! server
169
145
  raise Error::SubclassResponsibility, "_server_accept_connection!"
170
146
  end
171
147
 
172
148
  # Close a client connection.
173
- def _server_close_connection! stream
149
+ def _server_close_connection! in_stream, out_stream
174
150
  raise Error::SubclassResponsibility, "_server_close_connection!"
175
151
  end
176
152
  end
@@ -24,17 +24,17 @@ module ASIR
24
24
  opaque_result
25
25
  end
26
26
 
27
- def needs_message_identifier?
27
+ def needs_message_identifier? message
28
28
  @needs_message_identifier ||
29
- transports.any? { | t | t.needs_message_identifier? }
29
+ transports.any? { | t | t.needs_message_identifier?(message) }
30
30
  end
31
31
 
32
- def needs_message_timestamp?
32
+ def needs_message_timestamp? message
33
33
  @needs_message_timestamp ||
34
- transports.any? { | t | t.needs_message_timestamp? }
34
+ transports.any? { | t | t.needs_message_timestamp?(message) }
35
35
  end
36
36
 
37
- # Subclasses with multiple transport should override this method.
37
+ # Subclasses with multiple transports should override this method.
38
38
  def transports
39
39
  @transports ||= [ transport ]
40
40
  end
@@ -0,0 +1,33 @@
1
+ require 'asir/transport/delegation'
2
+
3
+ module ASIR
4
+ class Transport
5
+ # !SLIDE
6
+ # Select a Transport based on a Proc.
7
+ class Demux < self
8
+ include Delegation
9
+
10
+ # Proc returning actual transport to use.
11
+ # transport_proc.call(transport, message)
12
+ attr_accessor :transport_proc
13
+
14
+ # Only active during #send_message.
15
+ attr_accessor_thread :transport
16
+
17
+ # Support for Delegation mixin.
18
+ def transports; [ transport ]; end
19
+
20
+ def send_message message
21
+ with_attr! :transport, transport_proc.call(self, message) do
22
+ super
23
+ end
24
+ end
25
+
26
+ def _send_message message, message_payload
27
+ transport.send_message(message)
28
+ end
29
+ end
30
+ # !SLIDE END
31
+ end
32
+ end
33
+