asir 0.2.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. data/Gemfile +1 -2
  2. data/README.textile +4 -2
  3. data/VERSION +1 -1
  4. data/asir.gemspec +1 -4
  5. data/asir.riterate.yml +1 -0
  6. data/bin/asir +2 -1
  7. data/example/asir_control.sh +63 -1
  8. data/example/asir_control_client_http.rb +2 -2
  9. data/example/asir_control_client_resque.rb +16 -0
  10. data/example/asir_control_client_zmq.rb +3 -3
  11. data/example/config/asir_config.rb +20 -8
  12. data/example/ex02.rb +1 -1
  13. data/example/ex03.rb +2 -2
  14. data/example/ex04.rb +2 -2
  15. data/example/ex05.rb +1 -1
  16. data/example/ex06.rb +6 -5
  17. data/example/ex07.rb +2 -2
  18. data/example/ex08.rb +2 -2
  19. data/example/ex09.rb +2 -2
  20. data/example/ex10.rb +2 -2
  21. data/example/ex11.rb +5 -5
  22. data/example/ex12.rb +6 -6
  23. data/example/ex13.rb +4 -4
  24. data/example/ex14.rb +4 -4
  25. data/example/ex15.rb +2 -2
  26. data/example/ex16.rb +8 -8
  27. data/example/ex17.rb +12 -11
  28. data/example/ex18.rb +5 -5
  29. data/example/ex19.rb +3 -3
  30. data/example/ex20.rb +3 -3
  31. data/example/ex21.rb +3 -3
  32. data/example/ex22.rb +1 -1
  33. data/example/ex23.rb +2 -2
  34. data/example/ex24.rb +4 -4
  35. data/example/ex25.rb +41 -0
  36. data/example/example_helper.rb +38 -3
  37. data/example/sample_service.rb +4 -4
  38. data/hack_night/exercise/prob-3.rb +3 -3
  39. data/hack_night/exercise/prob-6.rb +2 -2
  40. data/hack_night/exercise/prob-7.rb +2 -2
  41. data/hack_night/solution/prob-2.rb +2 -2
  42. data/hack_night/solution/prob-3.rb +3 -3
  43. data/hack_night/solution/prob-6.rb +7 -6
  44. data/hack_night/solution/prob-7.rb +6 -6
  45. data/{spec → lab}/const_get_speed_spec.rb +0 -0
  46. data/lib/asir.rb +29 -7
  47. data/lib/asir/additional_data.rb +25 -0
  48. data/lib/asir/channel.rb +4 -5
  49. data/lib/asir/client.rb +29 -13
  50. data/lib/asir/config.rb +8 -0
  51. data/lib/asir/description.rb +34 -0
  52. data/lib/asir/environment.rb +96 -0
  53. data/lib/asir/error.rb +4 -1
  54. data/lib/asir/invoker.rb +14 -0
  55. data/lib/asir/main.rb +84 -103
  56. data/lib/asir/message.rb +1 -1
  57. data/lib/asir/poll_throttle.rb +53 -0
  58. data/lib/asir/retry_behavior.rb +1 -1
  59. data/lib/asir/thread_variable.rb +183 -0
  60. data/lib/asir/transport.rb +36 -23
  61. data/lib/asir/transport/beanstalk.rb +18 -52
  62. data/lib/asir/transport/conduit.rb +42 -0
  63. data/lib/asir/transport/connection_oriented.rb +32 -56
  64. data/lib/asir/transport/delegation.rb +5 -5
  65. data/lib/asir/transport/demux.rb +33 -0
  66. data/lib/asir/transport/file.rb +5 -3
  67. data/lib/asir/transport/payload_io.rb +8 -4
  68. data/lib/asir/transport/resque.rb +212 -0
  69. data/lib/asir/transport/stream.rb +19 -9
  70. data/lib/asir/transport/tcp_socket.rb +3 -2
  71. data/lib/asir/transport/zmq.rb +14 -17
  72. data/lib/asir/uri_config.rb +51 -0
  73. data/lib/asir/version.rb +1 -1
  74. data/spec/client_spec.rb +48 -0
  75. data/spec/demux_spec.rb +38 -0
  76. data/spec/json_spec.rb +0 -2
  77. data/spec/message_spec.rb +68 -0
  78. data/spec/performance_spec.rb +66 -0
  79. data/spec/spec_helper.rb +34 -0
  80. data/spec/thread_variable_spec.rb +135 -0
  81. data/spec/transport_spec.rb +82 -0
  82. metadata +28 -4
@@ -1,5 +1,7 @@
1
1
  require 'time'
2
+ require 'asir/thread_variable'
2
3
  require 'asir/message/delay'
4
+ require 'asir/transport/conduit'
3
5
 
4
6
  module ASIR
5
7
  # !SLIDE
@@ -11,7 +13,7 @@ module ASIR
11
13
  # Service: Send the Result to the Client.
12
14
  # Client: Receive the Result from the Service.
13
15
  class Transport
14
- include Log, Initialization, AdditionalData, Message::Delay
16
+ include Log, Initialization, AdditionalData, Message::Delay, ThreadVariable, Conduit
15
17
 
16
18
  attr_accessor :encoder, :decoder, :one_way
17
19
 
@@ -22,8 +24,8 @@ module ASIR
22
24
  # * Receive decoded Result.
23
25
  def send_message message
24
26
  @message_count ||= 0; @message_count += 1
25
- message.create_timestamp! if needs_message_timestamp?
26
- message.create_identifier! if needs_message_identifier?
27
+ message.create_timestamp! if needs_message_timestamp? message
28
+ message.create_identifier! if needs_message_identifier? message
27
29
  @before_send_message.call(self, message) if @before_send_message
28
30
  relative_message_delay! message
29
31
  message_payload = encoder.dup.encode(message)
@@ -57,13 +59,6 @@ module ASIR
57
59
  # Send Result to stream.
58
60
  def send_result result, stream, message_state
59
61
  message = result.message
60
- if @on_result_exception && result.exception
61
- begin
62
- @on_result_exception.call(self, result)
63
- rescue ::Exception => exc
64
- _log { [ :send_result, :result, result, :on_result_exception, exc ] }
65
- end
66
- end
67
62
  if @one_way && message.block
68
63
  message.block.call(result)
69
64
  else
@@ -85,7 +80,7 @@ module ASIR
85
80
  result = decoder.dup.decode(result_payload)
86
81
  if result && ! message.one_way
87
82
  if exc = result.exception
88
- exc.invoke!
83
+ invoker.invoke!(exc, self)
89
84
  else
90
85
  if ! @one_way && message.block
91
86
  message.block.call(result)
@@ -105,27 +100,27 @@ module ASIR
105
100
  attr_accessor :message_count
106
101
 
107
102
  # A Proc to call within #receive_message, after #_receive_message.
108
- # trans.after_receiver_message(trans, message)
103
+ # trans.after_receive_message(trans, message)
109
104
  attr_accessor :after_receive_message
110
105
 
111
106
  # A Proc to call within #send_message, before #_send_message.
112
107
  # trans.before_send_message(trans, message)
113
108
  attr_accessor :before_send_message
114
109
 
115
- # Proc to call after #_send_result if result.exception.
110
+ # Proc to call with #invoke_message! if result.exception.
116
111
  # trans.on_result_exception.call(trans, result)
117
112
  attr_accessor :on_result_exception
118
113
 
119
114
  # Proc to call with exception, if exception occurs within #serve_message!, but outside
120
115
  # Message#invoke!.
121
116
  #
122
- # trans.on_exception.call(trans, exception, :message, Message_instance)
117
+ # trans.on_exception.call(trans, exception, :message, Message_instance, nil)
123
118
  # trans.on_exception.call(trans, exception, :result, Message_instance, Result_instance)
124
119
  attr_accessor :on_exception
125
120
 
126
121
  attr_accessor :needs_message_identifier, :needs_message_timestamp
127
- alias :needs_message_identifier? :needs_message_identifier
128
- alias :needs_message_timestamp? :needs_message_timestamp
122
+ def needs_message_identifier? m; @needs_message_identifier; end
123
+ def needs_message_timestamp? m; @needs_message_timestamp; end
129
124
 
130
125
  attr_accessor :verbose
131
126
 
@@ -154,7 +149,7 @@ module ASIR
154
149
  rescue ::Exception => exc
155
150
  exception = original_exception = exc
156
151
  _log [ :message_error, exc ]
157
- @on_exception.call(self, exc, :message, message) if @on_exception
152
+ @on_exception.call(self, exc, :message, message, nil) if @on_exception
158
153
  ensure
159
154
  begin
160
155
  if message_ok
@@ -180,6 +175,7 @@ module ASIR
180
175
 
181
176
  # !SLIDE
182
177
  # Transport Server Support
178
+ attr_accessor :running
183
179
 
184
180
  def stop! force = false
185
181
  @running = false
@@ -225,12 +221,29 @@ module ASIR
225
221
 
226
222
  # Invokes the Message object, returns a Result object.
227
223
  def invoke_message! message
228
- _processing_message = @processing_message
229
- @processing_message = true
230
- wait_for_delay! message
231
- message.invoke!
232
- ensure
233
- @processing_message = _processing_message
224
+ result = nil
225
+ Transport.with_attr! :current, self do
226
+ with_attr! :message, message do
227
+ wait_for_delay! message
228
+ result = invoker.invoke!(message, self)
229
+ # Hook for Exceptions.
230
+ if @on_result_exception && result.exception
231
+ @on_result_exception.call(self, result)
232
+ end
233
+ end
234
+ end
235
+ result
236
+ end
237
+ # The current Message being handled.
238
+ attr_accessor_thread :message
239
+
240
+ # The current active Transport.
241
+ cattr_accessor_thread :current
242
+
243
+ # The Invoker responsible for invoking the Message.
244
+ attr_accessor :invoker
245
+ def invoker
246
+ @invoker ||= Invoker.new
234
247
  end
235
248
 
236
249
  # !SLIDE END
@@ -140,9 +140,8 @@ module ASIR
140
140
 
141
141
  # !SLIDE
142
142
  # Beanstalk Server
143
-
144
- def prepare_beanstalk_server!
145
- _log { "prepare_beanstalk_server! #{uri}" } if @verbose >= 1
143
+ def _server!
144
+ _log { "_server! #{uri}" } if @verbose >= 1
146
145
  @server = connect!(:try_max => nil,
147
146
  :try_sleep => 1,
148
147
  :try_sleep_increment => 0.1,
@@ -155,61 +154,28 @@ module ASIR
155
154
  end
156
155
  self
157
156
  end
158
- alias :prepare_server! :prepare_beanstalk_server!
159
-
160
- def run_beanstalk_server!
161
- _log :run_beanstalk_server! if @verbose >= 1
162
- with_server_signals! do
163
- @running = true
164
- while @running
165
- prepare_beanstalk_server! unless @server
166
- # Same socket for both in and out stream.
167
- serve_stream! @server, @server
168
- end
169
- end
170
- self
171
- ensure
172
- _server_close!
173
- end
174
- alias :run_server! :run_beanstalk_server!
175
-
176
- def serve_stream! in_stream, out_stream
177
- while @running
178
- begin
179
- serve_stream_message! in_stream, out_stream
180
- rescue ::Exception => exc
181
- _log [ :serve_stream_error, exc ]
182
- @running = false
183
- end
184
- end
185
- self
186
- ensure
187
- _server_close!
157
+
158
+ def _server_accept_connection! server
159
+ prepare_server! unless @server
160
+ [ @server, @server ]
188
161
  end
189
162
 
190
- def start_beanstalkd!
191
- _log { "run_beanstalkd! #{uri}" } if @verbose >= 1
192
- raise "already running #{@beanstalkd_pid}" if @beanstalkd_pid
193
- addr = @address ? "-l #{@address} " : ""
194
- cmd = "beanstalkd #{addr}-p #{port}"
195
- @beanstalkd_pid = Process.fork do
196
- _log { "Start beanstalkd: #{cmd} ..." } if @verbose >= 1
197
- exec(cmd)
198
- raise "exec #{cmd.inspect} failed"
199
- end
200
- _log { "Start beanstalkd: #{cmd} pid=#{@beanstalkd_pid.inspect}" } if @verbose >= 2
201
- self
163
+ def _server_close_connection! in_stream, out_stream
164
+ # NOTHING
202
165
  end
203
166
 
204
- def stop_beanstalkd!
205
- _log { "stop_beanstalkd! #{uri} pid=#{@beanstalkd_pid.inspect}" } if @verbose >= 1
206
- Process.kill 'TERM', @beanstalkd_pid
207
- Process.waitpid @beanstalkd_pid
208
- self
209
- ensure
210
- @beanstalkd_pid = nil
167
+ def stream_eof? stream
168
+ # Note: stream.eof? on a beanstalkd connection,
169
+ # will cause blocking read *forever* because
170
+ # beanstalk connections are long lived.
171
+ false
211
172
  end
212
173
 
174
+ def _start_conduit!
175
+ addr = address ? "-l #{address} " : ""
176
+ cmd = "beanstalkd #{addr}-p #{port}"
177
+ exec(cmd)
178
+ end
213
179
  end
214
180
  # !SLIDE END
215
181
  end # class
@@ -0,0 +1,42 @@
1
+ require 'asir/transport'
2
+
3
+ module Asir
4
+ class Transport
5
+ # Conduit service support.
6
+ module Conduit
7
+ def start_conduit! options = nil
8
+ opts = { :fork => true }
9
+ opts.update(options) if options
10
+ _log { "start_conduit! #{self}" } if @verbose >= 1
11
+ in_fork = opts[:fork]
12
+ raise "already running #{@conduit_pid} #{@conduit_cmd}" if @conduit_pid
13
+ if in_fork
14
+ @conduit_pid = ::Process.fork do
15
+ _log { "start_conduit! #{self} starting pid=#{$$.inspect}" } if @verbose >= 2
16
+ _start_conduit!
17
+ raise "Could not exec"
18
+ end
19
+ _log { "start_conduit! #{self} started pid=#{@conduit_pid.inspect}" } if @verbose >= 2
20
+ else
21
+ _start_conduit!
22
+ end
23
+ self
24
+ end
25
+
26
+ def stop_conduit!
27
+ if @conduit_pid
28
+ ::Process.kill 'TERM', @conduit_pid
29
+ ::Process.waitpid @conduit_pid
30
+ # File.unlink @redis_conf
31
+ end
32
+ self
33
+ ensure
34
+ @conduit_pid = nil
35
+ end
36
+
37
+ def _start_conduit!
38
+ raise Error::SubclassResponsibility
39
+ end
40
+ end
41
+ end
42
+ end
@@ -1,51 +1,13 @@
1
1
  require 'asir/transport/stream'
2
2
  require 'asir/transport/payload_io'
3
- require 'uri'
3
+ require 'asir/uri_config'
4
4
 
5
5
  module ASIR
6
6
  class Transport
7
7
  # !SLIDE
8
8
  # Connection-Oriented Transport
9
9
  class ConnectionOriented < Stream
10
- include PayloadIO
11
-
12
- attr_accessor :uri, :scheme, :port, :address
13
- alias :protocol :scheme
14
- alias :protocol= :scheme=
15
-
16
- def uri
17
- @uri ||= "#{scheme}://#{address}:#{port}"
18
- end
19
-
20
- def scheme
21
- @scheme ||=
22
- case
23
- when @uri
24
- URI.parse(@uri).scheme
25
- else
26
- 'tcp'.freeze
27
- end
28
- end
29
-
30
- def address
31
- @address ||=
32
- case
33
- when @uri
34
- URI.parse(@uri).host
35
- else
36
- '127.0.0.1'.freeze
37
- end
38
- end
39
-
40
- def port
41
- @port ||=
42
- case
43
- when @uri
44
- URI.parse(@uri).port
45
- else
46
- raise Error, "#{self.class}: port not set."
47
- end
48
- end
10
+ include PayloadIO, UriConfig
49
11
 
50
12
  # !SLIDE
51
13
  # Returns a connected Channel.
@@ -70,12 +32,12 @@ module ASIR
70
32
  # Returns raw client stream.
71
33
  def _connect!
72
34
  _log { "_connect! #{uri}" } if @verbose >= 1
73
- sock = _client_connect!
74
- _log { "_connect! socket=#{sock}" } if @verbose >= 1
75
- _after_connect! sock
76
- sock
35
+ stream = _client_connect!
36
+ _log { "_connect! stream=#{stream}" } if @verbose >= 1
37
+ _after_connect! stream
38
+ stream
77
39
  rescue ::Exception => err
78
- raise Error, "Cannot connect to #{self.class} #{uri}: #{err.inspect}", err.backtrace
40
+ raise err.class, "Cannot connect to #{self.class} #{uri}: #{err.inspect}", err.backtrace
79
41
  end
80
42
 
81
43
  # Subclasses can override.
@@ -128,30 +90,43 @@ module ASIR
128
90
  _server!
129
91
  rescue ::Exception => err
130
92
  _log [ "prepare_server! #{uri}", :exception, err ]
131
- raise Error, "Cannot prepare server on #{self.class} #{uri}: #{err.inspect}", err.backtrace
93
+ raise err.class, "Cannot prepare server on #{self.class} #{uri}: #{err.inspect}", err.backtrace
132
94
  end
133
95
 
134
96
  def run_server!
135
97
  _log { "run_server! #{uri}" } if @verbose >= 1
136
98
  with_server_signals! do
137
99
  @running = true
100
+ server_on_start!
138
101
  while @running
139
- stream = _server_accept_connection! @server
140
- _log { "run_server!: connected" } if @verbose >= 1
141
- begin
142
- # Same socket for both in and out stream.
143
- serve_stream! stream, stream
144
- ensure
145
- _server_close_connection!(stream)
146
- end
147
- _log { "run_server!: disconnected" } if @verbose >= 1
102
+ serve_connection!
148
103
  end
149
104
  end
150
105
  self
151
106
  ensure
107
+ server_on_stop!
152
108
  _server_close!
153
109
  end
154
110
 
111
+ def server_on_start!
112
+ end
113
+
114
+ def server_on_stop!
115
+ end
116
+
117
+ def serve_connection!
118
+ in_stream, out_stream = _server_accept_connection! @server
119
+ _log { "serve_connection!: connected #{in_stream} #{out_stream}" } if @verbose >= 1
120
+ _server_serve_stream! in_stream, out_stream
121
+ rescue Error::Terminate => err
122
+ @running = false
123
+ _log [ :serve_connection_terminate, err ]
124
+ ensure
125
+ _server_close_connection!(in_stream, out_stream)
126
+ _log { "serve_connection!: disconnected" } if @verbose >= 1
127
+ end
128
+ alias :_server_serve_stream! :_serve_stream!
129
+
155
130
  def _server!
156
131
  raise Error::SubclassResponsibility, "_server!"
157
132
  end
@@ -165,12 +140,13 @@ module ASIR
165
140
  end
166
141
 
167
142
  # Accept a client connection.
143
+ # Returns [ in_stream, out_stream ].
168
144
  def _server_accept_connection! server
169
145
  raise Error::SubclassResponsibility, "_server_accept_connection!"
170
146
  end
171
147
 
172
148
  # Close a client connection.
173
- def _server_close_connection! stream
149
+ def _server_close_connection! in_stream, out_stream
174
150
  raise Error::SubclassResponsibility, "_server_close_connection!"
175
151
  end
176
152
  end
@@ -24,17 +24,17 @@ module ASIR
24
24
  opaque_result
25
25
  end
26
26
 
27
- def needs_message_identifier?
27
+ def needs_message_identifier? message
28
28
  @needs_message_identifier ||
29
- transports.any? { | t | t.needs_message_identifier? }
29
+ transports.any? { | t | t.needs_message_identifier?(message) }
30
30
  end
31
31
 
32
- def needs_message_timestamp?
32
+ def needs_message_timestamp? message
33
33
  @needs_message_timestamp ||
34
- transports.any? { | t | t.needs_message_timestamp? }
34
+ transports.any? { | t | t.needs_message_timestamp?(message) }
35
35
  end
36
36
 
37
- # Subclasses with multiple transport should override this method.
37
+ # Subclasses with multiple transports should override this method.
38
38
  def transports
39
39
  @transports ||= [ transport ]
40
40
  end
@@ -0,0 +1,33 @@
1
+ require 'asir/transport/delegation'
2
+
3
+ module ASIR
4
+ class Transport
5
+ # !SLIDE
6
+ # Select a Transport based on a Proc.
7
+ class Demux < self
8
+ include Delegation
9
+
10
+ # Proc returning actual transport to use.
11
+ # transport_proc.call(transport, message)
12
+ attr_accessor :transport_proc
13
+
14
+ # Only active during #send_message.
15
+ attr_accessor_thread :transport
16
+
17
+ # Support for Delegation mixin.
18
+ def transports; [ transport ]; end
19
+
20
+ def send_message message
21
+ with_attr! :transport, transport_proc.call(self, message) do
22
+ super
23
+ end
24
+ end
25
+
26
+ def _send_message message, message_payload
27
+ transport.send_message(message)
28
+ end
29
+ end
30
+ # !SLIDE END
31
+ end
32
+ end
33
+