arpie 0.0.3 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README CHANGED
@@ -6,6 +6,9 @@ your implementation details).
6
6
 
7
7
  Arpie also provides a robust replay-protected RPC framework.
8
8
 
9
+ The useful core of arpie is a protocol stack that can be used to read/split/assemble/write
10
+ any data stream, but is tailored for packeted streaming data.
11
+
9
12
  The Arpie server uses one ruby-thread per client, the client runs entirely in the
10
13
  calling thread; though an example implementation for evented callbacks is provided.
11
14
 
@@ -27,7 +30,7 @@ to get the newest version.
27
30
 
28
31
  server = TCPServer.new(51210)
29
32
 
30
- e = Arpie::Server.new(Arpie::MarshalProtocol.new)
33
+ e = Arpie::Server.new(Arpie::MarshalProtocol.new, Arpie::SizedProtocol.new)
31
34
 
32
35
  e.handle do |server, ep, msg|
33
36
  ep.write_message msg.reverse
@@ -37,7 +40,7 @@ to get the newest version.
37
40
  server.accept
38
41
  end
39
42
 
40
- c = Arpie::Client.new(Arpie::MarshalProtocol.new)
43
+ c = Arpie::Client.new(Arpie::MarshalProtocol.new, Arpie::SizedProtocol.new)
41
44
  c.connect do
42
45
  TCPSocket.new("127.0.0.1", 51210)
43
46
  end
@@ -60,7 +63,7 @@ to get the newest version.
60
63
 
61
64
  server = TCPServer.new(51210)
62
65
 
63
- e = Arpie::ProxyServer.new(Arpie::MarshalProtocol.new)
66
+ e = Arpie::ProxyServer.new(Arpie::MarshalProtocol.new, Arpie::SizedProtocol.new)
64
67
 
65
68
  e.handle MyHandler.new
66
69
 
@@ -68,7 +71,7 @@ to get the newest version.
68
71
  server.accept
69
72
  end
70
73
 
71
- p = Arpie::ProxyClient.new(Arpie::MarshalProtocol.new)
74
+ p = Arpie::ProxyClient.new(Arpie::MarshalProtocol.new, Arpie::SizedProtocol.new)
72
75
  p.connect do |transport|
73
76
  TCPSocket.new("127.0.0.1", 51210)
74
77
  end
data/Rakefile CHANGED
@@ -9,7 +9,7 @@ include FileUtils
9
9
  # Configuration
10
10
  ##############################################################################
11
11
  NAME = "arpie"
12
- VERS = "0.0.3"
12
+ VERS = "0.0.4"
13
13
  CLEAN.include ["**/.*.sw?", "pkg", ".config", "rdoc", "coverage"]
14
14
  RDOC_OPTS = ["--quiet", "--line-numbers", "--inline-source", '--title', \
15
15
  "#{NAME}: A high-performing layered networking protocol framework. Simple to use, simple to extend.", \
@@ -1,4 +1,5 @@
1
1
  require 'arpie/protocol'
2
+ require 'arpie/xmlrpc'
2
3
  require 'arpie/server'
3
4
  require 'arpie/client'
4
5
  require 'arpie/proxy'
@@ -7,6 +7,7 @@ module Arpie
7
7
  #
8
8
  # See README for examples.
9
9
  class Client
10
+ # The protocol chain used.
10
11
  attr_reader :protocol
11
12
 
12
13
  # How often should this Client retry a connection.
@@ -22,8 +23,8 @@ module Arpie
22
23
  # unnecessary load in case of network failure.
23
24
  attr_accessor :connect_sleep
24
25
 
25
- def initialize protocol
26
- @protocol = protocol
26
+ def initialize *protocols
27
+ @protocol = Arpie::ProtocolChain.new(*protocols)
27
28
  @read_io = nil
28
29
  @write_io = nil
29
30
  @connector = lambda { raise ArgumentError, "No connector specified, cannot connect to Endpoint." }
@@ -69,7 +70,7 @@ module Arpie
69
70
  # Receive a message. Blocks until received.
70
71
  def read_message
71
72
  io_retry do
72
- message = @protocol.read_message(@read_io)
73
+ return @protocol.read_message(@read_io)
73
74
  end
74
75
  end
75
76
 
@@ -144,8 +145,8 @@ module Arpie
144
145
  class RPCClient < Client
145
146
  private :read_message, :write_message
146
147
 
147
- def initialize protocol
148
- super(protocol)
148
+ def initialize *protocols
149
+ super(*protocols)
149
150
 
150
151
  @on_pre_call = lambda {|client, message| }
151
152
  @on_post_call = lambda {|client, message, reply| }
@@ -2,146 +2,305 @@ require 'shellwords'
2
2
  require 'yaml'
3
3
 
4
4
  module Arpie
5
+ MTU = 1024
6
+ # Raised by arpie when a Protocol thinks the stream got corrupted
7
+ # (by calling stream_error!).
8
+ # This usually results in a dropped connection.
9
+ class StreamError < IOError ; end
10
+ # Raised by arpie when a Protocol needs more data to parse a packet.
11
+ # Usually only of relevance to the programmer when using Protocol#from directly.
12
+ class EIncomplete < RuntimeError ; end
13
+
14
+ # :stopdoc:
15
+ # Used internally by arpie.
16
+ class ESwallow < RuntimeError ; end
17
+ class ETryAgain < RuntimeError ; end
18
+ class YieldResult < RuntimeError
19
+ attr_reader :result
20
+ def initialize result
21
+ @result = result
22
+ end
23
+ end
24
+ # :startdoc:
5
25
 
6
- # A Protocol converts messages (which are arbitary objects)
7
- # to a suitable on-the-wire format, and back.
8
- class Protocol
9
- MTU = 1024
26
+ # A RPC call. You need to wrap all calls sent over RPC protocols in this.
27
+ class RPCall < Struct.new(:ns, :meth, :argv, :uuid); end
10
28
 
11
- private_class_method :new
29
+ # A ProtocolChain wraps one or more Protocols to provide a parser
30
+ # list, into which io data can be fed and parsed packets received; and
31
+ # vice versa.
32
+ class ProtocolChain
12
33
 
13
- attr_reader :message
34
+ # Array of Protocols.
35
+ attr_reader :chain
14
36
 
15
- def initialize
16
- @message = nil
17
- @buffer = ""
18
- reset
19
- end
37
+ # String holding all read, but yet unparsed bytes.
38
+ attr_reader :buffer
20
39
 
21
- # Reads data from +io+. Returns true, if a whole
22
- # message has been read, or false if more data is needed.
23
- # The read message can be retrieved via Protocol#message.
24
- def read_partial io
25
- @buffer << io.readpartial(MTU)
40
+ # A buffer holding all parsed, but unreturned messages.
41
+ attr_reader :messages
26
42
 
27
- if idx = complete?(@buffer)
28
- @message = from @buffer[0, idx]
29
- @buffer = @buffer[idx, -1] || ""
30
- return true
31
- end
43
+ # The endpoint class of this Protocol.
44
+ # Defaults to Arpie::Endpoint
45
+ attr_accessor :endpoint_class
46
+
47
+ # Create a new Chain. Supply an Array of Protocol
48
+ # instances, where the leftmost is the innermost.
49
+ #
50
+ # Example:
51
+ # MarshalProtocol.new, SizedProtocol.new
52
+ # would wrap marshalled data inside SizedProtocol.
53
+ def initialize *protocols
54
+ protocols.size > 0 or raise ArgumentError, "Specify at least one protocol."
55
+ protocols[-1].class::CAN_SEPARATE_MESSAGES or
56
+ raise ArgumentError,
57
+ "The outermost protocol needs to be able to " +
58
+ "separate messages in a stream (#{protocols.inspect} does not)."
59
+
60
+ @endpoint_class = Arpie::Endpoint
32
61
 
33
- return false
62
+ @chain = protocols
63
+ @buffer = ""
64
+ @messages = []
65
+ end
66
+
67
+ # Convert the given +message+ to wire format by
68
+ # passing it through all protocols in the chain.
69
+ def to message
70
+ ret = @chain.inject(message) {|msg, p|
71
+ p.to(msg)
72
+ }
73
+ end
74
+
75
+ # Convert the given +binary+ to message format
76
+ # by passing it through all protocols in the chain.
77
+ # May raise EStreamError or EIncomplete, in the case that
78
+ # +binary+ does not satisfy one of the protocols.
79
+ #
80
+ # Returns an array of messages, even if only one message
81
+ # was contained.
82
+ def from binary
83
+ r, w = IO.pipe
84
+ w.write(binary)
85
+ w.close
86
+ results = []
87
+ results << read_message(r) until false rescue begin
88
+ r.close
89
+ return results
90
+ end
91
+ raise "Interal error: should not reach this."
34
92
  end
35
93
 
36
- # Read a message from +io+. Block until a message
37
- # has been received.
94
+ # Read a message from +io+. Block until all protocols
95
+ # agree that a message has been received.
96
+ #
38
97
  # Returns the message.
39
98
  def read_message io
40
- select([io]) until read_partial(io)
41
- @message
42
- end
99
+ return @messages.shift if @messages.size > 0
100
+
101
+ messages = [@buffer]
102
+ chain = @chain.reverse
103
+ p_index = 0
104
+
105
+ while p_index < chain.size do p = chain[p_index]
106
+ cut_to_index = nil
107
+ messages_for_next = []
108
+
109
+ messages.each do |message|
110
+ cut_to_index = p.from(message) do |object|
111
+ messages_for_next << object
112
+ end rescue case $!
113
+
114
+ when YieldResult
115
+ messages_for_next.concat($!.result)
116
+ next
117
+
118
+ when ESwallow
119
+ messages.delete(message)
120
+ messages_for_next = messages
121
+ p_index -= 1
122
+ break
123
+
124
+ when EIncomplete
125
+ # All protocols above the io one need to wait for each
126
+ # one above to yield more messages.
127
+ if p_index > 0
128
+ # Unwind to the parent protocol and let it read in some
129
+ # more messages ..
130
+ messages_for_next = messages
131
+ messages_for_next.shift
132
+ p_index -= 1
133
+ break
134
+
135
+ # The first protocol manages +io+.
136
+ else
137
+ select([io])
138
+ @buffer << io.readpartial(MTU) rescue raise $!.class,
139
+ "#{$!.to_s}; unparseable bytes remaining in buffer: #{@buffer.size}"
140
+ retry
141
+ end
142
+
143
+ when ETryAgain
144
+ retry
43
145
 
44
- def write_raw_partial io, message
45
- io.write(message)
46
- end
146
+ else
147
+ raise
148
+ end # rescue case
149
+ end # messages.each
47
150
 
48
- # Write +message+ to +io+.
49
- def write_message io, message
50
- io.write(to message)
51
- end
151
+ raise "BUG: #{p.class.to_s}#from did not yield a message." if
152
+ messages_for_next.size == 0
52
153
 
53
- # Convert obj to on-the-wire format.
54
- def to obj
55
- obj
56
- end
154
+ messages = messages_for_next
57
155
 
58
- # Convert obj from on-the-wire-format.
59
- def from obj
60
- obj
156
+ if p_index == 0
157
+ if cut_to_index.nil? || cut_to_index < 0
158
+ raise "Protocol '#{p.class.to_s}'implementation faulty: " +
159
+ "from did return an invalid cut index: #{cut_to_index.inspect}."
160
+ else
161
+ @buffer[0, cut_to_index] = ""
162
+ end
163
+ end
164
+
165
+ p_index += 1
166
+ end # chain loop
167
+
168
+ message = messages.shift
169
+ @messages = messages
170
+ message
61
171
  end
62
172
 
63
- # Returns a Fixnum if the given obj contains a complete message.
64
- # The Fixnum is the index up to where the message runs; the rest
65
- # is assumed to be (part of) the next message.
66
- # Returns nil if obj does not describe a complete message (eg,
67
- # more data needs to be read).
68
- def complete? obj
69
- nil
173
+ # Write +message+ to +io+.
174
+ def write_message io, message
175
+ io.write(to message)
70
176
  end
71
177
 
72
- # Reset all state buffers. This is usually called
73
- # when the underlying connection drops, and any half-read
74
- # messages need to be discarded.
75
178
  def reset
76
- @message = nil
77
179
  @buffer = ""
78
180
  end
79
-
80
- def endpoint_klass
81
- Arpie::Endpoint
82
- end
83
181
  end
84
182
 
85
- # A simple separator-based protocol. This can be used to implement
86
- # newline-delimited communication.
87
- class SeparatorProtocol < Protocol
88
- public_class_method :new
89
-
90
- attr_accessor :separator
91
-
92
- def initialize separator = "\n"
93
- super()
94
- @separator = separator
95
- end
96
-
97
- def complete? obj
98
- obj.index(@separator)
99
- end
100
-
101
- def from obj
102
- obj.gsub(/#{Regexp.escape(@separator)}$/, "")
103
- end
183
+ # A Protocol converts messages (which are arbitary objects)
184
+ # to a suitable on-the-wire format, and back.
185
+ class Protocol
104
186
 
105
- def to obj
106
- obj + @separator
107
- end
108
- end
187
+ # Set this to true in child classes which implement
188
+ # message separation within a stream.
189
+ CAN_SEPARATE_MESSAGES = false
109
190
 
110
- # A linebased-protocol, which does shellwords-escaping/joining
111
- # on the lines; messages sent are arrays of parameters.
112
- # Note that all parameters are expected to be strings.
113
- class ShellwordsProtocol < SeparatorProtocol
191
+ # Convert obj to on-the-wire format.
114
192
  def to obj
115
- super Shellwords.join(obj)
193
+ obj
116
194
  end
117
195
 
118
- def from obj
119
- Shellwords.shellwords(super obj)
196
+ # Extract message(s) from +binary+.i
197
+ #
198
+ # Yields each message found, with all protocol-specifics stripped.
199
+ #
200
+ # Should call +incomplete+ when no message can be read yet.
201
+ #
202
+ # Must not block by waiting for multiple messages if a message
203
+ # can be yielded directly.
204
+ #
205
+ # Must not return without calling +incomplete+ or yielding a message.
206
+ #
207
+ # Must return the number of bytes these message(s) occupied in the stream,
208
+ # for truncating of the same.
209
+ # Mandatory when CAN_SEPARATE_MESSAGES is true for this class, but ignored
210
+ # otherwise.
211
+ def from binary, &block #:yields: message
212
+ yield binary
213
+ 0
214
+ end
215
+
216
+ # Call this within Protocol#from to reparse the current
217
+ # message.
218
+ def again!
219
+ raise ETryAgain
220
+ end
221
+
222
+ # Tell the protocol chain that the given chunk of data
223
+ # is not enough to construct a whole message.
224
+ # This breaks out of Protocol#from.
225
+ def incomplete!
226
+ raise EIncomplete
227
+ end
228
+
229
+ # Swallow the complete message currently passed to Protocol#from.
230
+ # Not advised for the outermost protocol, which works on io buffer streams
231
+ # and may swallow more than intended.
232
+ def gulp!
233
+ raise ESwallow
234
+ end
235
+ alias_method :swallow!, :gulp!
236
+
237
+ # Stow away a message in this protocols buffer for later reassembly.
238
+ # Optional argument: a token if you are planning to reassemble multiple
239
+ # interleaved/fragmented message streams.
240
+ #
241
+ # +binary+ is the binary packet you want to add to the assembly
242
+ # +token+ is a object which can be used to re-identify multiple concurrent assemblies
243
+ # +meta+ is a hash containing meta-information for this assembly
244
+ # each call to assemble! will merge these hashes, and pass them
245
+ # on to Protocol#assemble
246
+ def assemble! binary, token = :default, meta = {}
247
+ @stowbuffer ||= {}
248
+ @stowbuffer[token] ||= []
249
+ @stowbuffer[token] << binary
250
+
251
+ @metabuffer ||= {}
252
+ @metabuffer[token] ||= {}
253
+ @metabuffer[token].merge(meta)
254
+
255
+ assembled = []
256
+
257
+ assemble @stowbuffer[token], token, meta do |a|
258
+ assembled << a
259
+ end # rescue case $!
260
+ #when EIncomplet
261
+ # puts "Cant reassemble, asking for moarh"
262
+ # raise EIncomplete
263
+ #else
264
+ # raise
265
+ #end
266
+
267
+ assembled.size > 0 or raise "assemble! did not return any results."
268
+ raise YieldResult, assembled
269
+ end
270
+
271
+ # Called when we're trying to reassemble a stream of packets.
272
+ #
273
+ # Call incomplete! when not enough data is here to reassemble this stream,
274
+ # and yield all results of the reassembled stream.
275
+ def assemble binaries, token
276
+ raise NotImplementedError, "Tried to assemble! something, but no assembler defined."
277
+ end
278
+
279
+ # Call this if you think the stream has been corrupted, or
280
+ # non-protocol data arrived.
281
+ # +message+ is the text to display.
282
+ # +data+ is the optional misbehaving data for printing.
283
+ # This breaks out of Protocol#from.
284
+ def bogon! data = nil, message = nil
285
+ raise StreamError, "#{self.class.to_s}#{message.nil? ? " thinks the data is bogus" : ": " + message }#{data.nil? ? "" : ": " + data.inspect}."
120
286
  end
121
287
  end
122
288
 
289
+
123
290
  # A sample binary protocol, which simply prefixes each message with the
124
291
  # size of the data to be expected.
125
292
  class SizedProtocol < Protocol
126
- public_class_method :new
127
-
128
- def initialize
129
- super
130
- @max_message_size = 1024 * 1024
131
- end
293
+ CAN_SEPARATE_MESSAGES = true
132
294
 
133
- def complete? obj
134
- sz = obj.unpack("Q")[0]
135
- obj.size == sz + 8 ? sz + 8 : nil
295
+ def from binary
296
+ sz = binary.unpack('Q')[0] or incomplete!
297
+ binary.size >= sz + 8 or incomplete!
298
+ yield binary[8, sz]
299
+ 8 + sz
136
300
  end
137
301
 
138
- def from obj
139
- sz, data = obj.unpack("Qa*")
140
- data
141
- end
142
-
143
- def to obj
144
- [obj.size, obj].pack("Qa*")
302
+ def to object
303
+ [object.size, object].pack('Qa*')
145
304
  end
146
305
  end
147
306
 
@@ -149,132 +308,66 @@ module Arpie
149
308
  # this protocol. Served as an example, but a viable
150
309
  # choice for ruby-only production code.
151
310
  # Messages are arbitary objects.
152
- class MarshalProtocol < SizedProtocol
153
- def to obj
154
- super Marshal.dump(obj)
311
+ class MarshalProtocol < Protocol
312
+ def to object
313
+ Marshal.dump(object)
155
314
  end
156
315
 
157
- def from obj
158
- Marshal.load(super obj)
316
+ def from binary
317
+ yield Marshal.load(binary)
159
318
  end
160
319
  end
161
320
 
162
- # A protocol which encodes objects into YAML representation.
163
- # Messages are arbitary yaml-encodable objects.
164
- class YAMLProtocol < Arpie::Protocol
165
- public_class_method :new
166
-
167
- def complete? obj
168
- obj =~ /\.\.\.$/
169
- end
170
-
171
- def to obj
172
- YAML.dump(obj) + "...\n"
173
- end
321
+ # A simple separator-based protocol. This can be used to implement
322
+ # newline-delimited communication.
323
+ class SeparatorProtocol < Protocol
324
+ CAN_SEPARATE_MESSAGES = true
325
+ attr_accessor :separator
174
326
 
175
- def from obj
176
- YAML.load(obj)
327
+ def initialize separator = "\n"
328
+ @separator = separator
177
329
  end
178
- end
179
330
 
180
- # A RPC Protocol encapsulates RPCProtocol::Call
181
- # messages.
182
- class RPCProtocol < Protocol
331
+ def from binary
332
+ idx = binary.index(@separator) or incomplete!
333
+ yield binary[0, idx]
183
334
 
184
- # A RPC call.
185
- class Call < Struct.new(:ns, :meth, :argv, :uuid); end
186
- end
187
-
188
- # A XMLRPC Protocol based on rubys xmlrpc stdlib.
189
- # This does not encode HTTP headers; usage together with
190
- # a real webserver is advised.
191
- class XMLRPCProtocol < RPCProtocol
192
- public_class_method :new
193
-
194
- require 'xmlrpc/create'
195
- require 'xmlrpc/parser'
196
- require 'xmlrpc/config'
197
-
198
- VALID_MODES = [:client, :server].freeze
199
-
200
- attr_reader :mode
201
- attr_accessor :writer
202
- attr_accessor :parser
203
-
204
- def initialize mode, writer = XMLRPC::Create, parser = XMLRPC::XMLParser::REXMLStreamParser
205
- super()
206
- raise ArgumentError, "Not a valid mode, expecting one of #{VALID_MODES.inspect}" unless
207
- VALID_MODES.index(mode)
208
-
209
- @mode = mode
210
- @writer = writer.new
211
- @parser = parser.new
335
+ @separator.size + idx
212
336
  end
213
337
 
214
- def to obj
215
- case @mode
216
- when :client
217
- @writer.methodCall(obj.ns + obj.meth, *obj.argv)
218
-
219
- when :server
220
- case obj
221
- when Exception
222
- # TODO: wrap XMLFault
223
- else
224
- @writer.methodResponse(true, obj)
225
- end
226
- end
338
+ def to object
339
+ object.to_s + @separator
227
340
  end
341
+ end
228
342
 
229
- def from obj
230
- case @mode
231
- when :client
232
- @parser.parseMethodResponse(obj)[1]
233
-
234
- when :server
235
- vv = @parser.parseMethodCall(obj)
236
- RPCProtocol::Call.new('', vv[0], vv[1])
237
- end
343
+ # A linebased-protocol, which does shellwords-escaping/joining
344
+ # on the lines; messages sent are arrays of parameters.
345
+ # Note that all parameters are expected to be strings.
346
+ class ShellwordsProtocol < Protocol
347
+ def to object
348
+ raise ArgumentError, "#{self.class.to_s} can only encode arrays." unless
349
+ object.is_a?(Array)
350
+ Shellwords.join(object.map {|x| x.to_s })
238
351
  end
239
352
 
240
- def complete? obj
241
- case @mode
242
- when :client
243
- obj.index("</methodResponse>")
244
- when :server
245
- obj.index("</methodCall>")
246
- end
353
+ def from binary
354
+ yield Shellwords.shellwords(binary)
247
355
  end
248
356
  end
249
357
 
250
- # This simulates a very basic HTTP XMLRPC client/server.
251
- # It is not recommended to use this with production code.
252
- class HTTPXMLRPCProtocol < XMLRPCProtocol
253
- def to obj
254
- r = super
255
- case @mode
256
- when :client
257
- "GET / HTTP/1.[01]\r\nContent-Length: #{r.size}\r\n\r\n" + r
258
- when :server
259
- "HTTP/1.0 200 OK\r\nContent-Length: #{r.size}\r\n\r\n" + r
260
- end
261
- end
358
+ # A protocol which encodes objects into YAML representation.
359
+ # Messages are arbitary yaml-encodable objects.
360
+ class YAMLProtocol < Protocol
361
+ CAN_SEPARATE_MESSAGES = true
262
362
 
263
- def from obj
264
- # Simply strip all HTTP headers.
265
- header, obj = obj.split(/\r\n\r\n/, 2)
266
- super(obj)
363
+ def to object
364
+ YAML.dump(object) + "...\n"
267
365
  end
268
366
 
269
-
270
- def complete? obj
271
- # Complete if: has headers, has content-length, has data of content-length
272
- header, body = obj.split(/\r\n\r\n/, 2)
273
-
274
- header =~ /content-length:\s+(\d+)/i or return nil
275
-
276
- content_length = $1.to_i
277
- body.size == content_length ? header.size + 4 + body.size : nil
367
+ def from binary
368
+ index = binary =~ /^\.\.\.$/x or incomplete!
369
+ yield YAML.load(binary[0, index])
370
+ 4 + index
278
371
  end
279
372
  end
280
373
  end