object-stream 0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/COPYING +22 -0
- data/README.md +7 -0
- data/Rakefile +9 -0
- data/bench/pipe-socket.rb +52 -0
- data/examples/basic-usage.rb +43 -0
- data/examples/custom-class.rb +73 -0
- data/examples/maxbuf.rb +17 -0
- data/examples/outbox.rb +19 -0
- data/examples/pipe.rb +42 -0
- data/examples/read-without-block.rb +25 -0
- data/examples/slow-sender.rb +76 -0
- data/examples/socket.rb +43 -0
- data/examples/udp.rb +35 -0
- data/lib/object-stream-wrapper.rb +114 -0
- data/lib/object-stream.rb +324 -0
- data/test/test-basic.rb +176 -0
- data/test/test-consume.rb +51 -0
- data/test/test-expect.rb +80 -0
- data/test/test-inbox.rb +65 -0
- data/test/test-maxbuf.rb +22 -0
- data/test/test-outbox.rb +35 -0
- data/test/test-slow-sender.rb +79 -0
- metadata +110 -0
data/examples/udp.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
case ARGV[0]
|
2
|
+
when "marshal", "yaml", "json", "msgpack"
|
3
|
+
else
|
4
|
+
abort "Usage: #$0 marshal|yaml|json|msgpack"
|
5
|
+
end
|
6
|
+
|
7
|
+
type = ARGV.shift
|
8
|
+
|
9
|
+
require 'object-stream'
|
10
|
+
require 'socket'
|
11
|
+
|
12
|
+
Socket.do_not_reverse_lookup = true
|
13
|
+
s = UDPSocket.new; s.bind 'localhost', 0
|
14
|
+
t = UDPSocket.new; t.bind 'localhost', 0
|
15
|
+
s.connect *t.addr.values_at(2,1)
|
16
|
+
t.connect *s.addr.values_at(2,1)
|
17
|
+
|
18
|
+
th1 = Thread.new do
|
19
|
+
stream = ObjectStream.new(s, type: type)
|
20
|
+
10.times do |i|
|
21
|
+
stream << [i]
|
22
|
+
end
|
23
|
+
stream << "Bye."
|
24
|
+
end
|
25
|
+
|
26
|
+
th2 = Thread.new do
|
27
|
+
stream = ObjectStream.new(t, type: type)
|
28
|
+
stream.each do |obj|
|
29
|
+
p obj
|
30
|
+
break if /bye/i === obj
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
th1.join
|
35
|
+
th2.join
|
@@ -0,0 +1,114 @@
|
|
1
|
+
require 'object-stream'
|
2
|
+
|
3
|
+
# Utility wrapper for basic ObjectStream class. Adds three groups of
|
4
|
+
# functionality:
|
5
|
+
#
|
6
|
+
# * peer_name
|
7
|
+
#
|
8
|
+
# * expect
|
9
|
+
#
|
10
|
+
# * consume
|
11
|
+
#
|
12
|
+
class ObjectStreamWrapper
|
13
|
+
include Enumerable
|
14
|
+
|
15
|
+
# Not set by this library, but available for users to keep track of
|
16
|
+
# the peer in a symbolic, application-specific manner. See funl for
|
17
|
+
# an example.
|
18
|
+
attr_accessor :peer_name
|
19
|
+
|
20
|
+
def initialize *args, **opts
|
21
|
+
@stream = ObjectStream.new(*args, **opts)
|
22
|
+
@peer_name = "unknown"
|
23
|
+
@expected_class = nil
|
24
|
+
@consumers = []
|
25
|
+
unexpect
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_s
|
29
|
+
"#<Wrapped #{@stream.class} to #{peer_name}, io=#{@stream.inspect}>"
|
30
|
+
end
|
31
|
+
|
32
|
+
# Set the stream state so that subsequent objects returned by read will be
|
33
|
+
# instances of a custom class +cl+. Does not affect #consume.
|
34
|
+
# Class +cl+ should define cl.from_serialized, plus #to_json, #to_msgpack,
|
35
|
+
# etc. as needed by the underlying serialization library.
|
36
|
+
def expect cl
|
37
|
+
@expected_class = cl
|
38
|
+
end
|
39
|
+
|
40
|
+
# Turn off the custom class instantiation of #expect.
|
41
|
+
def unexpect; expect nil; end
|
42
|
+
|
43
|
+
# The block is appended to a queue of procs that are called for the
|
44
|
+
# subsequently read objects, instead of iterating over or returning them.
|
45
|
+
# Helps with handshake protocols. Not affected by #expect.
|
46
|
+
def consume &bl
|
47
|
+
@consumers << bl
|
48
|
+
end
|
49
|
+
|
50
|
+
def try_consume obj
|
51
|
+
if bl = @consumers.shift
|
52
|
+
bl[obj]
|
53
|
+
true
|
54
|
+
else
|
55
|
+
false
|
56
|
+
end
|
57
|
+
end
|
58
|
+
private :try_consume
|
59
|
+
|
60
|
+
def convert_to_expected obj
|
61
|
+
if @expected_class and not obj.kind_of? @expected_class
|
62
|
+
@expected_class.from_serialized(obj)
|
63
|
+
else
|
64
|
+
obj
|
65
|
+
end
|
66
|
+
end
|
67
|
+
private :convert_to_expected
|
68
|
+
|
69
|
+
def read
|
70
|
+
if block_given?
|
71
|
+
@stream.read do |obj|
|
72
|
+
try_consume(obj) or yield convert_to_expected(obj)
|
73
|
+
end
|
74
|
+
return nil
|
75
|
+
else
|
76
|
+
begin
|
77
|
+
obj = @stream.read
|
78
|
+
end while try_consume(obj)
|
79
|
+
convert_to_expected(obj)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def each
|
84
|
+
return to_enum unless block_given?
|
85
|
+
read {|obj| yield obj} until eof
|
86
|
+
rescue EOFError
|
87
|
+
end
|
88
|
+
|
89
|
+
def write *objects
|
90
|
+
@stream.write *objects
|
91
|
+
end
|
92
|
+
alias << write
|
93
|
+
|
94
|
+
def write_to_outbox *args, &bl
|
95
|
+
@stream.write_to_outbox *args, &bl
|
96
|
+
end
|
97
|
+
|
98
|
+
def eof?
|
99
|
+
@stream.eof?
|
100
|
+
end
|
101
|
+
alias eof eof?
|
102
|
+
|
103
|
+
def close
|
104
|
+
@stream.close
|
105
|
+
end
|
106
|
+
|
107
|
+
def closed?
|
108
|
+
@stream.closed?
|
109
|
+
end
|
110
|
+
|
111
|
+
def to_io
|
112
|
+
@stream.to_io
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,324 @@
|
|
1
|
+
# Stream of objects, with any underlying IO: File, Pipe, Socket, StringIO.
|
2
|
+
# Stream is bidirectional if the IO is bidirectional.
|
3
|
+
#
|
4
|
+
# Serializes objects using any of several serializers: marshal, yaml, json,
|
5
|
+
# msgpack. Works with select/readpartial if the serializer supports it (msgpack
|
6
|
+
# and yajl do).
|
7
|
+
#
|
8
|
+
# ObjectStream supports three styles of iteration: Enumerable, blocking read,
|
9
|
+
# and yielding (non-blocking) read.
|
10
|
+
module ObjectStream
|
11
|
+
include Enumerable
|
12
|
+
|
13
|
+
# The IO through which the stream reads and writes serialized object data.
|
14
|
+
attr_reader :io
|
15
|
+
|
16
|
+
# Number of outgoing objects that can accumulate before the outbox is
|
17
|
+
# serialized to the byte buffer (and possibly to the io).
|
18
|
+
attr_reader :max_outbox
|
19
|
+
|
20
|
+
MARSHAL_TYPE = "marshal".freeze
|
21
|
+
YAML_TYPE = "yaml".freeze
|
22
|
+
JSON_TYPE = "json".freeze
|
23
|
+
MSGPACK_TYPE = "msgpack".freeze
|
24
|
+
|
25
|
+
TYPES = [
|
26
|
+
MARSHAL_TYPE, YAML_TYPE, JSON_TYPE, MSGPACK_TYPE
|
27
|
+
]
|
28
|
+
|
29
|
+
DEFAULT_MAX_OUTBOX = 10
|
30
|
+
|
31
|
+
# Raised when maxbuf exceeded.
|
32
|
+
class OverflowError < StandardError; end
|
33
|
+
|
34
|
+
@stream_class_map =
|
35
|
+
Hash.new {|h,type| raise ArgumentError, "unknown type: #{type.inspect}"}
|
36
|
+
@mutex = Mutex.new
|
37
|
+
|
38
|
+
class << self
|
39
|
+
def new io, type: MARSHAL_TYPE, **opts
|
40
|
+
if io.kind_of? ObjectStream
|
41
|
+
raise ArgumentError,
|
42
|
+
"given io is already an ObjectStream: #{io.inspect}"
|
43
|
+
end
|
44
|
+
stream_class_for(type).new io, **opts
|
45
|
+
end
|
46
|
+
|
47
|
+
def stream_class_for type
|
48
|
+
cl = @stream_class_map[type]
|
49
|
+
return cl if cl.respond_to? :new
|
50
|
+
|
51
|
+
# Protect against race condition in msgpack and yajl extension
|
52
|
+
# initialization (bug #8374).
|
53
|
+
@mutex.synchronize do
|
54
|
+
return cl if cl.respond_to? :new
|
55
|
+
@stream_class_map[type] = cl.call
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def register_type type, &bl
|
60
|
+
@stream_class_map[type] = bl
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def initialize io, max_outbox: DEFAULT_MAX_OUTBOX, **opts
|
65
|
+
@io = io
|
66
|
+
@max_outbox = max_outbox
|
67
|
+
@inbox = nil
|
68
|
+
@outbox = []
|
69
|
+
end
|
70
|
+
|
71
|
+
def to_s
|
72
|
+
"#<#{self.class} io=#{io.inspect}>"
|
73
|
+
end
|
74
|
+
|
75
|
+
# If no block given, behaves just the same as #read_one. If block given,
|
76
|
+
# reads any available data and yields it to the block. This form is non-
|
77
|
+
# blocking, if supported by the underlying serializer (such as msgpack).
|
78
|
+
def read
|
79
|
+
if block_given?
|
80
|
+
read_from_inbox {|obj| yield obj}
|
81
|
+
read_from_stream {|obj| yield obj}
|
82
|
+
return nil
|
83
|
+
else
|
84
|
+
read_one
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
# Read one object from the stream, blocking if necessary. Returns the object.
|
89
|
+
# Raises EOFError at the end of the stream.
|
90
|
+
def read_one
|
91
|
+
if @inbox and not @inbox.empty?
|
92
|
+
return @inbox.shift
|
93
|
+
end
|
94
|
+
|
95
|
+
have_result = false
|
96
|
+
result = nil
|
97
|
+
until have_result
|
98
|
+
read do |obj| # might not read enough bytes to yield an obj
|
99
|
+
if have_result
|
100
|
+
(@inbox||=[]) << obj
|
101
|
+
else
|
102
|
+
have_result = true
|
103
|
+
result = obj
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
result
|
108
|
+
end
|
109
|
+
|
110
|
+
def read_from_inbox
|
111
|
+
if @inbox and not @inbox.empty?
|
112
|
+
@inbox.each {|obj| yield obj}
|
113
|
+
@inbox.clear
|
114
|
+
end
|
115
|
+
end
|
116
|
+
private :read_from_inbox
|
117
|
+
|
118
|
+
# Write the given objects to the stream, first flushing any objects in the
|
119
|
+
# outbox. Flushes the underlying byte buffer afterwards.
|
120
|
+
def write *objects
|
121
|
+
write_to_buffer *objects
|
122
|
+
flush_buffer
|
123
|
+
end
|
124
|
+
alias << write
|
125
|
+
|
126
|
+
# Push the given object into the outbox, to be written later when the outbox
|
127
|
+
# is flushed. If a block is given, it will be called when the outbox is
|
128
|
+
# flushed, and its value will be written instead.
|
129
|
+
def write_to_outbox object=nil, &bl
|
130
|
+
@outbox << (bl || object)
|
131
|
+
flush_outbox if @outbox.size > max_outbox
|
132
|
+
self
|
133
|
+
end
|
134
|
+
|
135
|
+
def flush_outbox
|
136
|
+
@outbox.each do |object|
|
137
|
+
object = object.call if object.kind_of? Proc
|
138
|
+
write_to_stream object
|
139
|
+
end
|
140
|
+
@outbox.clear
|
141
|
+
self
|
142
|
+
end
|
143
|
+
|
144
|
+
def write_to_buffer *objects
|
145
|
+
flush_outbox
|
146
|
+
objects.each do |object|
|
147
|
+
write_to_stream object
|
148
|
+
end
|
149
|
+
self
|
150
|
+
end
|
151
|
+
|
152
|
+
def flush_buffer
|
153
|
+
self
|
154
|
+
end
|
155
|
+
|
156
|
+
# Iterate through the (rest of) the stream of objects. Does not raise
|
157
|
+
# EOFError, but simply returns. All Enumerable and Enumerator methods are
|
158
|
+
# available.
|
159
|
+
def each
|
160
|
+
return to_enum unless block_given?
|
161
|
+
read {|obj| yield obj} until eof
|
162
|
+
rescue EOFError
|
163
|
+
end
|
164
|
+
|
165
|
+
def eof?
|
166
|
+
(!@inbox || @inbox.empty?) && io.eof?
|
167
|
+
end
|
168
|
+
alias eof eof?
|
169
|
+
|
170
|
+
# Call this if the most recent write was a #write_to_buffer without
|
171
|
+
# a #flush_buffer. If you only use #write, there's no need to close
|
172
|
+
# the stream in any special way.
|
173
|
+
def close
|
174
|
+
flush_outbox
|
175
|
+
io.close
|
176
|
+
end
|
177
|
+
|
178
|
+
def closed?
|
179
|
+
io.closed?
|
180
|
+
end
|
181
|
+
|
182
|
+
# Makes it possible to use stream in a select.
|
183
|
+
def to_io
|
184
|
+
io
|
185
|
+
end
|
186
|
+
|
187
|
+
class MarshalStream
|
188
|
+
include ObjectStream
|
189
|
+
|
190
|
+
ObjectStream.register_type MARSHAL_TYPE do
|
191
|
+
self
|
192
|
+
end
|
193
|
+
|
194
|
+
def read_from_stream
|
195
|
+
yield Marshal.load(io)
|
196
|
+
end
|
197
|
+
|
198
|
+
def write_to_stream object
|
199
|
+
Marshal.dump(object, io)
|
200
|
+
self
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
class YamlStream
|
205
|
+
include ObjectStream
|
206
|
+
|
207
|
+
ObjectStream.register_type YAML_TYPE do
|
208
|
+
require 'yaml'
|
209
|
+
self
|
210
|
+
end
|
211
|
+
|
212
|
+
def read_from_stream
|
213
|
+
YAML.load_stream(io) do |obj|
|
214
|
+
yield obj
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
def write_to_stream object
|
219
|
+
YAML.dump(object, io)
|
220
|
+
self
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
class JsonStream
|
225
|
+
include ObjectStream
|
226
|
+
|
227
|
+
ObjectStream.register_type JSON_TYPE do
|
228
|
+
require 'yajl'
|
229
|
+
require 'yajl/json_gem'
|
230
|
+
self
|
231
|
+
end
|
232
|
+
|
233
|
+
attr_accessor :chunk_size
|
234
|
+
|
235
|
+
DEFAULT_CHUNK_SIZE = 2000
|
236
|
+
|
237
|
+
def initialize io, chunk_size: DEFAULT_CHUNK_SIZE
|
238
|
+
super
|
239
|
+
@parser = Yajl::Parser.new
|
240
|
+
@encoder = Yajl::Encoder.new
|
241
|
+
@chunk_size = chunk_size
|
242
|
+
end
|
243
|
+
|
244
|
+
# Blocks only if no data available on io.
|
245
|
+
def read_from_stream(&bl)
|
246
|
+
@parser.on_parse_complete = bl
|
247
|
+
@parser << io.readpartial(chunk_size)
|
248
|
+
end
|
249
|
+
|
250
|
+
def write_to_stream object
|
251
|
+
@encoder.encode object, io
|
252
|
+
self
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
256
|
+
class MsgpackStream
|
257
|
+
include ObjectStream
|
258
|
+
|
259
|
+
ObjectStream.register_type MSGPACK_TYPE do
|
260
|
+
require 'msgpack'
|
261
|
+
self
|
262
|
+
end
|
263
|
+
|
264
|
+
attr_accessor :chunk_size
|
265
|
+
attr_accessor :maxbuf
|
266
|
+
|
267
|
+
DEFAULT_CHUNK_SIZE = 2000
|
268
|
+
DEFAULT_MAXBUF = 4000
|
269
|
+
|
270
|
+
def initialize io, chunk_size: DEFAULT_CHUNK_SIZE, maxbuf: DEFAULT_MAXBUF
|
271
|
+
super
|
272
|
+
@unpacker = MessagePack::Unpacker.new
|
273
|
+
# don't specify io, so don't have to read all of io in one loop
|
274
|
+
|
275
|
+
@packer = MessagePack::Packer.new(io)
|
276
|
+
@chunk_size = chunk_size
|
277
|
+
@maxbuf = maxbuf
|
278
|
+
end
|
279
|
+
|
280
|
+
# Blocks only if no data available on io.
|
281
|
+
def read_from_stream
|
282
|
+
fill_buffer(chunk_size)
|
283
|
+
checkbuf if maxbuf
|
284
|
+
read_from_buffer do |obj|
|
285
|
+
yield obj
|
286
|
+
end
|
287
|
+
end
|
288
|
+
|
289
|
+
def fill_buffer n
|
290
|
+
@unpacker.feed(io.readpartial(n))
|
291
|
+
end
|
292
|
+
|
293
|
+
def read_from_buffer
|
294
|
+
@unpacker.each do |obj|
|
295
|
+
yield obj
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
299
|
+
def checkbuf
|
300
|
+
if maxbuf and @unpacker.buffer.size > maxbuf
|
301
|
+
raise OverflowError,
|
302
|
+
"Exceeded buffer limit by #{@unpacker.buffer.size - maxbuf} bytes."
|
303
|
+
end
|
304
|
+
end
|
305
|
+
|
306
|
+
def write_to_stream object
|
307
|
+
@packer.write(object).flush
|
308
|
+
self
|
309
|
+
end
|
310
|
+
|
311
|
+
def write_to_buffer *objects
|
312
|
+
flush_outbox
|
313
|
+
objects.each do |object|
|
314
|
+
@packer.write(object)
|
315
|
+
end
|
316
|
+
self
|
317
|
+
end
|
318
|
+
|
319
|
+
def flush_buffer
|
320
|
+
@packer.flush
|
321
|
+
self
|
322
|
+
end
|
323
|
+
end
|
324
|
+
end
|