object-stream 0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/examples/udp.rb ADDED
@@ -0,0 +1,35 @@
1
+ case ARGV[0]
2
+ when "marshal", "yaml", "json", "msgpack"
3
+ else
4
+ abort "Usage: #$0 marshal|yaml|json|msgpack"
5
+ end
6
+
7
+ type = ARGV.shift
8
+
9
+ require 'object-stream'
10
+ require 'socket'
11
+
12
+ Socket.do_not_reverse_lookup = true
13
+ s = UDPSocket.new; s.bind 'localhost', 0
14
+ t = UDPSocket.new; t.bind 'localhost', 0
15
+ s.connect *t.addr.values_at(2,1)
16
+ t.connect *s.addr.values_at(2,1)
17
+
18
+ th1 = Thread.new do
19
+ stream = ObjectStream.new(s, type: type)
20
+ 10.times do |i|
21
+ stream << [i]
22
+ end
23
+ stream << "Bye."
24
+ end
25
+
26
+ th2 = Thread.new do
27
+ stream = ObjectStream.new(t, type: type)
28
+ stream.each do |obj|
29
+ p obj
30
+ break if /bye/i === obj
31
+ end
32
+ end
33
+
34
+ th1.join
35
+ th2.join
@@ -0,0 +1,114 @@
1
+ require 'object-stream'
2
+
3
+ # Utility wrapper for basic ObjectStream class. Adds three groups of
4
+ # functionality:
5
+ #
6
+ # * peer_name
7
+ #
8
+ # * expect
9
+ #
10
+ # * consume
11
+ #
12
+ class ObjectStreamWrapper
13
+ include Enumerable
14
+
15
+ # Not set by this library, but available for users to keep track of
16
+ # the peer in a symbolic, application-specific manner. See funl for
17
+ # an example.
18
+ attr_accessor :peer_name
19
+
20
+ def initialize *args, **opts
21
+ @stream = ObjectStream.new(*args, **opts)
22
+ @peer_name = "unknown"
23
+ @expected_class = nil
24
+ @consumers = []
25
+ unexpect
26
+ end
27
+
28
+ def to_s
29
+ "#<Wrapped #{@stream.class} to #{peer_name}, io=#{@stream.inspect}>"
30
+ end
31
+
32
+ # Set the stream state so that subsequent objects returned by read will be
33
+ # instances of a custom class +cl+. Does not affect #consume.
34
+ # Class +cl+ should define cl.from_serialized, plus #to_json, #to_msgpack,
35
+ # etc. as needed by the underlying serialization library.
36
+ def expect cl
37
+ @expected_class = cl
38
+ end
39
+
40
+ # Turn off the custom class instantiation of #expect.
41
+ def unexpect; expect nil; end
42
+
43
+ # The block is appended to a queue of procs that are called for the
44
+ # subsequently read objects, instead of iterating over or returning them.
45
+ # Helps with handshake protocols. Not affected by #expect.
46
+ def consume &bl
47
+ @consumers << bl
48
+ end
49
+
50
+ def try_consume obj
51
+ if bl = @consumers.shift
52
+ bl[obj]
53
+ true
54
+ else
55
+ false
56
+ end
57
+ end
58
+ private :try_consume
59
+
60
+ def convert_to_expected obj
61
+ if @expected_class and not obj.kind_of? @expected_class
62
+ @expected_class.from_serialized(obj)
63
+ else
64
+ obj
65
+ end
66
+ end
67
+ private :convert_to_expected
68
+
69
+ def read
70
+ if block_given?
71
+ @stream.read do |obj|
72
+ try_consume(obj) or yield convert_to_expected(obj)
73
+ end
74
+ return nil
75
+ else
76
+ begin
77
+ obj = @stream.read
78
+ end while try_consume(obj)
79
+ convert_to_expected(obj)
80
+ end
81
+ end
82
+
83
+ def each
84
+ return to_enum unless block_given?
85
+ read {|obj| yield obj} until eof
86
+ rescue EOFError
87
+ end
88
+
89
+ def write *objects
90
+ @stream.write *objects
91
+ end
92
+ alias << write
93
+
94
+ def write_to_outbox *args, &bl
95
+ @stream.write_to_outbox *args, &bl
96
+ end
97
+
98
+ def eof?
99
+ @stream.eof?
100
+ end
101
+ alias eof eof?
102
+
103
+ def close
104
+ @stream.close
105
+ end
106
+
107
+ def closed?
108
+ @stream.closed?
109
+ end
110
+
111
+ def to_io
112
+ @stream.to_io
113
+ end
114
+ end
@@ -0,0 +1,324 @@
1
+ # Stream of objects, with any underlying IO: File, Pipe, Socket, StringIO.
2
+ # Stream is bidirectional if the IO is bidirectional.
3
+ #
4
+ # Serializes objects using any of several serializers: marshal, yaml, json,
5
+ # msgpack. Works with select/readpartial if the serializer supports it (msgpack
6
+ # and yajl do).
7
+ #
8
+ # ObjectStream supports three styles of iteration: Enumerable, blocking read,
9
+ # and yielding (non-blocking) read.
10
+ module ObjectStream
11
+ include Enumerable
12
+
13
+ # The IO through which the stream reads and writes serialized object data.
14
+ attr_reader :io
15
+
16
+ # Number of outgoing objects that can accumulate before the outbox is
17
+ # serialized to the byte buffer (and possibly to the io).
18
+ attr_reader :max_outbox
19
+
20
+ MARSHAL_TYPE = "marshal".freeze
21
+ YAML_TYPE = "yaml".freeze
22
+ JSON_TYPE = "json".freeze
23
+ MSGPACK_TYPE = "msgpack".freeze
24
+
25
+ TYPES = [
26
+ MARSHAL_TYPE, YAML_TYPE, JSON_TYPE, MSGPACK_TYPE
27
+ ]
28
+
29
+ DEFAULT_MAX_OUTBOX = 10
30
+
31
+ # Raised when maxbuf exceeded.
32
+ class OverflowError < StandardError; end
33
+
34
+ @stream_class_map =
35
+ Hash.new {|h,type| raise ArgumentError, "unknown type: #{type.inspect}"}
36
+ @mutex = Mutex.new
37
+
38
+ class << self
39
+ def new io, type: MARSHAL_TYPE, **opts
40
+ if io.kind_of? ObjectStream
41
+ raise ArgumentError,
42
+ "given io is already an ObjectStream: #{io.inspect}"
43
+ end
44
+ stream_class_for(type).new io, **opts
45
+ end
46
+
47
+ def stream_class_for type
48
+ cl = @stream_class_map[type]
49
+ return cl if cl.respond_to? :new
50
+
51
+ # Protect against race condition in msgpack and yajl extension
52
+ # initialization (bug #8374).
53
+ @mutex.synchronize do
54
+ return cl if cl.respond_to? :new
55
+ @stream_class_map[type] = cl.call
56
+ end
57
+ end
58
+
59
+ def register_type type, &bl
60
+ @stream_class_map[type] = bl
61
+ end
62
+ end
63
+
64
+ def initialize io, max_outbox: DEFAULT_MAX_OUTBOX, **opts
65
+ @io = io
66
+ @max_outbox = max_outbox
67
+ @inbox = nil
68
+ @outbox = []
69
+ end
70
+
71
+ def to_s
72
+ "#<#{self.class} io=#{io.inspect}>"
73
+ end
74
+
75
+ # If no block given, behaves just the same as #read_one. If block given,
76
+ # reads any available data and yields it to the block. This form is non-
77
+ # blocking, if supported by the underlying serializer (such as msgpack).
78
+ def read
79
+ if block_given?
80
+ read_from_inbox {|obj| yield obj}
81
+ read_from_stream {|obj| yield obj}
82
+ return nil
83
+ else
84
+ read_one
85
+ end
86
+ end
87
+
88
+ # Read one object from the stream, blocking if necessary. Returns the object.
89
+ # Raises EOFError at the end of the stream.
90
+ def read_one
91
+ if @inbox and not @inbox.empty?
92
+ return @inbox.shift
93
+ end
94
+
95
+ have_result = false
96
+ result = nil
97
+ until have_result
98
+ read do |obj| # might not read enough bytes to yield an obj
99
+ if have_result
100
+ (@inbox||=[]) << obj
101
+ else
102
+ have_result = true
103
+ result = obj
104
+ end
105
+ end
106
+ end
107
+ result
108
+ end
109
+
110
+ def read_from_inbox
111
+ if @inbox and not @inbox.empty?
112
+ @inbox.each {|obj| yield obj}
113
+ @inbox.clear
114
+ end
115
+ end
116
+ private :read_from_inbox
117
+
118
+ # Write the given objects to the stream, first flushing any objects in the
119
+ # outbox. Flushes the underlying byte buffer afterwards.
120
+ def write *objects
121
+ write_to_buffer *objects
122
+ flush_buffer
123
+ end
124
+ alias << write
125
+
126
+ # Push the given object into the outbox, to be written later when the outbox
127
+ # is flushed. If a block is given, it will be called when the outbox is
128
+ # flushed, and its value will be written instead.
129
+ def write_to_outbox object=nil, &bl
130
+ @outbox << (bl || object)
131
+ flush_outbox if @outbox.size > max_outbox
132
+ self
133
+ end
134
+
135
+ def flush_outbox
136
+ @outbox.each do |object|
137
+ object = object.call if object.kind_of? Proc
138
+ write_to_stream object
139
+ end
140
+ @outbox.clear
141
+ self
142
+ end
143
+
144
+ def write_to_buffer *objects
145
+ flush_outbox
146
+ objects.each do |object|
147
+ write_to_stream object
148
+ end
149
+ self
150
+ end
151
+
152
+ def flush_buffer
153
+ self
154
+ end
155
+
156
+ # Iterate through the (rest of) the stream of objects. Does not raise
157
+ # EOFError, but simply returns. All Enumerable and Enumerator methods are
158
+ # available.
159
+ def each
160
+ return to_enum unless block_given?
161
+ read {|obj| yield obj} until eof
162
+ rescue EOFError
163
+ end
164
+
165
+ def eof?
166
+ (!@inbox || @inbox.empty?) && io.eof?
167
+ end
168
+ alias eof eof?
169
+
170
+ # Call this if the most recent write was a #write_to_buffer without
171
+ # a #flush_buffer. If you only use #write, there's no need to close
172
+ # the stream in any special way.
173
+ def close
174
+ flush_outbox
175
+ io.close
176
+ end
177
+
178
+ def closed?
179
+ io.closed?
180
+ end
181
+
182
+ # Makes it possible to use stream in a select.
183
+ def to_io
184
+ io
185
+ end
186
+
187
+ class MarshalStream
188
+ include ObjectStream
189
+
190
+ ObjectStream.register_type MARSHAL_TYPE do
191
+ self
192
+ end
193
+
194
+ def read_from_stream
195
+ yield Marshal.load(io)
196
+ end
197
+
198
+ def write_to_stream object
199
+ Marshal.dump(object, io)
200
+ self
201
+ end
202
+ end
203
+
204
+ class YamlStream
205
+ include ObjectStream
206
+
207
+ ObjectStream.register_type YAML_TYPE do
208
+ require 'yaml'
209
+ self
210
+ end
211
+
212
+ def read_from_stream
213
+ YAML.load_stream(io) do |obj|
214
+ yield obj
215
+ end
216
+ end
217
+
218
+ def write_to_stream object
219
+ YAML.dump(object, io)
220
+ self
221
+ end
222
+ end
223
+
224
+ class JsonStream
225
+ include ObjectStream
226
+
227
+ ObjectStream.register_type JSON_TYPE do
228
+ require 'yajl'
229
+ require 'yajl/json_gem'
230
+ self
231
+ end
232
+
233
+ attr_accessor :chunk_size
234
+
235
+ DEFAULT_CHUNK_SIZE = 2000
236
+
237
+ def initialize io, chunk_size: DEFAULT_CHUNK_SIZE
238
+ super
239
+ @parser = Yajl::Parser.new
240
+ @encoder = Yajl::Encoder.new
241
+ @chunk_size = chunk_size
242
+ end
243
+
244
+ # Blocks only if no data available on io.
245
+ def read_from_stream(&bl)
246
+ @parser.on_parse_complete = bl
247
+ @parser << io.readpartial(chunk_size)
248
+ end
249
+
250
+ def write_to_stream object
251
+ @encoder.encode object, io
252
+ self
253
+ end
254
+ end
255
+
256
+ class MsgpackStream
257
+ include ObjectStream
258
+
259
+ ObjectStream.register_type MSGPACK_TYPE do
260
+ require 'msgpack'
261
+ self
262
+ end
263
+
264
+ attr_accessor :chunk_size
265
+ attr_accessor :maxbuf
266
+
267
+ DEFAULT_CHUNK_SIZE = 2000
268
+ DEFAULT_MAXBUF = 4000
269
+
270
+ def initialize io, chunk_size: DEFAULT_CHUNK_SIZE, maxbuf: DEFAULT_MAXBUF
271
+ super
272
+ @unpacker = MessagePack::Unpacker.new
273
+ # don't specify io, so don't have to read all of io in one loop
274
+
275
+ @packer = MessagePack::Packer.new(io)
276
+ @chunk_size = chunk_size
277
+ @maxbuf = maxbuf
278
+ end
279
+
280
+ # Blocks only if no data available on io.
281
+ def read_from_stream
282
+ fill_buffer(chunk_size)
283
+ checkbuf if maxbuf
284
+ read_from_buffer do |obj|
285
+ yield obj
286
+ end
287
+ end
288
+
289
+ def fill_buffer n
290
+ @unpacker.feed(io.readpartial(n))
291
+ end
292
+
293
+ def read_from_buffer
294
+ @unpacker.each do |obj|
295
+ yield obj
296
+ end
297
+ end
298
+
299
+ def checkbuf
300
+ if maxbuf and @unpacker.buffer.size > maxbuf
301
+ raise OverflowError,
302
+ "Exceeded buffer limit by #{@unpacker.buffer.size - maxbuf} bytes."
303
+ end
304
+ end
305
+
306
+ def write_to_stream object
307
+ @packer.write(object).flush
308
+ self
309
+ end
310
+
311
+ def write_to_buffer *objects
312
+ flush_outbox
313
+ objects.each do |object|
314
+ @packer.write(object)
315
+ end
316
+ self
317
+ end
318
+
319
+ def flush_buffer
320
+ @packer.flush
321
+ self
322
+ end
323
+ end
324
+ end