object-stream 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 40d4571e5abdd3d97b76a1b8694b7cdeabab60b9
+  data.tar.gz: 33e6108c55492c43b65c3ce6d9eecad2d9686cd7
+SHA512:
+  metadata.gz: 1db1cb56ff1db7d59d90ebf8d619674c72d8dd1945e2134e62a86b7ff262d7ab3765e747d80ddfd12a4fd2a44c7cf781f1cd48d7948b255730ae1debf92248fe
+  data.tar.gz: 9ce4e08d1c58c07d64172cc9cee30e7fcb43f77492184070872269728f59f8776d26339a118fd9809b36352f9f46661af4f5814b3a9cedcb31968b8fdaa65b1f

data/COPYING

@@ -0,0 +1,22 @@
+Copyright (c) 2013, Joel VanderWerf, vjoel@users.sourceforge.net
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met: 
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer. 
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution. 
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,7 @@
+object-stream
+=============
+
+Stream objects over IO using Marshal, JSON, YAML, or Msgpack.
+
+Note that JSON (using Yajl) and Msgpack both permit suspending partial reads when other input is available, while Marshal and YAML need to read complete objects. So, using Marshal or YAML in a single thread (such as a select loop) may lead to blocking if one stream has not produced a complete object. See examples/slow-sender.rb.
+

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rake'
+require 'rake/testtask'
+
+desc "Run tests"
+Rake::TestTask.new :test do |t|
+  t.libs << "lib"
+  t.libs << "ext"
+  t.test_files = FileList["test/**/*.rb"]
+end

data/bench/pipe-socket.rb

@@ -0,0 +1,52 @@
+require 'benchmark'
+require 'object-stream'
+require 'socket'
+
+def run(
+    rd:       nil,
+    wr:       nil,
+    objects:  1..10,
+    type:     ObjectStream::MSGPACK_TYPE)
+
+  pid = fork do
+    stream = ObjectStream.new(wr, type: type)
+    objects.each do |object|
+      stream << object
+    end
+  end
+  wr.close
+
+  stream = ObjectStream.new(rd, type: type)
+  count = stream.inject(0) do |n, object|
+    n + 1
+  end
+  raise unless count == objects.size
+  stream.close
+
+ensure
+  Process.wait pid if pid
+end
+
+begin # warmup
+  rd, wr = IO.pipe
+  run rd: rd, wr: wr
+
+  rd, wr = UNIXSocket.pair
+  run rd: rd, wr: wr
+end
+
+tuples = (0...10_000).map {|i|
+  [i, 42.42, true, "foo", {"bar" => "baz"}]
+}
+
+Benchmark.bm(10) do |bench|
+  bench.report "pipe" do
+    rd, wr = IO.pipe
+    run rd: rd, wr: wr, objects: tuples
+  end
+  
+  bench.report "socket" do
+    rd, wr = UNIXSocket.pair
+    run rd: rd, wr: wr, objects: tuples
+  end
+end

data/examples/basic-usage.rb

@@ -0,0 +1,43 @@
+case ARGV[0]
+when "marshal", "yaml", "json", "msgpack"
+else
+  abort "Usage: #$0 marshal|yaml|json|msgpack"
+end
+
+require 'object-stream'
+require 'tmpdir'
+
+begin
+  dir = Dir.mktmpdir "stream-"
+  dumpfile = File.join(dir, "dump")
+
+  type = ARGV.shift
+  
+  File.open(dumpfile, "w") do |f|
+    stream = ObjectStream.new(f, type: type)
+    p stream
+    stream << "foo" << [:bar, 42] << {"String" => "string"}
+  end
+  
+  data = File.read(dumpfile)
+  puts "===== #{data.size} bytes:"
+  case type
+  when ObjectStream::MARSHAL_TYPE, ObjectStream::MSGPACK_TYPE
+    puts data.inspect
+  when ObjectStream::YAML_TYPE, ObjectStream::JSON_TYPE
+    puts data
+  end
+  puts "====="
+  
+  a = File.open(dumpfile, "r") do |f|
+    stream = ObjectStream.new(f, type: type)
+    stream.map do |obj|
+      #p obj
+      obj
+    end
+  end
+  p a
+
+ensure
+  FileUtils.remove_entry dir
+end

data/examples/custom-class.rb

@@ -0,0 +1,73 @@
+case ARGV[0]
+when "marshal", "yaml", "json", "msgpack"
+else
+  abort "Usage: #$0 marshal|yaml|json|msgpack"
+end
+
+require 'object-stream'
+require 'tmpdir'
+
+begin
+  dir = Dir.mktmpdir "stream-"
+  dumpfile = File.join(dir, "dump")
+
+  type = ARGV.shift
+  
+  class A
+    def initialize x, y
+      @x, @y = x, y
+    end
+    
+    def to_msgpack pk = nil
+      case pk
+      when MessagePack::Packer
+        pk.write_array_header(2)
+        pk.write @x
+        pk.write @y
+        return pk
+      
+      else # nil or IO
+        MessagePack.pack(self, pk)
+      end
+    end
+    
+    def to_json
+      [@x, @y].to_json
+    end
+    
+    def self.from_serialized ary
+      new *ary
+    end
+  end
+
+  File.open(dumpfile, "w") do |f|
+    stream = ObjectStream.new(f, type: type)
+    p stream
+    stream << "foo" << [:bar, 42] << {"String" => "string"} << "A" << A.new(3,6)
+  end
+  
+  data = File.read(dumpfile)
+  puts "===== #{data.size} bytes:"
+  case type
+  when ObjectStream::MARSHAL_TYPE, ObjectStream::MSGPACK_TYPE
+    puts data.inspect
+  when ObjectStream::YAML_TYPE, ObjectStream::JSON_TYPE
+    puts data
+  end
+  puts "====="
+  
+  a = File.open(dumpfile, "r") do |f|
+    stream = ObjectStream.new(f, type: type)
+    stream.map do |obj|
+      #p obj
+      stream.expect {obj == "A" and A}
+        # code inside block won't be executed in cases where object class
+        # is encoded in the data itself (marshal, yaml)
+      obj
+    end
+  end
+  p a
+
+ensure
+  FileUtils.remove_entry dir
+end

data/examples/maxbuf.rb

@@ -0,0 +1,17 @@
+# The MsgpackStream can detect when too many bytes have been read without
+# complete parsing of an object.
+
+require 'object-stream'
+require 'stringio'
+
+sio = StringIO.new
+stream = ObjectStream.new(sio, type: ObjectStream::MSGPACK_TYPE)
+stream << "This is a very long sentence, and possibly a denial-of-service attack!"
+
+sio.rewind
+stream = ObjectStream.new(sio, type: ObjectStream::MSGPACK_TYPE, maxbuf: 20)
+begin
+  stream.to_a
+rescue ObjectStream::OverflowError => ex
+  puts ex # => Exceeded buffer limit by 53 bytes.
+end

data/examples/outbox.rb

@@ -0,0 +1,19 @@
+require 'object-stream'
+require 'stringio'
+
+sio = StringIO.new
+stream = ObjectStream.new(sio, type: ObjectStream::MSGPACK_TYPE)
+stream.write_to_outbox "and now for something"
+stream << "completely different"
+
+sio.rewind
+stream = ObjectStream.new(sio, type: ObjectStream::MSGPACK_TYPE)
+puts stream.to_a
+
+__END__
+
+Output:
+
+and now for something
+completely different
+

data/examples/pipe.rb

@@ -0,0 +1,42 @@
+case ARGV[0]
+when "marshal", "yaml", "json", "msgpack"
+else
+  abort "Usage: #$0 marshal|yaml|json|msgpack"
+end
+
+require 'object-stream'
+
+begin
+  type = ARGV.shift
+
+  rd, wr = IO.pipe
+
+  pid = fork do
+    stream = ObjectStream.new(wr, type: type)
+    10.times do |i|
+      stream << [i] # box the int because json needs it
+    end
+  end
+
+  wr.close
+  stream = ObjectStream.new(rd, type: type)
+
+  puts "try to select"
+  until stream.eof? # Otherwise, EOFError is raised at end.
+    select([stream]) # Note stream usable as IO.
+    
+    # Use #read instead of #each or #map so that, in the case of msgpack and
+    # yajl, only the available bytes in io are copied to the stream's buffer
+    # and parsed to objects.
+    stream.read do |obj|
+      p obj
+    end
+
+    puts "select again"
+  end
+  puts "rd.eof? = #{rd.eof?}"
+  stream.close
+
+ensure
+  Process.wait pid if pid
+end

data/examples/read-without-block.rb

@@ -0,0 +1,25 @@
+require 'object-stream'
+require 'socket'
+
+begin
+  type = ObjectStream::MSGPACK_TYPE
+
+  s, t = UNIXSocket.pair
+
+  pid = fork do
+    stream = ObjectStream.new(s, type: type)
+    10.times do |i|
+      stream << [i]
+    end
+  end
+
+  s.close
+  stream = ObjectStream.new(t, type: type)
+
+  until stream.eof?
+    p stream.read
+  end
+
+ensure
+  Process.wait pid if pid
+end

data/examples/slow-sender.rb

@@ -0,0 +1,76 @@
+case ARGV[0]
+when "marshal", "yaml", "json", "msgpack"
+else
+  abort "Usage: #$0 marshal|yaml|json|msgpack"
+end
+
+require 'object-stream'
+require 'socket'
+require 'stringio'
+
+begin
+  type = ARGV.shift
+
+  s, t = UNIXSocket.pair
+
+  pid = fork do
+    sio = StringIO.new
+    stream = ObjectStream.new(sio, type: type)
+    10.times do |i|
+      stream << "foo bar #{i}"
+    end
+    
+    sio.rewind
+    stream2 = ObjectStream.new(sio, type: type)
+    stream2.each_with_index do |x, i|
+      raise unless x == "foo bar #{i}"
+    end
+    
+    sio.rewind
+    data = sio.read
+    pos = data.index "bar 5"
+    raise unless pos < data.size - 10 # assume strings not munged
+    s.write data[0...pos]
+    puts "simulating a slow sender -- " +
+         "see if receiver blocks or stops reading and goes back to select"
+    sleep 0.1
+    s.write data[pos...pos+1]
+    sleep 0.1
+    s.write data[pos+1...pos+2]
+    sleep 0.1
+    s.write data[pos+2..-1]
+  end
+
+  s.close
+  stream = ObjectStream.new(t, type: type)
+
+  select_count = 0
+  empty_read_count = 0
+  until stream.eof? # Otherwise, EOFError is raised at end.
+    select_count += 1
+    puts "select #{select_count}"
+    select([stream]) # Note stream usable as IO.
+    
+    # Use #read instead of #each or #map so that, in the case of msgpack and
+    # yajl, only the available bytes in io are copied to the stream's buffer
+    # and parsed to objects.
+    obj_count = 0
+    stream.read do |obj|
+      p obj
+      obj_count += 1
+    end
+    if obj_count == 0
+      empty_read_count += 1
+    end
+  end
+  
+  stream.close
+  if select_count > 1 and empty_read_count > 0
+    puts "For #{type}, slow sender did not block the receiver!"
+  else
+    puts "For #{type}, slow sender blocked the receiver!"
+  end
+
+ensure
+  Process.wait pid if pid
+end

data/examples/socket.rb

@@ -0,0 +1,43 @@
+case ARGV[0]
+when "marshal", "yaml", "json", "msgpack"
+else
+  abort "Usage: #$0 marshal|yaml|json|msgpack"
+end
+
+require 'object-stream'
+require 'socket'
+
+begin
+  type = ARGV.shift
+
+  s, t = UNIXSocket.pair
+
+  pid = fork do
+    stream = ObjectStream.new(s, type: type)
+    10.times do |i|
+      stream << [i] # box the int because json needs it
+    end
+  end
+
+  s.close
+  stream = ObjectStream.new(t, type: type)
+
+  puts "try to select"
+  until stream.eof? # Otherwise, EOFError is raised at end.
+    select([stream]) # Note stream usable as IO.
+    
+    # Use #read instead of #each or #map so that, in the case of msgpack and
+    # yajl, only the available bytes in io are copied to the stream's buffer
+    # and parsed to objects.
+    stream.read do |obj|
+      p obj
+    end
+
+    puts "select again"
+  end
+  puts "t.eof? = #{t.eof?}"
+  stream.close
+
+ensure
+  Process.wait pid if pid
+end