obuf 1.1.0 → 1.2.0

data/.travis.yml

@@ -1,4 +1,8 @@
 rvm:
   - 1.8.7
-  - 1.9.2
-  - 1.9.3
+  - 1.9.3
+  - 2.0.0
+  - 2.1.1
+  - 2.2.0
+  - jruby
+  - ruby-head

data/Gemfile

@@ -1,7 +1,8 @@
-source :rubygems
+source 'https://rubygems.org'
 
 group :development do
-  gem "jeweler"
+  gem "bundler"
+  gem "flexmock", "~>0.8"
   gem "rake"
-  gem "flexmock", "~> 0.8"
+  gem "jeweler", '1.8.4' # Last one without the stupid nokogiri dependency
 end

data/History.txt

@@ -1,3 +1,7 @@
+=== 1.2.0 / 2015-04-17
+
+* Add Obuf::Lens to provide access to externally managed IO objects and files
+
 === 1.1.0 / 2011-08-02
 
 * Stores every item from the Enumerable passed to the constructor

data/{README.rdoc → README.md}

@@ -1,37 +1,42 @@
-= obuf
-
-* http://github.com/julik/obuf
-
-== DESCRIPTION:
+# obuf
 
 A simple Ruby object buffer. Use this if you need to temporarily store alot of serializable Ruby objects.
 
-== FEATURES/PROBLEMS:
-
-* Thread safe (for as far as I can gauge it)
-* Gives both read and write access to the storage
-
-== SYNOPSIS:
-
     obuf = Obuf.new
     5_000_000.times{  obuf.push(compute_some_object) } # no memory inflation
     obuf.each do | stored_object | # objects are restored one by one
       # do something with stored_object
     end
 
-== REQUIREMENTS:
+You can also write objects in one process, and recover them in another, using `Obuf::Lens`:
+
+    # In fork process
+    File.open('/tmp/output.bin', 'a') do |f|
+      Obuf::Lens.new(f) << my_object
+    end
+    
+    # In join process after forks completed
+    File.open('/tmp/output.bin', 'r') do |f|
+      Obuf::Lens.new(f).each do | object_written_by_fork |
+        puts object_written_by_fork.inspect
+      end
+    end
+
+The `Lens` is what is used under the hood in the main Obuf as well.
+
+## Requirements
 
 * Ruby 1.8.6+
 
-== INSTALL:
+## Installation
 
 * gem install obuf
 
-== LICENSE:
+## License
 
 (The MIT License)
 
-Copyright (c) 2011 Julik Tarkhanov <me@julik.nl>
+Copyright (c) 2011-2015 Julik Tarkhanov <me@julik.nl>
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/obuf.rb

@@ -19,13 +19,12 @@ require "thread" # required for ruby 18
 #
 # Both reading and writing aim to be threadsafe
 class Obuf
-  VERSION = "1.1.0"
+  VERSION = "1.2.0"
+  require File.dirname(__FILE__) + "/obuf/lens"
+  require File.dirname(__FILE__) + "/obuf/protected_lens"
   
   include Enumerable
   
-  DELIM = "\t"
-  END_RECORD = "\n"
-  
   # Returns the number of objects stored so far
   attr_reader :size
   
@@ -34,13 +33,15 @@ class Obuf
   def initialize(enumerable = [])
     @sem = Mutex.new
     @store = Tempfile.new("obuf")
-    @store.set_encoding(Encoding::BINARY) if @store.respond_to?(:set_encoding)
     @store.binmode
     @size = 0
     
+    @lens = Obuf::ProtectedLens.new(@store)
+    
     # Store everything from the enumerable in self
-    enumerable.each(&method(:push))
+    enumerable.each { |e| push(e) }
     
+    # ...and yield self for any configuration
     yield self if block_given?
   end
   
@@ -51,14 +52,10 @@ class Obuf
   
   # Store an object
   def push(object_to_store)
-    blob = marshal_object(object_to_store)
-    @sem.synchronize do
-      @store.write(blob.size)
-      @store.write(DELIM)
-      @store.write(blob)
-      @store.write(END_RECORD)
+    @sem.synchronize {
+      @lens << object_to_store
       @size += 1
-    end
+    }
     object_to_store
   end
   
@@ -68,7 +65,8 @@ class Obuf
   # methods are also available (but be careful with Enumerable#map and to_a)
   def each
     with_separate_read_io do | iterable |
-      @size.times { yield(recover_object_from(iterable)) }
+      reading_lens = Obuf::Lens.new(iterable)
+      @size.times { yield(reading_lens.recover_object) }
     end
   end
   
@@ -89,15 +87,8 @@ class Obuf
   
   def recover_at(idx)
     with_separate_read_io do | iterable |
-      iterable.seek(0)
-      
-      # Do not unmarshal anything but wind the IO in fixed offsets
-      idx.times do
-        skip_bytes = iterable.gets("\t").to_i
-        iterable.seek(iterable.pos + skip_bytes)
-      end
-      
-      recover_object_from(iterable)
+      reading_lens = Obuf::Lens.new(iterable)
+      reading_lens.recover_at(idx)
     end
   end
   
@@ -105,38 +96,15 @@ class Obuf
   # and iterate through that (we will have one IO handle per loop nest)
   def with_separate_read_io
     # Ensure all data is written before we read it
-    @sem.synchronize { @store.flush }
+    iterable = @sem.synchronize do
+      @store.flush
+      File.open(@store.path, "rb")
+    end
     
-    iterable = File.open(@store.path, "rb")
     begin
       yield(iterable)
     ensure
       iterable.close
     end
   end
-  
-  def recover_object_from(io)
-    # Up to the tab is the amount of bytes to read
-    demarshal_bytes = io.gets("\t").to_i
-    
-    # When at end of IO return nil
-    return nil if demarshal_bytes == 0
-    
-    blob = io.read(demarshal_bytes)
-    demarshal_object(blob)
-  end
-  
-  # This method is only used internally. 
-  # Override this if you need non-default marshalling 
-  # (don't forget to also override demarshal_object)
-  def marshal_object(object_to_store)
-    d = Marshal.dump(object_to_store)
-  end
-  
-  # This method is only used internally. 
-  # Override this if you need non-default demarshalling
-  # (don't forget to also override marshal_object)
-  def demarshal_object(blob)
-    Marshal.load(blob)
-  end
 end

data/lib/obuf/lens.rb

@@ -0,0 +1,70 @@
+require 'thread'
+
+# Provides a per-object iterator on top of any IO object or pipe
+class Obuf::Lens
+  include Enumerable
+  
+  DELIM = "\t"
+  END_RECORD = "\n"
+  
+  # Creates a new Lens on top of a File object, IO object or pipe.
+  # The object given should support +seek+, +gets+, and +read+ to recover
+  # objects, and +write+ to dump objects.
+  def initialize(io_or_pipe)
+    @io = io_or_pipe
+  end
+  
+  # Store an object
+  def <<(object_to_store)
+    blob = marshal_object(object_to_store)
+    @io.write(blob.size)
+    @io.write(DELIM)
+    @io.write(blob)
+    @io.write(END_RECORD)
+    object_to_store
+  end
+  
+  # Recover Nth object
+  def recover_at(idx)
+    @io.seek(0)
+    # Do not unmarshal anything but wind the IO in fixed offsets
+    idx.times do
+      skip_bytes = @io.gets("\t").to_i
+      @io.seek(@io.pos + skip_bytes + 1)
+    end
+    recover_object
+  end
+  
+  # Recover the object at the current position in the IO. Returns +nil+
+  # if there is nothing to recover or the backing buffer is empty.
+  def recover_object
+    # Up to the tab is the amount of bytes to read
+    demarshal_bytes = @io.gets("\t").to_i
+    
+    # When at end of IO return nil
+    return nil if demarshal_bytes == 0
+    
+    blob = @io.read(demarshal_bytes)
+    demarshal_object(blob)
+  end
+  
+  def each
+    yield(recover_object) until @io.eof?
+  end
+  
+  private
+  
+  # This method is only used internally. 
+  # Override this if you need non-default marshalling 
+  # (don't forget to also override demarshal_object)
+  def marshal_object(object_to_store)
+    Marshal.dump(object_to_store)
+  end
+  
+  # This method is only used internally. 
+  # Override this if you need non-default demarshalling
+  # (don't forget to also override marshal_object)
+  def demarshal_object(blob)
+    Marshal.load(blob)
+  end
+end

data/lib/obuf/protected_lens.rb

@@ -0,0 +1,23 @@
+require 'thread'
+
+# Similar to Obuf::Lens but protects all the operations that change the IO offset
+# with a Mutex.
+class Obuf::ProtectedLens < Obuf::Lens
+  def initialize(io)
+    super
+    @mutex = Mutex.new
+  end
+  
+  # Store an object
+  def <<(object_to_store)
+    @mutex.synchronize { super }
+  end
+  
+  def recover_at(idx)
+    @mutex.synchronize { super }
+  end
+  
+  def recover_object
+    @mutex.synchronize { super }
+  end
+end

data/obuf.gemspec

@@ -5,49 +5,54 @@
 
 Gem::Specification.new do |s|
   s.name = "obuf"
-  s.version = "1.1.0"
+  s.version = "1.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Julik Tarkhanov"]
-  s.date = "2012-02-27"
+  s.date = "2015-04-17"
   s.description = "Stores marshaled temporary objects on-disk in a simple Enumerable"
   s.email = "me@julik.nl"
   s.extra_rdoc_files = [
-    "README.rdoc"
+    "README.md"
   ]
   s.files = [
     ".autotest",
     ".travis.yml",
     "Gemfile",
     "History.txt",
-    "README.rdoc",
+    "README.md",
     "Rakefile",
     "lib/obuf.rb",
+    "lib/obuf/lens.rb",
+    "lib/obuf/protected_lens.rb",
     "obuf.gemspec",
     "test/test_obuf.rb"
   ]
   s.homepage = "http://github.com/julik/obuf"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.15"
+  s.rubygems_version = "1.8.23.2"
   s.summary = "Ruby disk-backed object buffer"
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<jeweler>, [">= 0"])
-      s.add_development_dependency(%q<rake>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, [">= 0"])
       s.add_development_dependency(%q<flexmock>, ["~> 0.8"])
+      s.add_development_dependency(%q<rake>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, ["= 1.8.4"])
     else
-      s.add_dependency(%q<jeweler>, [">= 0"])
-      s.add_dependency(%q<rake>, [">= 0"])
+      s.add_dependency(%q<bundler>, [">= 0"])
       s.add_dependency(%q<flexmock>, ["~> 0.8"])
+      s.add_dependency(%q<rake>, [">= 0"])
+      s.add_dependency(%q<jeweler>, ["= 1.8.4"])
     end
   else
-    s.add_dependency(%q<jeweler>, [">= 0"])
-    s.add_dependency(%q<rake>, [">= 0"])
+    s.add_dependency(%q<bundler>, [">= 0"])
     s.add_dependency(%q<flexmock>, ["~> 0.8"])
+    s.add_dependency(%q<rake>, [">= 0"])
+    s.add_dependency(%q<jeweler>, ["= 1.8.4"])
   end
 end
 

data/test/test_obuf.rb

@@ -4,6 +4,7 @@ require "flexmock"
 require "flexmock/test_unit"
 require "stringio"
 
+# We are limited to flexmock 0.8 on Ruby 1.8
 # http://redmine.ruby-lang.org/issues/4882
 # https://github.com/jimweirich/flexmock/issues/4
 # https://github.com/julik/flexmock/commit/4acea00677e7b558bd564ec7c7630f0b27d368ca
@@ -87,15 +88,6 @@ class TestObuf < Test::Unit::TestCase
     assert_equal "E\r\nWow!", a[4]
   end
   
-  def test_random_access
-    a = Obuf.new
-    letters = ("A".."Z").map{|e| "#{e}\r\nWow!"}.to_a
-    letters.map(&a.method(:push))
-    
-    assert_equal "B\r\nWow!", a[1]
-    assert_equal "E\r\nWow!", a[4]
-  end
-
   def test_random_access_out_of_bounds
     a = Obuf.new
     letters = ("A".."Z").map{|e| "#{e}\r\nWow!"}.to_a
@@ -119,3 +111,74 @@ class TestObuf < Test::Unit::TestCase
     assert_equal 0, a.size
   end
 end
+
+class TestLens < Test::Unit::TestCase
+  def test_lens_write
+    mock_io = flexmock()
+    lens = Obuf::Lens.new(mock_io)
+    flexmock(mock_io).should_receive(:write){|written|
+      assert_kind_of Fixnum, written
+    }
+    flexmock(mock_io).should_receive(:write).with("\t")
+    flexmock(mock_io).should_receive(:write){|byte_blob|
+      assert_kind_of String, byte_blob
+    }
+    flexmock(mock_io).should_receive(:write).with("\n")
+    lens << "Hi there!"
+  end
+  
+  def test_lens_recover_object_after_write
+    mock_io = StringIO.new
+    lens = Obuf::Lens.new(mock_io)
+    lens << "Hi there!"
+    
+    recovered = lens.recover_object
+    assert_nil recovered, "The IO is at the end now"
+    
+    mock_io.rewind
+    recovered = lens.recover_object
+    assert_equal "Hi there!", recovered
+  end
+  
+  def test_lens_each
+    mock_io = StringIO.new
+    
+    writing_lens = Obuf::Lens.new(mock_io)
+    writing_lens << "Hi there!"
+    writing_lens << 98121
+    writing_lens << [123, :a]
+    
+    mock_io.rewind
+    
+    reading_lens = Obuf::Lens.new(mock_io)
+    items = []
+    reading_lens.each do | item |
+      items << item
+    end
+    
+    assert_equal "Hi there!", items[0]
+    assert_equal 98121, items[1]
+    assert_equal [123, :a], items[2]
+  end
+  
+  def test_lens_recover_at
+    mock_io = StringIO.new
+    
+    writing_lens = Obuf::Lens.new(mock_io)
+    writing_lens << "Hi there!"
+    writing_lens << 98121
+    writing_lens << [123, :a]
+    
+    reading_lens = Obuf::Lens.new(mock_io)
+    # Also try to recover object at index 3, which does not exist in the underlying buffer
+    objects_at_positions = (0..3).to_a.reverse.map do | i |
+      reading_lens.recover_at(i)
+    end
+    
+    assert_nil objects_at_positions[0]
+    assert_equal [123, :a], objects_at_positions[1]
+    assert_equal 98121, objects_at_positions[2]
+    assert_equal "Hi there!", objects_at_positions[3]
+  end
+  
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: obuf
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,22 +9,43 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-27 00:00:00.000000000 Z
+date: 2015-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: jeweler
-  requirement: &4139570 !ruby/object:Gem::Requirement
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: flexmock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
   type: :development
   prerelease: false
-  version_requirements: *4139570
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &4139080 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,32 +53,44 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *4139080
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: flexmock
-  requirement: &4138610 !ruby/object:Gem::Requirement
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 1.8.4
   type: :development
   prerelease: false
-  version_requirements: *4138610
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.8.4
 description: Stores marshaled temporary objects on-disk in a simple Enumerable
 email: me@julik.nl
 executables: []
 extensions: []
 extra_rdoc_files:
-- README.rdoc
+- README.md
 files:
 - .autotest
 - .travis.yml
 - Gemfile
 - History.txt
-- README.rdoc
+- README.md
 - Rakefile
 - lib/obuf.rb
+- lib/obuf/lens.rb
+- lib/obuf/protected_lens.rb
 - obuf.gemspec
 - test/test_obuf.rb
 homepage: http://github.com/julik/obuf
@@ -73,6 +106,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 4056659205467737583
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -81,7 +117,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 1.8.23.2
 signing_key: 
 specification_version: 3
 summary: Ruby disk-backed object buffer