obuf 1.0.0

data/.autotest

@@ -0,0 +1,23 @@
+# -*- ruby -*-
+
+require 'autotest/restart'
+
+# Autotest.add_hook :initialize do |at|
+#   at.extra_files << "../some/external/dependency.rb"
+#
+#   at.libs << ":../some/external"
+#
+#   at.add_exception 'vendor'
+#
+#   at.add_mapping(/dependency.rb/) do |f, _|
+#     at.files_matching(/test_.*rb$/)
+#   end
+#
+#   %w(TestA TestB).each do |klass|
+#     at.extra_class_map[klass] = "test/test_misc.rb"
+#   end
+# end
+
+# Autotest.add_hook :run_command do |at|
+#   system "rake build"
+# end

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2011-08-02
+
+* 1 major enhancement
+
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,7 @@
+.autotest
+History.txt
+Manifest.txt
+README.rdoc
+Rakefile
+lib/obuf.rb
+test/test_obuf.rb

data/README.rdoc

@@ -0,0 +1,53 @@
+= obuf
+
+* http://github.com/julik/obuf
+
+== DESCRIPTION:
+
+A simple Ruby object buffer. Use this if you need to temporarily store alot of serializable Ruby objects.
+
+== FEATURES/PROBLEMS:
+
+* Thread safe (for as far as I can gauge it)
+* Gives both read and write access to the storage
+
+== SYNOPSIS:
+
+    obuf = Obuf.new
+    5_000_000.times{  obuf.push(compute_some_object) } # no memory inflation
+    obuf.each do | stored_object | # objects are restored one by one
+      # do something with stored_object
+    end
+
+== REQUIREMENTS:
+
+* Ruby 1.8.6+
+
+== INSTALL:
+
+* gem install obuf
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2011 Julik Tarkhanov <me@julik.nl>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,15 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+Hoe.spec 'obuf' do | p |
+  p.developer('Julik Tarkhanov', 'me@julik.nl')
+
+  p.readme_file   = 'README.rdoc'
+  p.extra_rdoc_files  = FileList['*.rdoc'] + FileList['*.txt']
+  p.extra_dev_deps = {"flexmock" => "~> 0.8", "cli_test" => "~>1.0"}
+  p.clean_globs = File.read(File.dirname(__FILE__) + "/.gitignore").split(/\s/).to_a
+end
+
+# vim: syntax=ruby

data/lib/obuf.rb

@@ -0,0 +1,127 @@
+require "tempfile"
+
+# An object buffer for Ruby objects. Use it to sequentially store a shitload
+# of objects on disk and then retreive them one by one. Make sure to call clear when done
+# with it to discard the stored blob. It can be used like a disk-based object buffer.
+# (Tracksperanto stores parsed trackers into it)
+#
+#  a = Tracksperanto::Accumulator.new
+#  parse_big_file do | one_node |
+#    a.push(one_node)
+#  end
+#
+#  a.size #=> 30932 # We've stored 30 thousand objects on disk without breaking a sweat
+#  a.each do | node_read_from_disk |
+#     # do something with node that has been recovered from disk
+#  end
+#
+#  a.clear # ensure that the file is deleted
+#
+# Both reading and writing aim to be threadsafe (writing operations will be locked with a mutex, and )
+class Obuf
+  VERSION = "1.0.0"
+  
+  include Enumerable
+  
+  DELIM = "\t"
+  END_RECORD = "\n"
+  
+  # Returns the number of objects stored so far
+  attr_reader :size
+  
+  def initialize
+    @sem = Mutex.new
+    @store = create_store
+    @store.set_encoding(Encoding::BINARY) if @store.respond_to?(:set_encoding)
+    @store.binmode
+    
+    @size = 0
+    super
+  end
+  
+  # Tells whether the buffer is empty
+  def empty?
+    @size.zero?
+  end
+  
+  # Store an object
+  def push(object_to_store)
+    blob = marshal_object(object_to_store)
+    @sem.synchronize do
+      @store.write(blob.size)
+      @store.write(DELIM)
+      @store.write(blob)
+      @store.write(END_RECORD)
+      @size += 1
+    end
+    object_to_store
+  end
+  
+  alias_method :<<, :push
+  
+  # Retreive each stored object in succession. All other Enumerable
+  # methods are also available (but be careful with Enumerable#map and to_a)
+  def each
+    with_separate_read_io do | iterable |
+      @size.times { yield(recover_object_from(iterable)) }
+    end
+  end
+  
+  # Calls close! on the datastore and deletes the objects in it
+  def clear
+    @store.close!
+    @size = 0
+  end
+  
+  # Retreive a concrete object at index
+  def [](idx)
+    idx.respond_to?(:each) ? idx.map{|i| recover_at(i) } : recover_at(idx)
+  end
+  
+  private
+  
+  def recover_at(idx)
+    with_separate_read_io do | iterable |
+      iterable.seek(0)
+      
+      # Do not unmarshal anything but wind the IO in fixed offsets
+      idx.times do
+        skip_bytes = iterable.gets("\t").to_i
+        iterable.seek(iterable.pos + skip_bytes)
+      end
+      
+      recover_object_from(iterable)
+    end
+  end
+  
+  # We first ensure that we have a disk-backed file, then reopen it as read-only
+  # and iterate through that (we will have one IO handle per loop nest)
+  def with_separate_read_io
+    # Ensure all data is written before we read it
+    @sem.synchronize { @store.flush }
+    
+    iterable = File.open(@store.path, "r")
+    begin
+      yield(iterable)
+    ensure
+      iterable.close
+    end
+  end
+  
+  def marshal_object(object_to_store)
+    d = Marshal.dump(object_to_store)
+  end
+  
+  def recover_object_from(io)
+    # Up to the tab is the amount of bytes to read
+    demarshal_bytes = io.gets("\t").to_i
+    blob = io.read(demarshal_bytes)
+    
+    Marshal.load(blob)
+  end
+  
+  # In case you need to use something else (say, a StringIO)
+  def create_store
+    Tempfile.new("obuf")
+  end
+end

data/test/test_obuf.rb

@@ -0,0 +1,85 @@
+require "test/unit"
+require "obuf"
+require "flexmock"
+require "flexmock/test_unit"
+
+# http://redmine.ruby-lang.org/issues/4882
+# https://github.com/jimweirich/flexmock/issues/4
+# https://github.com/julik/flexmock/commit/4acea00677e7b558bd564ec7c7630f0b27d368ca
+class FlexMock::PartialMockProxy
+  def singleton?(method_name)
+    @obj.singleton_methods.include?(method_name.to_s)
+  end
+end
+
+class TestObuf < Test::Unit::TestCase
+  def test_accumulator_saves_objs
+    a = Obuf.new
+    values = [3, {:foo => "bar"}, "foo"]
+    values.map(&a.method(:push))
+    
+    assert_equal 3, a.size
+    assert_equal values, a.map{|e| e }, "Should return the same elements from the storage"
+  end
+  
+  def test_accumulator_saves_shitload_of_objs
+    a = Obuf.new
+    50_000.times { a.push("A string" => rand) }
+    assert_equal 50_000, a.size
+  end
+  
+  def test_accumulator_saves_few_strings_with_a_tab
+    a = Obuf.new
+    4.times { a.push("A \tstring") }
+    a.each {|e| assert_equal "A \tstring", e }
+  end
+  
+  def test_accumulator_empty
+    a = Obuf.new
+    assert a.empty?
+    a.push(1)
+    assert !a.empty?
+  end
+  
+  def test_accumulator_supports_nested_iteration
+    a = Obuf.new
+    ("A".."Z").each{|e| a << e}
+    
+    accumulated = []
+    seen_g = false
+    a.each do | first_level |
+      if first_level == "G"
+        seen_g = true
+        # Force a nested iteration and break it midway
+        a.each do | second_level |
+          accumulated.push(second_level)
+          break if second_level == "E"
+        end
+      elsif seen_g
+        assert_equal "H", first_level
+        return
+      end
+    end
+  end
+  
+  def test_random_access
+    a = Obuf.new
+    letters = ("A".."Z").map{|e| "#{e}\r\nWow!"}.to_a
+    letters.map(&a.method(:push))
+    
+    assert_equal "B\r\nWow!", a[1]
+    assert_equal "E\r\nWow!", a[4]
+  end
+  
+  def test_clear_calls_close_on_buffer
+    io = Tempfile.new("testing")
+    flexmock(io).should_receive(:close!).once
+    flexmock(Tempfile).should_receive(:new).once.with("obuf").and_return(io)
+    
+    a = Obuf.new
+    40.times { a.push("A string" => rand) }
+    assert_equal 40, a.size
+    a.clear
+    assert_equal 0, a.size
+  end
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification 
+name: obuf
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 1.0.0
+platform: ruby
+authors: 
+- Julik Tarkhanov
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-02 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: flexmock
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: "0.8"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: cli_test
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: "1.0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: "2.10"
+  type: :development
+  version_requirements: *id003
+description: A simple Ruby object buffer. Use this if you need to temporarily store alot of serializable Ruby objects.
+email: 
+- me@julik.nl
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.rdoc
+files: 
+- .autotest
+- History.txt
+- Manifest.txt
+- README.rdoc
+- Rakefile
+- lib/obuf.rb
+- test/test_obuf.rb
+- .gemtest
+homepage: http://github.com/julik/obuf
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: obuf
+rubygems_version: 1.8.5
+signing_key: 
+specification_version: 3
+summary: A simple Ruby object buffer
+test_files: 
+- test/test_obuf.rb