external 0.1.0

data/test/ext_arr_test.rb

@@ -0,0 +1,534 @@
+require File.join(File.dirname(__FILE__), 'external_test_helper.rb') 
+require 'ext_arr'
+require 'fileutils'
+
+class ExtArrTest < Test::Unit::TestCase
+  include Benchmark
+  include TestArray
+  
+  attr_reader :ea, :tempfile
+
+  def setup
+    # cls represents an array
+    @cls = ExtArr
+    
+    @tempfile = Tempfile.new("eatest")
+    @tempfile << string
+    @tempfile.pos = 0
+
+    @ea = ExtArr.new(@tempfile)
+    @ea.index.concat(index)
+  end
+  
+  def teardown
+    @tempfile.close unless @tempfile.closed?
+  end
+  
+  def string
+    "abcdefgh"
+  end
+  
+  def array
+    ["abc", "de", "fgh"]
+  end
+  
+  def index
+    [[0,3],[3,2],[5,3]]
+  end
+  
+  def filepath(path)
+    File.join(File.dirname(__FILE__), "ext_arr", path)
+  end
+  
+  #
+  # doc tests
+  #
+  
+  def test_readme_doc_for_ext_arr
+    ea = ExtArr[1, 2.2, "cat", {:key => 'value'}]
+    assert_equal "cat", ea[2]  
+    assert_equal({:key => 'value'}, ea.last)
+    ea << [:a, :b]
+    assert_equal [1, 2.2, "cat", {:key => 'value'}, [:a, :b]], ea.to_a 
+    
+    assert_equal Tempfile, ea.io.class 
+    ea.io.rewind
+    assert_equal "--- 1\n--- 2.2\n--- cat\n--- \n:key: value\n--- \n- :a\n- :b\n", ea.io.read
+
+    assert_equal ExtInd, ea.index.class 
+    assert_equal [[0, 6], [6, 8], [14, 8], [22, 17], [39, 15]], ea.index.to_a  
+    
+    Tempfile.open("test_readme_doc_for_ext_arr") do |file|
+  	  file << "--- 1\n--- 2.2\n--- cat\n--- \n:key: value\n--- \n- :a\n- :b\n"
+  	  file.flush
+
+      index_filepath = ExtArr.default_index_filepath(file.path)
+  	  assert !File.exists?(index_filepath)
+
+  	  ea = ExtArr.new(file)
+  	  assert_equal [], ea.to_a 
+  	  ea.reindex 
+  	  assert_equal [1, 2.2, "cat", {:key => 'value'}, [:a, :b]], ea.to_a
+  	end
+  end
+  
+  #
+  # test setup
+  #
+  
+  def test_setup
+    assert_equal ExtArr, @cls
+    
+    assert_equal string, tempfile.read
+    assert_equal tempfile.path, ea.io.path
+    assert_equal index, ea.index.to_a
+  end
+  
+  #
+  # initialize tests
+  #
+  
+  def test_initialize
+    ea = ExtArr.new
+      
+    assert_equal Tempfile, ea.io.class
+    assert_equal "", ea.io.read
+  
+    assert ea.index.cached?
+    assert_equal [], ea.index.to_a
+  end
+  
+  def test_initialize_with_existing_file_and_index
+    File.open(filepath("input.txt")) do |file|
+      ea = ExtArr.new(file)
+      
+      assert_equal 'input.txt', File.basename(ea.io.path)
+      assert_equal string, ea.io.read
+      assert_equal index, ea.index.to_a
+      
+      ea.close
+    end
+  end
+  
+  def test_initialize_load_specified_index_file
+    File.open(filepath('without_index.txt')) do |file|
+      alt_index = filepath('input.index')
+      
+      assert File.exists?(alt_index)
+      
+      ea = ExtArr.new(file, :index_file => alt_index)
+      assert_equal File.read(alt_index).unpack("I*"), ea.index.to_a.flatten
+      ea.close
+    end
+  end
+  
+  def test_initialize_in_uncached_mode_uses_external_index
+    ea = ExtArr.new(nil, :cached => false)
+      
+    assert_equal Tempfile, ea.io.class
+    assert_equal "", ea.io.read
+  
+    assert_equal ExtInd, ea.index.class
+    assert_equal [], ea.index.read
+  end
+  
+  def test_initialize_with_existing_file_and_non_existant_index_file_creates_index_file_in_uncached_mode
+    File.open(filepath('without_index.txt')) do |file|
+      index_file = filepath('without_index.index')
+      begin
+        assert !File.exists?(index_file)
+        ea = ExtArr.new(file, :cached => false)
+      
+        assert_equal ExtInd, ea.index.class
+        assert File.exists?(ea.index.io.path)
+      
+        ea.close
+      ensure
+        FileUtils.rm(index_file) if File.exists?(index_file)
+      end  
+    end
+  end
+  
+  #
+  # test reindex
+  #
+  
+  def test_reindex
+    arr = [1, 2, 3.3, "cat", {:key => 'value'}]
+    StringIO.open("--- 1\n--- 2\n--- 3.3\n--- cat\n--- \n:key: value\n") do |strio|
+      ea = ExtArr.new(strio)
+      ea.reindex
+      
+      assert_equal [[0, 6], [6, 6], [12, 8], [20, 8], [28, 17]], ea.index.to_a
+      assert_equal arr, ea.to_a
+    end
+  end
+  
+  # def test_reindex_yaml_strings
+  #   arr = [[1,2,3].to_yaml]
+  #   StringIO.open(arr.to_yaml) do |strio|
+  #     ea = ExtArr.new(strio)
+  #     ea.reindex
+  #     assert_equal arr, ea.to_a
+  #   end
+  # end
+  
+  
+  #
+  # test close 
+  #
+  
+  def test_close_multiple_times_does_not_raise_error
+    ea.close
+    ea.close
+  end
+  
+  def test_close_closed_index_if_uncached
+    ea = ExtArr.new(nil, :cached => false)
+    assert !ea.index.closed?
+    assert !ea.closed?
+    
+    ea.close
+
+    assert ea.index.closed?
+    assert ea.closed?
+  end
+  
+  #
+  # entry_to_str, str_to_entry test
+  #
+
+  def test_entry_to_str_and_str_to_entry_are_inverse_functions_for_objects_responding_to_to_yaml
+    [
+      nil, true, false,
+      :a, :symbol,
+      '', 'a', "abcde fghij", "1234", "with\nnewline", " \r \n \t  ", " ", "\t", [1,2,3].to_yaml, 
+      0, 1, -1, 1.1, 18446744073709551615,
+      [], [1,2,3], 
+      {}, {:key => 'value', 'another' => 1}, 
+      Time.now
+    ].each do |obj|
+      str = ea.entry_to_str(obj)
+      assert_equal obj, ea.str_to_entry(str)
+    end
+    
+    # FLUNK CASES! 
+    ["\r", "\n", "\r\n", "string_with_\r\n_internal"].each do |obj|
+      str = ea.entry_to_str(obj)
+      assert_not_equal obj, ea.str_to_entry(str)
+    end
+  end
+  
+  def test_strings_and_numerics_can_be_converted_from_their_to_s
+    ['a', "abcde fghij"].each  do |obj|
+      assert obj.kind_of?(String)
+      assert_equal obj, ea.str_to_entry(obj.to_s)
+    end
+    
+    [1, -1, 1.1, 18446744073709551615].each  do |obj|
+      assert obj.kind_of?(Numeric)
+      assert_equal obj, ea.str_to_entry(obj.to_s)
+    end
+  end
+
+  #############################
+  # Modified Array methods tests
+  #############################
+
+  def test_LSHIFT # '<<'
+    a = @cls[]
+    a << 1
+    assert_equal(@cls[1], a)
+    a << 2 << 3
+    assert_equal(@cls[1, 2, 3], a)
+    a << nil << 'cat'
+    assert_equal(@cls[1, 2, 3, nil, 'cat'], a)
+    
+    # Changes: when you add a to itself, the version at 
+    # the << line is added, not the one that appears in the
+    # comparison
+    #a << a
+    #assert_equal(@cls[1, 2, 3, nil, 'cat', a], a)
+    
+    b = @cls.new
+    a << b
+    assert_equal(@cls[1, 2, 3, nil, 'cat', b], a)
+  end
+
+  def test_ASET # '[]='
+    a = @cls[*(0..99).to_a]
+    assert_equal(0, a[0] = 0)
+    assert_equal(@cls[0] + @cls[*(1..99).to_a], a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(0, a[10,10] = 0)
+    assert_equal(@cls[*(0..9).to_a] + @cls[0] + @cls[*(20..99).to_a], a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(0, a[-1] = 0)
+    assert_equal(@cls[*(0..98).to_a] + @cls[0], a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(0, a[-10, 10] = 0)
+    assert_equal(@cls[*(0..89).to_a] + @cls[0], a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(0, a[0,1000] = 0)
+    assert_equal(@cls[0] , a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(0, a[10..19] = 0)
+    assert_equal(@cls[*(0..9).to_a] + @cls[0] + @cls[*(20..99).to_a], a)
+
+    b = @cls[*%w( a b c )]
+    a = @cls[*(0..99).to_a]
+    assert_equal(b, a[0,1] = b)
+    assert_equal(b + @cls[*(1..99).to_a], a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(b, a[10,10] = b)
+    assert_equal(@cls[*(0..9).to_a] + b + @cls[*(20..99).to_a], a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(b, a[-1, 1] = b)
+    assert_equal(@cls[*(0..98).to_a] + b, a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(b, a[-10, 10] = b)
+    assert_equal(@cls[*(0..89).to_a] + b, a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(b, a[0,1000] = b)
+    assert_equal(b , a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(b, a[10..19] = b)
+    assert_equal(@cls[*(0..9).to_a] + b + @cls[*(20..99).to_a], a)
+
+    # Ruby 1.8 feature change:
+    # assigning nil does not remove elements.
+=begin
+    a = @cls[*(0..99).to_a]
+    assert_equal(nil, a[0,1] = nil)
+    assert_equal(@cls[*(1..99).to_a], a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(nil, a[10,10] = nil)
+    assert_equal(@cls[*(0..9).to_a] + @cls[*(20..99).to_a], a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(nil, a[-1, 1] = nil)
+    assert_equal(@cls[*(0..98).to_a], a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(nil, a[-10, 10] = nil)
+    assert_equal(@cls[*(0..89).to_a], a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(nil, a[0,1000] = nil)
+    assert_equal(@cls[] , a)
+
+    a = @cls[*(0..99).to_a]
+    assert_equal(nil, a[10..19] = nil)
+    assert_equal(@cls[*(0..9).to_a] + @cls[*(20..99).to_a], a)
+=end
+
+    # Changes: should have @cls in definition
+
+    a = @cls[1, 2, 3]
+    a[1, 0] = a
+    #assert_equal([1, 1, 2, 3, 2, 3], a)
+    assert_equal(@cls[1, 1, 2, 3, 2, 3], a)
+
+    a = @cls[1, 2, 3]
+    a[-1, 0] = a
+    #assert_equal([1, 2, 1, 2, 3, 3], a)
+    assert_equal(@cls[1, 2, 1, 2, 3, 3], a)
+  end
+  
+  def test_concat
+    # Currently there are issues with this...
+    
+    # assert_equal(@cls[1, 2, 3, 4],     @cls[1, 2].concat(@cls[3, 4]))
+    # assert_equal(@cls[1, 2, 3, 4],     @cls[].concat(@cls[1, 2, 3, 4]))
+    # assert_equal(@cls[1, 2, 3, 4],     @cls[1, 2, 3, 4].concat(@cls[]))
+    # assert_equal(@cls[],               @cls[].concat(@cls[]))
+    # assert_equal(@cls[@cls[1, 2], @cls[3, 4]], @cls[@cls[1, 2]].concat(@cls[@cls[3, 4]]))
+    # 
+    # a = @cls[1, 2, 3]
+    # a.concat(a)
+    # assert_equal([1, 2, 3, 1, 2, 3], a)
+  end
+  
+  def test_eql?
+    assert(@cls[].eql?(@cls[]))
+    assert(@cls[1].eql?(@cls[1]))
+    assert(@cls[1, 1, 2, 2].eql?(@cls[1, 1, 2, 2]))
+    
+    # Changes: converting values to strings and back to 
+    # numerics causes equal values to be equal, regardless
+    # of whether they were entered as different types initially
+    #assert(!@cls[1.0, 1.0, 2.0, 2.0].eql?(@cls[1, 1, 2, 2]))
+    assert(@cls[1.0, 1.0, 2.0, 2.0].eql?(@cls[1, 1, 2, 2]))
+  end
+  
+  def test_to_a
+    a = @cls[ 1, 2, 3 ]
+    a_id = a.__id__
+    assert_equal(a, a.to_a)
+    
+    # Changes: can't do this comparison by object id
+    #assert_equal(a_id, a.to_a.__id__)
+  end
+
+  
+  #######################
+  # Benchmark tests
+  #######################
+
+  #
+  # benchmarks
+  #
+  
+  def ea_bm_test(mode, length=20, array=nil, &block)
+    benchmark_test(length) do |x|
+      unless array
+        array = []
+        1.upto(10000) {|i| array << i.to_s }
+      end
+       
+      begin
+        ea = ExtArr[*array]
+        yield(x, "", ea)
+      ensure
+        ea.close if index
+      end
+
+      yield(x, "array reference", array)
+    end
+     
+    array
+  end
+  
+  def test_element_reference_speed_for_ext_arr
+    n = 10
+    ea_bm_test('r') do |x, type, ea|
+      puts type
+      
+      x.report("#{n}kx [index]") { (n*1000).times { ea[1000] } }
+      x.report("#{n}kx [range]") { (n*1000).times { ea[1000..1000] } }
+      x.report("#{n}kx [s,1]") { (n*1000).times { ea[1000, 1] } }
+      x.report("#{n}kx [s,10]") { (n*1000).times { ea[1000, 10] } }
+      #x.report("#{n}kx [s,100]") { (n*1000).times { ea[1000, 100] } }   
+      
+      puts
+    end
+  end
+
+  def test_element_assignment_speed_for_ext_arr
+    ea_bm_test('r+') do |x, type, index|
+      puts type
+      
+      n = 1
+      obj = "abcde"
+      
+      x.report("#{n}kx [index]=") do 
+        (n*1000).times { ea[1000] = obj  } 
+      end      
+      x.report("#{n}kx [range]=") do 
+        (n*1000).times { ea[1000..1000] = [obj] } 
+      end  
+      x.report("#{n}kx [s,1]=") do 
+        (n*1000).times { ea[1000,1] = [obj] } 
+      end
+      
+      puts
+    end
+  end
+  
+end
+
+class HOLD
+  
+  #
+  # scan collect
+  #
+  
+  def test_scan_collect_collects_block_results
+    aio_test do |aio|
+      indicies = []
+      offsets = []
+      result = aio.scan_collect do |scanner, offset, index|
+        offsets << offset
+        indicies << index
+        scanner.scan(/\w/)
+      end
+      
+      assert_equal [0, 0, 0], offsets
+      assert_equal index, indicies
+      assert_equal ['a','d', 'f'], result
+      
+      # now with reduced chunk size
+      aio.chunk_size = 3
+      indicies = []
+      offsets = []
+      result = aio.scan_collect do |scanner, offset, index|
+        offsets << offset
+        indicies << index
+        scanner.scan(/\w/)
+      end
+      
+      assert_equal [0, 3, 5], offsets
+      assert_equal index, indicies
+      assert_equal ['a','d', 'f'], result
+    end
+  end
+  
+  def test_scan_collect_with_subsets
+    aio_test do |aio|
+      case_test(
+        #[0] => ['a'],
+        #[1] => ['a'],
+        #[-4] => nil,
+        [0..1] => ['a','d'],
+        [0...1] => ['a'],
+        [0...0] => [],
+        [0, 2] => ['a','d'],
+        [0, 0] => []
+      ) do |subset, expected|
+        indicies = []
+        result = aio.scan_collect(aio.index[*subset]) do |scanner, offset, index|
+          indicies << index
+          scanner.scan(/\w/)
+        end
+        
+        assert_equal index[*subset], indicies, subset
+        assert_equal expected, result, subset
+      end
+    end
+  end
+  
+  def test_scan_collect_raises_error_on_scan_past_index_end
+    aio_test do |aio|
+      assert_raise(RuntimeError) do
+        aio.scan_collect do |scanner, offset, index|
+          scanner.skip_until(/$/)
+        end
+      end
+    end
+  end
+  
+  #
+  # chunk test
+  #
+  
+  def test_chunk_chunks_on_io_length_not_length
+    saio = setup_saio
+    assert_equal 3, saio.length
+    assert_equal string.length, saio.io.length
+    
+    saio.chunk do |o, l|
+      assert_equal [0, saio.io.length], [o,l]
+    end
+  end
+end