external 0.1.0

data/test/ext_arc_test.rb

@@ -0,0 +1,286 @@
+require File.join(File.dirname(__FILE__), 'external_test_helper.rb') 
+require 'ext_arc'
+
+class ExtArcTest < Test::Unit::TestCase
+  include Benchmark
+  
+  attr_reader :ea
+
+  def setup
+    @ea = ExtArc.new
+  end
+  
+  #
+  # readme doc test
+  #
+  
+  def test_readme_doc_for_ext_arc
+    arc = ExtArc[">swift", ">brown", ">fox"]
+    assert_equal ">fox", arc[2]
+    assert_equal [">swift", ">brown", ">fox"], arc.to_a
+
+    assert_equal Tempfile, arc.io.class
+    arc.io.rewind
+    assert_equal ">swift>brown>fox", arc.io.read
+
+  	Tempfile.open('test_readme_doc_for_ext_arc') do |file|
+  	  file << ">swift>brown>fox"
+  	  file.flush
+
+  	  arc = ExtArc.new(file)
+  	  assert_equal [], arc.to_a
+  	  arc.reindex_by_sep(:sep_string => ">", :entry_follows_sep => true)
+  	  assert_equal [">swift", ">brown", ">fox"], arc.to_a
+  	  
+  	  arc = ExtArc.new(file)
+  	  assert_equal [], arc.to_a
+  	  arc.reindex_by_scan(/>\w*/)
+  	  assert_equal [">swift", ">brown", ">fox"], arc.to_a
+  	end
+  end
+
+  #
+  # entry_to_str, str_to_entry test
+  #
+
+  def test_entry_to_str_simply_stringifies_entry
+    obj = "abc"
+    assert_equal obj.to_s, ea.entry_to_str(obj)
+    
+    obj = 1
+    assert_equal obj.to_s, ea.entry_to_str(obj)
+  end
+  
+  def test_entry_to_str_simply_return_input
+    obj = "abc"
+    assert_equal obj.object_id, ea.str_to_entry(obj).object_id
+  end
+  
+  #####################################
+  # indexing tests
+  #####################################
+  
+  def reindex_by_line_test(expected, options={}, &block)
+    cases = {
+      :end_midline => "012\n\n56\n\n9",
+      :end_on_line => "012\n\n56\n\n9\n", 
+      :end_on_break => "012\n\n56\n\n9\n\n", 
+      :no_break => "0123456789",
+      :backing_breaks => "012\n\n\n\n\n\n9",
+      :cr_lf => "012\r\n\r\n56\r\n\r\n9"
+      }
+      
+    cases.each_pair do |key, string|
+      next unless expected.has_key?(key)
+
+      Tempfile.open("reindex_by_line") do |tempfile|
+        # MUST SET binmode so that these tests work properly on Windows
+        tempfile.binmode
+        tempfile << string
+        tempfile.flush
+  
+        begin
+          ea = ExtArc.new(tempfile)
+          ea.reindex_by_line(options, &block)
+          assert_equal expected[key], ea.to_a, key
+        ensure
+          ea.close
+        end
+      end
+    end
+  end
+  
+  def reindex_by_scan_test(expected, carryover_limit=nil, options={}, &block)
+    cases = {
+      :end_midline => "012\n\n56\n\n9",
+      :end_on_line => "012\n\n56\n\n9\n", 
+      :end_on_break => "012\n\n56\n\n9\n\n", 
+      :no_break => "0123456789",
+      :backing_breaks => "012\n\n\n\n\n\n9",
+      :cr_lf => "012\r\n\r\n56\r\n\r\n9"}
+    
+    cases.each_pair do |key, string|
+      next unless expected.has_key?(key)
+
+      Tempfile.open("reindex_by_scan") do |tempfile|
+        tempfile.binmode
+        tempfile << string
+        tempfile.flush
+
+        begin
+          ea = ExtArc.new(tempfile)
+          ea.reindex_by_scan
+          assert_equal expected[key].length, ea.length, key
+          assert_equal expected[key], ea.to_a, key
+        ensure
+          ea.close
+        end
+      end
+    end
+  end
+  
+  def test_reindex_treats_each_line_as_break_by_default
+    reindex_by_line_test(
+      :end_midline => ["012\n", "\n", "56\n", "\n", "9"],
+      :end_on_line => ["012\n", "\n", "56\n", "\n", "9\n"],
+      :end_on_break => ["012\n", "\n", "56\n", "\n", "9\n", "\n"],
+      :no_break => ["0123456789"],
+      :backing_breaks => ["012\n", "\n", "\n", "\n", "\n", "\n", "9"],
+      :cr_lf => ["012\r\n", "\r\n", "56\r\n", "\r\n", "9"])
+   
+    reindex_by_scan_test(
+      :end_midline => ["012\n", "\n", "56\n", "\n", "9"],
+      :end_on_line => ["012\n", "\n", "56\n", "\n", "9\n"],
+      :end_on_break => ["012\n", "\n", "56\n", "\n", "9\n", "\n"],
+      :no_break => ["0123456789"],
+      :backing_breaks => ["012\n", "\n", "\n", "\n", "\n", "\n", "9"],
+      :cr_lf => ["012\r\n", "\r\n", "56\r\n", "\r\n", "9"])
+  end
+
+  def BROKEN_test_reindex_by_scan_with_chunk_size_less_than_full_length
+    reindex_by_scan_test({
+      :end_midline => ["012\n", "\n", "56\n", "\n", "9"],
+      :end_on_line => ["012\n", "\n", "56\n", "\n", "9\n"],
+      :end_on_break => ["012\n", "\n", "56\n", "\n", "9\n", "\n"],
+      :no_break => ["0123456789"],
+      :backing_breaks => ["012\n", "\n", "\n", "\n", "\n", "\n", "9"],
+      :cr_lf => ["012\r\n", "\r\n", "56\r\n", "\r\n", "9"]},
+      nil,
+      :buffer_size => 3)  
+  end
+
+  def test_reindex_block_determines_if_line_is_a_break
+    reindex_by_line_test(
+      :end_midline => ["012\n\n", "56\n\n", "9"],
+      :end_on_line => ["012\n\n", "56\n\n", "9\n"],
+      :end_on_break => ["012\n\n", "56\n\n", "9\n\n"],
+      :no_break => ["0123456789"],
+      :backing_breaks => ["012\n\n", "\n", "\n", "\n", "\n", "9"],
+      :cr_lf => ["012\r\n\r\n", "56\r\n\r\n", "9"]) do |line|
+        line.strip.empty?
+      end
+  end
+  
+  def test_reindex_breaking_before
+    reindex_by_line_test({
+      :end_midline => ["012\n", "\n56\n", "\n9"],
+      :end_on_line => ["012\n", "\n56\n", "\n9\n"],
+      :end_on_break => ["012\n", "\n56\n", "\n9\n", "\n"],
+      :no_break => ["0123456789"],
+      :backing_breaks => ["012\n", "\n", "\n", "\n", "\n", "\n9"],
+      :cr_lf => ["012\r\n", "\r\n56\r\n", "\r\n9"]},
+      :break_before => true) do |line|
+        line.strip.empty?
+      end
+  end
+    
+  def test_reindex_excluding_break
+    reindex_by_line_test({
+      :end_midline => ["012\n", "56\n", "9"],
+      :end_on_line => ["012\n", "56\n", "9\n"],
+      :end_on_break => ["012\n", "56\n", "9\n"],
+      :no_break => ["0123456789"],
+      :backing_breaks => ["012\n", "9"],
+      :cr_lf => ["012\r\n", "56\r\n", "9"]},
+      :exclude_break => true) do |line|
+        line.strip.empty?
+      end
+  end
+  
+  def test_reindex_breaking_before_and_excluding_break
+    # note this is the same as simply excluding the break
+    reindex_by_line_test({
+      :end_midline => ["012\n", "56\n", "9"],
+      :end_on_line => ["012\n", "56\n", "9\n"],
+      :end_on_break => ["012\n", "56\n", "9\n"],
+      :no_break => ["0123456789"],
+      :backing_breaks => ["012\n", "9"],
+      :cr_lf => ["012\r\n", "56\r\n", "9"]},
+      :exclude_break => true,
+      :break_before => true) do |line|
+        line.strip.empty?
+      end
+  end
+  
+  def test_reindex_with_alt_sep_string
+    reindex_by_line_test({
+      :end_midline => ["012\n\n", "56\n\n", "9"],
+      :end_on_line => ["012\n\n", "56\n\n", "9\n"],
+      :end_on_break => ["012\n\n", "56\n\n", "9\n\n"],
+      :no_break => ["0123456789"],
+      :backing_breaks => ["012\n\n", "\n\n", "\n\n",  "9"],
+      :cr_lf => ["012\r\n\r\n56\r\n\r\n9"]},
+      :sep_string => "\n\n")
+      
+    reindex_by_line_test({
+      :end_midline => ["012\n\n56", "\n\n9"],
+      :end_on_line => ["012\n\n56", "\n\n9\n"],
+      :end_on_break => ["012\n\n56", "\n\n9\n\n"],
+      :no_break => ["0123456", "789"],
+      :backing_breaks => ["012\n\n\n\n\n\n9"],
+      :cr_lf => ["012\r\n\r\n56", "\r\n\r\n9"]},
+      :sep_string => "56")
+  end
+
+  # #
+  # # file format tests
+  # #
+  # 
+  # def file_format_test(path, &block)
+  #   begin
+  #     filepath = tempfile('format')
+  #     
+  #     FileUtils.cp(ifs.filepath(:root, path), filepath + ".txt")
+  #     FileUtils.cp(input_base + '.index', filepath + ".index")
+  #     
+  #     uaio = ExtArr.open(filepath + ".txt", 'ru')
+  #     yield(uaio)
+  #   ensure
+  #     uaio.close
+  #   end
+  # end
+  # 
+  # def test_cr_lf_file
+  #   file_format_test('cr_lf_input.txt') do |aio|
+  #     aio.reindex_by_line
+  #         
+  #     assert_equal "012\r\n", aio[0]
+  #     assert_equal "56\r\n", aio[1]
+  #     assert_equal "9", aio[2]
+  #   end
+  # end
+  # 
+  # def test_parse_from_lf_file
+  #   file_format_test('lf_input.txt') do |aio|
+  #     aio.reindex_by_line
+  #         
+  #     assert_equal "012\n", aio[0]
+  #     assert_equal "56\n", aio[1]
+  #     assert_equal "9", aio[2]
+  #   end
+  # end
+  # 
+  # def test_parse_from_alt_sep
+  #   file_format_test('alt_sep.txt') do |aio|
+  #     aio.reindex_by_line do |line|
+  #       line =~ /^>/
+  #     end
+  #   
+  #     assert_equal ">abc\r\n", aio[0]
+  #     assert_equal ">def\r\n", aio[1]
+  #     assert_equal ">gh", aio[2]
+  #   end
+  # end
+
+  #######################
+  #  Benchmarks
+  #######################
+  
+  def break_test_reindex_speed
+    aiobm_test('r') do |x, type, aio|
+      
+      x.report("#{type} by_line") { aio.reindex_by_line }
+      x.report("#{type} by_scan") { aio.reindex_by_scan }
+    end
+  end
+end

data/test/ext_arr/alt_sep.txt

@@ -0,0 +1,3 @@
+>abc
+>def
+>gh

data/test/ext_arr/cr_lf_input.txt

@@ -0,0 +1,3 @@
+012
+56
+9

data/test/ext_arr/input.txt

@@ -0,0 +1 @@
+abcdefgh

data/test/ext_arr/inputb.txt

@@ -0,0 +1 @@
+abcdefgh

data/test/ext_arr/lf_input.txt

@@ -0,0 +1,3 @@
+012
+56
+9

data/test/ext_arr/lines.txt

@@ -0,0 +1,19 @@
+line 1
+
+line 2
+
+line 3
+
+line 4
+
+line 5
+
+line 6
+
+line 7
+
+line 8
+
+line 9
+
+line 10

data/test/ext_arr/without_index.txt

@@ -0,0 +1 @@
+abcdefgh