arrayio 0.1.0

data/README

@@ -0,0 +1,76 @@
+# = ArrayIO 
+# Array-like behavior for archival files.  
+#
+# == Introduction
+# Archival files contain many entries following a regular pattern.  These files often grow very large, making them
+# inconvenient or impossible to parse directly into memory.  ArrayIO provides an easy way to index these files
+# so that array-like calls can retrieve entries on the fly.
+#
+# Internally ArrayIO keeps an IO object for the archive, and an index of ranges recording where each entry
+# begins and ends.  The index is an array, unless ArrayIO operates in 'uncached' mode.  In this case the index is
+# a separate file from which the ranges can be looked up.  Uncached mode is useful when dealing with an extremely
+# large number of ranges (entries) that would chew up lots of memory if kept in an array.
+#
+# When operating in a writable mode, entries can be added, inserted, and deleted.  All changes are recorded in the index
+# and will not reflect the actual order of entries in the archive unless consolidated.  
+#
+# Example: If you add an entry at index 1000, the index records the range at index 1000, but the actual entry is appended 
+# to the end of the archive.  If you delete an entry, it remains in the archive, but the range is removed from the index.  
+# Consolidation re-writes entries in their proper order and removes deleted entries.
+#
+# Notes:
+#   - BE CAREFUL to specify the correct mode when you open up an ArrayIO - as with File, a 'w' mode will overwrite 
+#   the ENTIRE archive file.  It is safer to use the append mode 'a'.
+#
+# Copyright (c) 2007 Simon Chiang
+# Version:	0.1
+# Licence:	MIT-Style
+#
+# == Usage
+#
+#   # Open, autoindex and work with 'archive.txt'
+#   ArrayIO.open('archive.txt', 'r') do |aio|
+#     aio[100] # -> entry 100
+#     aio[100] = "new entry" # -> reassigns entry 100
+#     aio.each do |entry|
+#       # do something
+#     end
+#   end
+#
+#   # Open 'archive.txt' in uncached mode.  This creates a file 'archive.index'
+#   # that will be filled with the entry ranges.  You can specify where entries 
+#   # begin and end using options and a block in +reindex+.
+#   # If the block returns true, the line in considered the beginning of
+#   # an entry.  This block looks for entries delmited by '>' like:
+#   #   > entry 0
+#   #   still entry0
+#   #   > entry 1
+#   #   
+#   aio = ArrayIO.new('archive.txt', 'ru')
+#   aio.reindex do |line|
+#     line =~ /^>/
+#   end
+#   aio.close
+#
+#  # Subclass ArrayIO by overwriting +str_to_entry+, +entry_to_str+, and +reindex+
+#  # EntryIO parses entries as above, and functions like:
+#  #  entryio[0] # => [0, "\nstill entry 0"] 
+#  #  entryio[1] # => [1, ""] 
+#  #  entryio[1] = [100, " is the new entry"] # => writes "> entry 100 is the new entry"
+#  #  entryio[1] # => [100, " is the new entry"] 
+#  class EntryIO
+#    def str_to_entry(str)
+#       str =~ /^> entry (\d+)(.*)$"
+#       [$1, $2]
+#    end
+#
+#    def entry_to_str(entry)
+#      "> entry #{entry[0]}#{entry[1]}"
+#    end
+#
+#    def reindex(options={}, &block)
+#      super(options) do |line|
+#         block_given? ? yield(line) : line =~ /^>/
+#      end
+#    end
+#  end

data/lib/array_io.rb

@@ -0,0 +1,381 @@
+require 'stringio'
+
+class ArrayIO
+	attr_reader :io, :io_index
+	
+	include Enumerable
+	
+	class << self
+		def open(fd, mode='r', index_file=nil, auto_index=true, &block)
+			aio = self.new(fd, mode, index_file)
+			
+			if block_given?
+				aio.reindex if auto_index && aio.empty?
+				yield(aio)
+				aio.close
+			else
+				aio
+			end
+		end
+	end
+	
+	def initialize(fd, mode='r', io_index=nil)
+		if mode =~ /u/i
+			mode = mode.delete('u')
+			@cached = false
+		else
+			@cached = true
+		end
+		
+		if mode =~ /s/i
+			mode = mode.delete('s')
+			@strio = true
+		else
+			@strio = false
+		end
+		
+		if strio?
+			mode = mode.delete('s')
+			@io = StringIO.open(fd, mode)
+			io_index = '' if io_index.nil?
+		else
+			@io = File.open(fd, mode)
+			set_binmode(io)
+			io_index = default_index_file if io_index.nil?
+		end
+
+		self.io_index = io_index
+	end
+	
+	def default_index_file
+		io.path.chomp(File.extname(io.path)) + '.index'
+	end
+	
+	#
+	# array functionality
+	#
+	
+	def size
+		length
+	end
+	
+	def length 
+		if cached?
+			io_index.length
+		else
+			index_length/RANGE_SIZE
+		end
+	end
+	
+	def at(index)
+		self[index]
+	end
+	
+	def [](input, length=nil)
+		range = range(input, length)
+		case range
+		when Array
+			range.collect { |r| read_entry(r) }
+		else
+			read_entry(range)
+		end
+	end
+	
+	def []=(input, entry)
+		string = entry_to_str(entry)
+		
+		range_begin = strio? ? io.string.length :  io.stat.size 
+		range_end = range_begin + string.length
+		range = format_range(range_begin, range_end)
+		
+		# do this first in case io is not open for writing.
+		set_pos(io, range_begin)
+		io << string
+		
+		if cached?
+			io_index[input] = range
+		else
+			pos = index_pos(input)
+			if strio?
+				io_index.string[pos...pos+RANGE_SIZE] = range
+			else
+				#io_index.close
+			end
+		end
+	end
+	
+	def fetch(index, default=nil, &block)
+		index += index_length if index < 0 
+		val = (index >= length ? default : self[index])
+		block_given? ? yield(val) : val
+	end
+	
+	def first(n=nil)
+		n.nil? ? self[0] : self[0,n]
+	end
+	
+	def each_index(&block)
+		raise LocalJumpError("no block given") unless block_given?
+		
+		0.upto(length-1, &block)
+	end
+
+	def each(&block)
+		raise LocalJumpError("no block given") unless block_given?
+		
+		if cached?
+			io_index.each do |range|
+				yield( read_entry(range) )
+			end
+		else
+			io_index.pos = 0
+			while !io_index.eof?
+				begin_index, end_index = io_index.read(RANGE_SIZE).unpack(FORMAT)
+				yield( read_entry(begin_index..end_index) )
+			end
+		end
+	end
+	
+	def empty?
+		length == 0
+	end
+	
+	def last(n=nil)
+		return self[-1] if n.nil?
+		
+		start = length-n
+		start = 0 if start < 0
+		self[start, n]
+	end
+	
+	def values_at(*selectors)
+		selectors.collect {|s| self[s]}.flatten
+	end
+	
+	#
+	# arrayio functionality
+	#
+	
+	def cached?
+		@cached
+	end
+	
+	def strio?
+		@strio
+	end
+	
+	def load_index(index_file)
+		input = strio? ? index_file : (File.exists?(index_file) ? File.read(index_file) : '')
+		@io_index = parse_index(input)
+	end
+	
+	def dump_index(index_file=default_index_file)
+		dumping_to_io = io_index.respond_to?(:path) ? 
+				File.expand_path(io_index.path) != File.expand_path(index_file) :
+				false
+
+		File.open(index_file, 'w') do |file|
+			set_binmode(file)
+			if cached?
+				io_index.each do |range|
+					file << [range.begin, range.end].pack(FORMAT)
+				end
+			else
+				file << io_index.read
+			end
+		end unless dumping_to_io
+		
+		index_file
+	end
+	
+	def close
+		io.close
+		io_index.close unless cached?
+	end
+	
+	def str_to_entry(string)
+		string
+	end
+	
+	def entry_to_str(entry)
+		entry.to_s
+	end
+
+	def range(input, length=nil)
+		if cached?
+			if length.nil?
+				return io_index[input]
+			else
+				return io_index[input, length] 
+			end
+		end
+			
+		unless length.nil?
+			return nil if length < 0
+			input = input...(input + length)
+		end
+
+		if input.kind_of?(Range)
+			begin_pos = index_pos(input.begin)
+			return nil if begin_pos < 0 || begin_pos >= index_length 
+			
+			end_pos = index_pos(input.end) + (input.exclude_end? ? 0 : RANGE_SIZE)
+			read_length = end_pos-begin_pos
+			return [] if read_length <= 0
+
+			set_pos(io_index, begin_pos)
+			parse_index(io_index.read(read_length))
+		else
+			begin_pos = index_pos(input)
+			return nil if begin_pos < 0 || begin_pos >= index_length
+	
+			set_pos(io_index, begin_pos)
+			array = io_index.read(RANGE_SIZE).unpack(FORMAT)
+			array.first...array.last
+		end
+	end
+
+	def reindex(options={}, &block)
+		io.rewind
+		
+		options = {
+			:sep_string => $/,
+			:break_before => false,
+			:exclude_break => false,
+			:limit => nil
+		}.merge(options)
+		
+		sep_string = options[:sep_string]
+		break_before = options[:break_before]
+		exclude_break = options[:exclude_break]
+		
+		limit = options[:limit]
+		total_count = 0
+		
+		if cached?
+			self.io_index = []
+		else
+			io_index.close 
+			if strio?
+				self.io_index = ''
+			else
+				self.io_index = io_index.path
+			end
+		end
+		
+		last_pos = 0
+		current_pos = 0
+		range_begin = 0
+		io.each_line(sep_string) do |line|
+			# Note positions MUST be built up using line.length
+			# io.pos cannot return positions greater than ~2.1e9
+			last_pos = current_pos
+			current_pos += line.length
+			
+			if (block_given? ? yield(line) : true)
+				range_end = (break_before || exclude_break) ? last_pos : current_pos
+				unless range_end == range_begin
+					io_index << format_range(range_begin, range_end)
+					total_count += 1
+				end
+				range_begin = (break_before && !exclude_break) ? last_pos : current_pos
+		
+				break unless limit.nil? || total_count < limit
+			end
+		end
+		if limit.nil? || total_count < limit
+			range_end = current_pos
+			unless range_end == range_begin
+				io_index << format_range(range_begin, range_end)
+			end
+		end
+	
+		# this must be done to re-stat Files, assuring index_length is the length of the io_index file.
+		unless cached? || strio?
+			io_index.close 
+			self.io_index = io_index.path 
+		end
+	
+		self
+	end
+	
+	protected
+	
+	def read_entry(range)
+		return nil if range.nil?
+		
+		set_pos(io, range.begin)
+		str_to_entry( io.read(range.end-range.begin) )
+	end
+	
+	POSITION_MAX = 2100000000
+	# Positions larger than ~2.1e9 cannot be directly given to +pos+ for File objects.
+	# +set_pos+ incrementally seeks to positions beyond the maximum, if necessary.
+	def set_pos(io, pos)
+		if pos < POSITION_MAX
+			io.pos = pos
+		else
+			# note sysseek appears to be necessary here, rather than io.seek
+			io.pos = 0
+			while pos > POSITION_MAX
+				pos -= POSITION_MAX
+				io.sysseek(POSITION_MAX, IO::SEEK_CUR)
+			end
+			io.sysseek(pos, IO::SEEK_CUR)
+		end
+	end
+	
+	FORMAT = 'I*'
+	RANGE_SIZE = 8
+
+	def format_range(range_begin, range_end)
+		cached? ? range_begin...range_end : [range_begin, range_end].pack(FORMAT)
+	end
+	
+	def index_length
+		strio? ? io_index.string.length : io_index.stat.size 
+	end
+	
+	def index_pos(index)
+		RANGE_SIZE*index + (index < 0 ? index_length : 0)
+	end
+	
+	def parse_index(input)
+		last = nil
+		results = []
+		input.unpack(FORMAT).each do |current|
+			if last.nil?
+				last = current 
+			else
+				results << (last...current)
+				last = nil
+			end
+		end
+		results
+	end
+	
+	def io_index=(input)
+		if input.kind_of?(Array) || input.kind_of?(StringIO) || input.kind_of?(File)
+			@io_index = input
+		else
+			case 
+			when strio? && cached?
+				load_index(input) 
+			when strio?
+				@io_index = StringIO.new(input, 'a+')
+			when cached?
+				load_index(input) 
+			else
+				@io_index = File.open(input, 'a+')
+				set_binmode(io_index)
+			end
+		end
+	end
+	
+	# ArrayIO requires Files to operate in binmode on Windows.  Otherwise ranges get
+	# improperly shifted, and additionally the unpacking frame gets shifted due to improper
+	# handling of cr (\r) characters.
+	def set_binmode(file)
+		file.binmode unless RUBY_PLATFORM.index('mswin').nil?
+	end
+end

data/lib/inspect_array_io.rb

@@ -0,0 +1,28 @@
+require 'array_io'
+require 'pp'
+
+class InspectArrayIO
+	class << self
+		def inspect(aio, options={})
+			options = {
+				:from_start => 100,
+				:from_end => 100}.merge(options)
+		
+			puts "N indexed entries: #{aio.length}"
+		
+			puts "****************************"
+			puts "First #{options[:from_start]} bytes:"
+			aio.io.pos = 0
+			pp aio.io.read(options[:from_start])
+			puts "First entry:"
+			puts aio[0]
+			
+			puts "****************************"
+			puts "Last #{options[:from_end]} bytes:"
+			aio.io.seek(-options[:from_end], IO::SEEK_END)
+			pp aio.io.read(options[:from_end] + 10)
+			puts "Last entry:"
+			puts aio[-1]
+		end
+	end
+end

data/test/array_io/alt_sep.txt

@@ -0,0 +1,3 @@
+>abc
+>def
+>gh

data/test/array_io/cr_lf_input.txt

@@ -0,0 +1,3 @@
+012
+56
+9

data/test/array_io/input.txt

@@ -0,0 +1 @@
+abcdefgh

data/test/array_io/inputb.txt

@@ -0,0 +1 @@
+abcdefgh

data/test/array_io/lf_input.txt

@@ -0,0 +1,3 @@
+012
+56
+9

data/test/array_io/lines.txt

@@ -0,0 +1,19 @@
+line 1
+
+line 2
+
+line 3
+
+line 4
+
+line 5
+
+line 6
+
+line 7
+
+line 8
+
+line 9
+
+line 10

data/test/array_io/without_index.txt

@@ -0,0 +1 @@
+abcdefgh

data/test/array_io_test.rb

@@ -0,0 +1,569 @@
+require File.join(File.dirname(__FILE__), 'arrayio_test_helper.rb') 
+
+class ArrayIOTest < Test::Unit::TestCase
+	include Benchmark
+	
+
+	def aio_filepath(path)
+		File.join(File.dirname(__FILE__), 'array_io', path)
+	end
+	
+	#
+	# file initialize tests
+	#
+	
+	def test_initialize_with_index
+		aio = ArrayIO.new(aio_filepath('input.txt'))
+		
+		assert_equal 'input.txt', File.basename(aio.io.path)
+		assert_equal "abcdefgh", aio.io.read
+		
+		assert_equal [1...1, 2...4, 6...10], aio.io_index
+		aio.close
+	end
+		
+	def test_initialize_uncached_with_index
+		# non-windows only
+		if RUBY_PLATFORM.index('mswin').nil?
+			aio = ArrayIO.new(aio_filepath('input.txt'), 'ru')
+			
+			assert aio.io_index.kind_of?(File)
+			assert_equal 'input.index', File.basename(aio.io_index.path)
+			assert_equal [1,1,2,4,6,10], aio.io_index.read.unpack('I*')
+			
+			aio.close
+		end
+	end
+	
+	def test_initialize_uncached_with_index_binary_mode
+		# windows only
+		unless RUBY_PLATFORM.index('mswin').nil?
+			aio = ArrayIO.new(aio_filepath('inputb.txt'), 'ru')
+			
+			assert aio.io_index.kind_of?(File)
+			assert_equal 'inputb.index', File.basename(aio.io_index.path)
+			assert_equal [1,1,2,4,6,10], aio.io_index.read.unpack('I*')
+			
+			aio.close
+		end
+	end
+	
+	def test_initialize_without_index
+		aio = ArrayIO.new(aio_filepath('without_index.txt'))
+		
+		assert_equal [], aio.io_index
+		aio.close
+	end
+	
+	def test_initialize_with_alternate_index
+		aio = ArrayIO.new(aio_filepath('without_index.txt'), 'r', aio_filepath('input.index'))
+		
+		assert_equal [1...1, 2...4, 6...10], aio.io_index
+		aio.close
+	end
+	
+	def test_initialize_with_array_index
+		array = [0...1,2...3]
+		aio = ArrayIO.new(aio_filepath('input.txt'), 'r', array)
+		assert_equal array, aio.io_index
+	end
+		
+	#
+	# string initialize tests
+	#
+	
+	def test_initialize_string
+		aio = ArrayIO.new('abcdefgh', 'rs')
+		
+		assert aio.io.kind_of?(StringIO)
+		assert_equal "abcdefgh", aio.io.read
+		
+		assert_equal [], aio.io_index
+		aio.close
+	end
+	
+	def test_initialize_string_uncached
+		aio = ArrayIO.new('abcdefgh', 'rsu')
+		
+		assert aio.io_index.kind_of?(StringIO)
+		assert_equal '', aio.io_index.string
+		aio.close
+	end
+	
+	def setup_aio
+		ArrayIO.new("abcdefgh", 'r+s', [0...3, 3...5, 5...8])
+	end
+	
+	def test_initialize_string_with_array
+		aio = setup_aio
+		
+		assert aio.io.kind_of?(StringIO)
+		assert_equal "abcdefgh", aio.io.read
+		
+		assert_equal [0...3, 3...5, 5...8], aio.io_index
+		aio.close
+	end
+	
+	def setup_uaio
+		ArrayIO.new("abcdefgh", 'r+su', [0,3,3,5,5,8].pack('I*'))
+	end
+	
+	def test_initialize_string_uncached_with_index
+		uaio = setup_uaio
+		
+		assert uaio.io.kind_of?(StringIO)
+		assert_equal "abcdefgh", uaio.io.read
+		
+		assert uaio.io_index.kind_of?(StringIO)
+		assert_equal [0,3,3,5,5,8], uaio.io_index.read.unpack('I*')
+		uaio.close
+	end
+	
+	#
+	# range tests
+	#
+
+	def test_range
+		[setup_aio, setup_uaio].each do |aio|
+			assert_equal 0...3, aio.range(0)
+			assert_equal 3...5, aio.range(1)
+			assert_equal 5...8, aio.range(2)
+			assert_nil aio.range(3)
+			
+			assert_equal 5...8, aio.range(-1)
+			assert_equal 3...5, aio.range(-2)
+			assert_equal 0...3, aio.range(-3)
+			assert_nil aio.range(-4)
+		end
+	end
+
+	def test_range_when_input_is_start_length
+		array = [0...3, 3...5, 5...8]
+
+		[setup_aio, setup_uaio].each do |aio|
+			[0..0, 0...0, 0..1, 0..2, 0..-1, 0..3, 0...3, -4..1, 5..10, -3..-5].each do |r|
+				start = r.begin
+				length = r.end - r.begin
+				
+				assert_equal array[start, length], aio.range(start, length), r.to_s
+			end
+		end
+	end
+	
+	def test__range_when_input_is_range
+		array = [0...3, 3...5, 5...8]
+		
+		[setup_aio, setup_uaio].each do |aio|
+			[0..0, 0...0, 0..1, 0..2, 0..-1, 0..3, 0...3, -4..1, 5..10, -3..-5].each do |r|
+				assert_equal array[r], aio.range(r), r.to_s
+			end	
+		end		
+	end
+
+	def test_range_speed
+		benchmark_test do
+			aio = setup_aio
+			uaio = setup_uaio
+			bm(12) do |x|
+				x.report("10k index") { 10000.times { aio.range(1) } }
+				x.report("10k range") { 10000.times { aio.range(1..1) } }
+				x.report("10k s,l") { 10000.times { aio.range(1, 1) } }
+		
+				x.report("10k uindex") { 10000.times { uaio.range(1) } }
+				x.report("10k urange") { 10000.times { uaio.range(1..1) } }
+				x.report("10k us,l") { 10000.times { uaio.range(1, 1) } }
+			end
+		end
+	end
+	
+	#
+	# get tests
+	#
+	
+	def test_get_when_input_is_index
+		[setup_aio, setup_uaio].each do |aio|
+			assert_equal "abc", aio[0]
+			assert_equal "de", aio[1]
+			assert_equal "fgh", aio[2]
+			assert_nil aio[3]
+			
+			assert_equal "fgh", aio[-1]
+			assert_equal "de", aio[-2]
+			assert_equal "abc", aio[-3]
+			assert_nil aio[-4]
+		end
+	end
+
+	def test_get_when_input_is_start_length
+		array = ["abc", "de", "fgh"]
+		
+		[setup_aio, setup_uaio].each do |aio|
+			[0..0, 0...0, 0..1, 0..2, 0..-1, 0..3, 0...3, -4..1, 5..10, -3..-5, -1..5].each do |r|
+				start = r.begin
+				length = r.end - r.begin
+				
+				assert_equal array[start, length], aio[start, length], r.to_s
+			end
+		end
+	end
+
+	def test_get_when_input_is_range
+		array = ["abc", "de", "fgh"]
+		
+		[setup_aio, setup_uaio].each do |aio|
+			[0..0, 0...0, 0..1, 0..2, 0..-1, 0..3, 0...3, -4..1, 5..10, -3..-5].each do |r|
+				assert_equal array[r], aio[r], r.to_s
+			end
+		end
+	end
+	
+	def test_get_speed
+		benchmark_test do
+			array = []
+			0.upto(10000-1) {|i| array << "line #{i}\n" }
+			
+			aio = ArrayIO.new(array.join(''), 'rs')	
+			aio.reindex
+			
+			uaio = ArrayIO.new(array.join(''), 'rsu')	
+			uaio.reindex
+
+			assert_equal "line 1000\n", aio[1000]
+			bm(12) do |x|
+				x.report("10k index") { 10000.times { aio[1000] } }
+				x.report("10k range") { 10000.times { aio[1000...1000] } }
+				x.report("10k s,l") { 10000.times { aio[1000, 1] } }
+				
+				x.report("10k uindex") { 10000.times { uaio[1000] } }
+				x.report("10k urange") { 10000.times { uaio[1000..1000] } }
+				x.report("10k us,l") { 10000.times { uaio[1000, 1] } }
+			end
+		end
+	end
+
+	#
+	# test insert
+	#
+
+	def test_insert
+		[setup_aio, setup_uaio].each do |aio|
+			assert_equal "abc", aio[0]
+			assert_equal "fgh", aio[-1]
+			assert_equal "de", aio[1]
+			assert_equal 3, aio.length
+			
+			aio[0] = 'xyz'
+			aio[-1] = 'pq'
+			aio[1] = 'mno'
+			
+			assert_equal "xyz",aio[0]
+			assert_equal "pq", aio[-1]
+			assert_equal "mno", aio[1]
+			assert_equal 3, aio.length
+			
+			assert_equal 'abcdefghxyzpqmno', aio.io.string
+		end
+	end
+	
+	def test_insert_raises_error_if_io_is_not_writable
+		aio = ArrayIO.new('', 'rs')
+		assert_raise(IOError) { aio[0] = 'abc'}
+	end
+	
+	def test_insert_speed
+		benchmark_test do
+			array = []
+			0.upto(10000-1) {|i| array << "line #{i}\n" }
+			
+			aio = ArrayIO.new(array.join(''), 'r+s')	
+			aio.reindex
+			
+			uaio = ArrayIO.new(array.join(''), 'r+su')	
+			uaio.reindex
+			
+			bm(12) do |x|
+				x.report("10k index=") { 10000.times { aio[1000] = "line 1000\n" } }
+				#x.report("10k range=") { 10000.times { aio[1000..1000] = "line 1000\n" } }
+				#x.report("10k s,l=") { 10000.times { aio[1000, 1] = "line 1000\n" } }
+				
+				x.report("10k uindex=") { 10000.times { uaio[1000] = "line 1000\n" } }
+				#x.report("10k crange=") { 10000.times { caio[1000..1000] = "line 1000\n" } }
+				#x.report("10k cs,l=") { 10000.times { caio[1000, 1] = "line 1000\n" } }
+			end
+		end
+	end
+
+	#
+	# indexing tests
+	#
+	
+	def reindex_test(expected, options={}, &block)
+		cases = {
+			:end_midline => "012\n\n56\n\n9",
+			:end_on_line => "012\n\n56\n\n9\n", 
+			:end_on_break => "012\n\n56\n\n9\n\n", 
+			:no_break => "0123456789",
+			:backing_breaks => "012\n\n\n\n\n\n9",
+			:cr_lf => "012\r\n\r\n56\r\n\r\n9"}
+			
+		cases.each_pair do |key, string|
+			next unless expected.has_key?(key)
+			
+			['rs', 'rsu'].each do |mode|
+				aio = ArrayIO.new(string, mode)
+				aio.reindex(options, &block)
+				
+				assert_equal expected[key].length, aio.length, "#{mode} #{key}"
+				assert_equal expected[key], aio[0..-1], "#{mode} #{key}"
+			end
+		end
+	end
+	
+	def test_reindex_treats_each_line_as_break_by_default
+		reindex_test(
+			:end_midline => ["012\n", "\n", "56\n", "\n", "9"],
+			:end_on_line => ["012\n", "\n", "56\n", "\n", "9\n"],
+			:end_on_break => ["012\n", "\n", "56\n", "\n", "9\n", "\n"],
+			:no_break => ["0123456789"],
+			:backing_breaks => ["012\n", "\n", "\n", "\n", "\n", "\n", "9"],
+			:cr_lf => ["012\r\n", "\r\n", "56\r\n", "\r\n", "9"])
+	end
+	
+	def test_reindex_block_determines_if_line_is_a_break
+		reindex_test(
+			:end_midline => ["012\n\n", "56\n\n", "9"],
+			:end_on_line => ["012\n\n", "56\n\n", "9\n"],
+			:end_on_break => ["012\n\n", "56\n\n", "9\n\n"],
+			:no_break => ["0123456789"],
+			:backing_breaks => ["012\n\n", "\n", "\n", "\n", "\n", "9"],
+			:cr_lf => ["012\r\n\r\n", "56\r\n\r\n", "9"]) do |line|
+				line.strip.empty?
+			end
+	end
+	
+	def test_reindex_breaking_before
+		reindex_test({
+			:end_midline => ["012\n", "\n56\n", "\n9"],
+			:end_on_line => ["012\n", "\n56\n", "\n9\n"],
+			:end_on_break => ["012\n", "\n56\n", "\n9\n", "\n"],
+			:no_break => ["0123456789"],
+			:backing_breaks => ["012\n", "\n", "\n", "\n", "\n", "\n9"],
+			:cr_lf => ["012\r\n", "\r\n56\r\n", "\r\n9"]},
+			:break_before => true) do |line|
+				line.strip.empty?
+			end
+	end
+		
+	def test_reindex_excluding_break
+		reindex_test({
+			:end_midline => ["012\n", "56\n", "9"],
+			:end_on_line => ["012\n", "56\n", "9\n"],
+			:end_on_break => ["012\n", "56\n", "9\n"],
+			:no_break => ["0123456789"],
+			:backing_breaks => ["012\n", "9"],
+			:cr_lf => ["012\r\n", "56\r\n", "9"]},
+			:exclude_break => true) do |line|
+				line.strip.empty?
+			end
+	end
+	
+	def test_reindex_breaking_before_and_excluding_break
+		# note this is the same as simply excluding the break
+		reindex_test({
+			:end_midline => ["012\n", "56\n", "9"],
+			:end_on_line => ["012\n", "56\n", "9\n"],
+			:end_on_break => ["012\n", "56\n", "9\n"],
+			:no_break => ["0123456789"],
+			:backing_breaks => ["012\n", "9"],
+			:cr_lf => ["012\r\n", "56\r\n", "9"]},
+			:exclude_break => true,
+			:break_before => true) do |line|
+				line.strip.empty?
+			end
+	end
+	
+	def test_reindex_with_alt_sep_string
+		reindex_test({
+			:end_midline => ["012\n\n", "56\n\n", "9"],
+			:end_on_line => ["012\n\n", "56\n\n", "9\n"],
+			:end_on_break => ["012\n\n", "56\n\n", "9\n\n"],
+			:no_break => ["0123456789"],
+			:backing_breaks => ["012\n\n", "\n\n", "\n\n",  "9"],
+			:cr_lf => ["012\r\n\r\n56\r\n\r\n9"]},
+			:sep_string => "\n\n")
+			
+		reindex_test({
+			:end_midline => ["012\n\n56", "\n\n9"],
+			:end_on_line => ["012\n\n56", "\n\n9\n"],
+			:end_on_break => ["012\n\n56", "\n\n9\n\n"],
+			:no_break => ["0123456", "789"],
+			:backing_breaks => ["012\n\n\n\n\n\n9"],
+			:cr_lf => ["012\r\n\r\n56", "\r\n\r\n9"]},
+			:sep_string => "56")
+	end
+			
+	#
+	# file tests
+	#
+
+	def index_test(path, &block)
+		aio = ArrayIO.open(aio_filepath(path), 'ru')
+		
+		begin
+			yield(aio)
+		ensure
+			aio.close
+			index_path = aio.io_index.path
+			File.delete(index_path) if index_path && File.exists?(index_path)
+		end
+	end
+	
+	def test_cr_lf_file
+		index_test('cr_lf_input.txt') do |aio|
+			aio.reindex
+					
+			assert_equal "012\r\n", aio[0]
+			assert_equal "56\r\n", aio[1]
+			assert_equal "9", aio[2]
+		end
+	end
+	
+	def test_parse_from_lf_file
+		index_test('lf_input.txt') do |aio|
+			aio.reindex
+					
+			assert_equal "012\n", aio[0]
+			assert_equal "56\n", aio[1]
+			assert_equal "9", aio[2]
+		end
+	end
+	
+	def test_parse_from_alt_sep
+		index_test('alt_sep.txt') do |aio|
+			aio.reindex do |line|
+				line =~ /^>/
+			end
+		
+			assert_equal ">abc\r\n", aio[0]
+			assert_equal ">def\r\n", aio[1]
+			assert_equal ">gh", aio[2]
+		end
+	end
+	
+	def test_reindex_speed
+		benchmark_test do
+			n = 10000
+			filepath = aio_filepath("reindex_speed.txt")
+			
+			bm(12) do |x|
+				x.report("building file") do
+					File.open(filepath, 'w') do |file|
+						0.upto(n-1) {|i| file << "line #{i}\n" }
+					end
+				end
+				puts "   #{n} lines"
+				
+				begin
+					aio = ArrayIO.new(filepath, 'r')	
+					uaio = ArrayIO.new(filepath, 'ru')	
+					
+					x.report("reindex") { aio.reindex }
+					assert_equal n, aio.length, 'aio'
+					assert_equal "line 1000\r\n", aio[1000], 'aio'
+					
+					x.report("ureindex") { uaio.reindex}
+					assert_equal n, uaio.length, 'uaio'
+					assert_equal "line 1000\r\n", uaio[1000], 'uaio'
+				ensure
+					aio.close
+					uaio.close
+					index_path = uaio.io_index.path
+					File.delete(index_path) if index_path && File.exists?(index_path)
+					File.delete(filepath)
+				end
+			end
+		end
+	end
+		
+	#
+	# array behavior tests
+	#
+	
+	def array_test_setup
+		[setup_aio, ["abc", "de", "fgh"]]
+	end
+	
+	def test_length_and_size
+		aio, array = array_test_setup
+		
+		assert_equal array.length, aio.length
+		assert_equal array.size, aio.size
+	end
+	
+	def test_fetch
+		aio, array = array_test_setup
+		
+		assert_equal array.fetch(0), aio.fetch(0)
+		assert_equal array.fetch(0, "default"), aio.fetch(0, "default")
+		assert_equal array.fetch(3, "default"), aio.fetch(3, "default")
+		
+		array.fetch(1) do |arr|
+			aio.fetch(1) do |a|
+				assert_equal arr, a
+			end
+		end
+	end
+	
+	def test_first
+		aio, array = array_test_setup
+		
+		assert_equal array.first, aio.first
+		assert_equal array.first(2), aio.first(2)
+		assert_equal array.first(10), aio.first(10)
+	end
+	
+	def test_each_index
+		aio, array = array_test_setup
+
+		aio_result = []
+		aio.each_index {|i| aio_result << i}
+		array_result = []
+		array.each_index {|i| array_result << i}
+		
+		assert_equal array_result, aio_result
+	end
+	
+	def test_each
+		aio, array = array_test_setup
+
+		aio_result = []
+		aio.each {|i| aio_result << i}
+		array_result = []
+		array.each {|i| array_result << i}
+		
+		assert_equal array_result, aio_result
+	end
+	
+	def test_empty
+		aio, array = array_test_setup
+		
+		assert_equal array.empty?, aio.empty?
+		assert_equal [].empty?, ArrayIO.new('', 'rs', []).empty?
+	end
+	
+	def test_last
+		aio, array = array_test_setup
+		
+		assert_equal array.last, aio.last
+		assert_equal array.last(2), aio.last(2)
+		assert_equal array.last(10), aio.last(10)
+	end
+	
+	def btest_values_at
+		aio, array = array_test_setup
+		
+		assert_equal array.values_at, aio.values_at
+		assert_equal array.values_at(2), aio.values_at(2)
+		assert_equal array.values_at(1, 10, 3..3, 1..2), aio.values_at(1, 10, 3..3, 1..2)
+	end
+end

data/test/arrayio_test_helper.rb

@@ -0,0 +1,72 @@
+require 'test/unit'
+require 'array_io'
+require 'benchmark'
+require 'pp'
+
+# The testing subset code here is taken from an as-yet unreleased gem 'prosperity'
+#
+# These subsets facilitate testing by using the ENV variables specified on the command line
+# to indicate which tests to run.  The ENV variables are set by rake, so this code implicitly 
+# assumes that you're running your tests through rake.
+# 
+class Test::Unit::TestCase
+	def run_subset?(type)
+		ENV[type] == "true" || ENV["ALL"] == "true" 
+	end
+
+	def match_regexp?(type, obj, default=true)
+		return default if ENV["ALL"] == "true"
+		return default unless ENV[type]
+		
+		str = ""
+		PP.singleline_pp(obj, str)
+		str =~ Regexp.new(ENV[type])
+	end
+	
+  	def extended_test(&block) 
+		subset_test("EXTENDED", "x", &block)
+	end
+	
+	def benchmark_test(&block) 
+		subset_test("BENCHMARK", "b") do
+			puts calling_method
+			block.call
+		end
+	end
+
+	def case_test(hash, &block)
+		if match_regexp?("CASE_TEST", calling_method)
+			hash.each_pair do |testcase, expected|
+				yield(testcase, expected) if match_regexp?("CASE", testcase)
+		     end
+		end
+	end
+
+	protected
+		
+	# Calling method iterates over the call stack, and returns the first calling
+	# method name that matches the input pattern (by default /^test/)
+	def calling_method(pattern=/^test/)
+		0.upto(caller.length) do |i|
+			caller[i] =~ /:in `(.*)'$/
+			method_name = $1
+			return method_name if method_name =~ pattern
+		end
+			
+		''
+	end
+		
+	def subset_test(type, skip, &block)
+		type = type.upcase
+		type_test = "#{type}_TEST"
+		if run_subset?(type) || ENV[type_test]
+			if match_regexp?(type_test,  calling_method)
+				block.call 
+			else
+				print skip
+			end
+		else
+			print skip
+		end
+	end
+end

data/test/arrayio_test_suite.rb

@@ -0,0 +1,4 @@
+$:.unshift File.join(File.dirname(__FILE__), '../lib')
+
+ENV["ALL"] = 'true'
+Dir.glob("./**/*_test.rb").each {|test| require test}

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.0
+specification_version: 1
+name: arrayio
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2007-03-15 00:00:00 -06:00
+summary: Array-like behavior for archival files.
+require_paths: 
+- lib
+email: simon.chiang@uchsc.edu
+homepage: http://rubyforge.org/projects/arrayio/
+rubyforge_project: 
+description: 
+autorequire: arrayio
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Simon Chiang
+files: 
+- test/arrayio_test_helper.rb
+- test/arrayio_test_suite.rb
+- test/array_io
+- test/array_io_test.rb
+- test/array_io/alt_sep.txt
+- test/array_io/cr_lf_input.txt
+- test/array_io/input.index
+- test/array_io/input.txt
+- test/array_io/inputb.index
+- test/array_io/inputb.txt
+- test/array_io/lf_input.txt
+- test/array_io/lines.txt
+- test/array_io/without_index.txt
+- lib/array_io.rb
+- lib/inspect_array_io.rb
+- README
+test_files: 
+- test/arrayio_test_suite.rb
+rdoc_options: []
+
+extra_rdoc_files: 
+- README
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: []
+