line_iterator 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NTkwMWEzNDM5ZmM1YmM5ZmU4MWYwNTYxYzc4N2MxNzA1Yjk2NThkNg==
+  data.tar.gz: !binary |-
+    YjM2MDdjNGRhYTk0MDE3OTMxMWFmMjhhN2E4MjdhYTczODcyMWJmYw==
+SHA512:
+  metadata.gz: !binary |-
+    MjIxNmE3NzMyMDg3NTliN2IwOGQ1MmVlMjU4NmM0ZDgzNGRmOTgwZTZhYWM1
+    MzViNGQwZTdkYjdlZTQ2ZGJmNjFkN2IxOGIzMDEyNDI5YTllNWEwMmYwZGZk
+    MmMyMjA2Zjk3ZmNlYTQ5ZGJlZDE2YjkxZmI3ODlkOGI1NDA0MTA=
+  data.tar.gz: !binary |-
+    ZmU0OGNmZWMxY2QxNzY1ZmM5MDczZTVjODRiNWE0MjQ0NzJlMzBlMWRhYjdj
+    Y2NmMTcyZDY2MzZlOTdjMjMxNGM0YzlmOTk4NzNjYWYwNmVmOTczODc2ZTUz
+    NTQ5YzJhNGVkMmE1ODIxNDUxNWM4NjE3ZmJiYmM3YmUxYWE0Zjk=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.1
+  - jruby-head

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in line_iterator.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Bill Dueber
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,247 @@
+# LineIterator
+
+A simple iterator designed to deal easiy with line-oriented text files.
+
+## Features
+
+* Automatically deal with files that end in .gz (via zlib)
+* Methods like #next and #each (aliased to #next_line and #each_line) return the line data with the line endings already `chomp`ed off
+* Skip forward or backwards by lines (limit on skipping backwards)
+* Track line numbers (staring with 1) no matter how you use each_line, next_line, skip (forward and backwards), etc.
+* Allow line-oriented records (where a record is a set of lines). The default implementation detends end-of-record as a blank line, but subclassing is easily for other types of line-oriented records.
+
+
+## Installation
+
+    $ gem install line_iterator
+
+## Basic Usage
+
+First, two quick warnings:
+
+* The record iteration stuff works fine, but I may change the interface to return a smarter object than just an array (so I can pass along, e.g. the starting and ending line numbers in the file for that record).
+* This works fine under MRI, fails hard under Jruby 1.7.9 (Null Pointer exception), and runs fine on jruby 1.7.10 in both 1.9 and 2.0 mode.
+
+
+### Getting a new iterator
+
+~~~ruby
+
+  require 'line_iterator'
+
+  # opening a file
+  iter = LineIterator.open('myfile.txt')
+  iter = LineIterator.open(already_open_file_or_io_object)
+  iter = LineIterator.open('myfile.txt.gz') # automatically use Zlib::GzipReader
+  iter = LineIterator.open($stdin, :gzip=>true) # manually set gzip status
+
+~~~
+
+### Getting line data and line numbers
+
+It's an enumerator, so you can use #next, #each, etc.
+
+For clarity you can use `#each_line` and `#next_line` as aliases of 
+`#each` and `#next`, respectively.
+
+Access the line number of the last returned (or just consumed via `#skip`) line via `#last_line_number` (where the first line is line 1, just like in an editor or when using head/tail/whatever)
+  
+~~~ruby  
+  iter.each do |line|
+    puts "Line number #{iter.last_line_number} : #{line}"
+  end
+  
+  # Line numbers work regardless of how you move through the iterator
+  
+  iter = LineIterator.open('myfile.txt')
+  iter.next
+  iter.next
+  iter.last_line_number #=> 2 
+  iter.each do |line|
+    puts iter.last_line_number
+    break
+  end #=> 3, since the two calls to #next advanced the line number
+
+
+  # There's a special iterator, `#each_with_line_number`, to mirror 
+  # `#each_with_index` but keep proper track of line numbers
+  
+  iter.each_with_line_number do |line, line_num|
+    #...
+  end
+
+  # You can use things like #map, but of course that'll read in the whole 
+  # input.
+  
+  iter.map{|line| [iter.last_line_number, line.size]}
+
+~~~  
+
+### Skipping forward and backward
+
+A `LineIterator` can skip forward or (in some cases) backwards.
+
+Calling `#skip` just skips the next line. Calling `#skip(num)` will skip forward `num` lines, or move to the end of the file if you run out of data.
+
+**Unlike `#next`, `#skip` will never throw `StopIteration`**. 
+
+If you call `#skip` with a negative number, the LineIterator will attempt to back up via an internal buffer (set at 100 lines). If you try to back up further than the available data allows, you'll get an `IndexError`.  
+
+~~~ruby
+
+  iter = LineIterator.open('myfile.txt')
+  iter.skip 5
+  iter.last_line_number #=> 5
+  iter.skip(-3)
+  iter.last_line_number #=> 2
+  iter.next #=> <the third line of the file>
+  iter.last_line_number #=> 3
+
+  iter = LineIterator.open('myfile.txt')
+  iter.skip(1_000_000) #=> doesn't raise an error no matter how far you skip
+  iter.next #=> StopIteration error
+  
+  iter = LineIterator.open('myfile.txt')
+  iter.skip(10)
+  iter.skip(-100) #=> IndexError
+  
+~~~
+
+
+## Dealing with records
+
+`LineIterator` has a simple line-oriented record interface. By default, it separates files into records on blank lines (lines with nothing but optional whitespace in them) and returns a "record" that simply consists of an array containing the appropriate lines from the file.
+
+Like the line-based commands, the contents of the returned array are already `#chomp`ed.
+
+Note two things:
+
+* There's no `#skip` backwards implemented for records; you can use `#skip_record` or `#skip_records(n)` to skip records forward.
+* If you mix `#next_record / #each_record` with `#next_line` / `#each_line` / `#skip`, things are usually going to get *really* screwey. Mixed use is not really supported.
+
+### Using blank-line delimited records
+
+
+Given the file:
+
+~~~
+One Hat
+Two Hat
+
+Red Hat
+Blue Hat
+
+by Dr. Seuss
+~~~
+
+We can use the record interface as follows:
+
+~~~ruby
+
+iter = LineIterator.new('onehat.txt')
+x = iter.next_record #=> ['One Hat', 'Two Hat']
+y = iter.next_record #=> ['Red Hat', 'Blue Hat']
+
+iter.last_record_number #=> 2
+
+iter.each_record do |rec|
+  puts rec
+end #=> Show the one remaining record, ['by Dr. Seuss']
+
+~~~
+
+### Changing the end-of-record pattern
+
+Maybe you have records that are separated by a line with nothing on it but dashes? You can set the pattern used to detect the end of a record by setting `#end_of_record_pattern`
+
+Given the file:
+
+~~~
+Bill Dueber
+1234 Sample st.
+Ann Arbor, MI 4813
+-----
+Mike Dueber
+1350 N. Nowhere
+St. Paul, MN 55117
+~~~
+
+...you could get the two records out of it as follows:
+
+~~~ruby
+iter = LineIterator.new('addresses.txt')
+iter.end_of_record_pattern = /\A--+\s*\Z/
+iter.each_record do |rec|
+  # do something with the arrays of lines returned
+end
+~~~
+
+
+### Sublcassing `LineIterator` for different kinds of records
+
+You can subclass `LineIterator` and override the method `#end_of_record(buff)` to return true when there's an end of record. Usually this invovles calling `line, line_number = peek` to see what's coming up next.
+
+The buffer passed in is the contents of the record so far.
+
+Here's a simple implementation of a subclass that deals with records that are identified by contiguous lines that all have the same prefix string.
+
+Given the file:
+
+~~~
+001 Red
+001 White
+001 Blue
+002 One
+002 Two
+003 Alpha
+003 Beta
+003 Gamma
+003 Delta
+~~~
+
+...we have three records, where the end of record is identified by the numeric prefix changing from one line to the next (or, for the last record, by the end of the input).
+
+We can easily subclass `LineIterator` to take care of this case.
+
+~~~ruby
+
+  # Subclass to override end_of_record(buff)
+  # We introduce a new instance variable to track the most recent prefix
+  class PrefixBasedRecordIterator < LineIterator
+    PREFIXP = /^(\d+)\s+/
+    def prefix(line)
+      (PREFIXP.match(line))[1]
+    end
+  
+    def end_of_record(buff)
+      return true if self.done
+      line, line_no = peek
+      p = prefix(line)
+      if p != @previous_prefix
+        @previous_prefix = p
+        return  !(buff.empty?)
+      else
+        return false
+      end
+    end
+  end
+
+  iter = PrefixBasedRecordIterator.new('prefix_file.txt')
+  rec = iter.next_record
+    #=> ['001 Red', '001 White', '001 Blue']
+  rec = iter.next_record
+    #=> ['002 One', '002 Two']
+
+~~~
+
+
+
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+end
+
+task :default => :test

data/lib/line_iterator.rb

@@ -0,0 +1,195 @@
+require "line_iterator/version"
+require 'zlib'
+
+class LineIterator
+  include Enumerable
+  
+  BUFFER_SIZE = 100
+
+  attr_reader :last_line_number, :done, :last_record_number
+  attr_accessor :end_of_record_pattern
+  
+  def to_s
+    "#{self.class} <#{@f}, last_line_number: #{last_line_number}>"
+  end
+  
+  alias_method :inspect, :to_s
+  
+  
+  # Open up the input. If it's a string, assume a filename and open it up.
+  # Also, run it through GzipReader if the filename ends in .gz or if
+  # :gzip => true was passed in the opts
+  def initialize(input, opts={})
+    if input.is_a? IO
+      @f = input
+      # deal with IO object
+    else # assume string
+      @f = File.open(input)
+      opts[:gzip] ||= (input =~ /\.gz\Z/)
+    end
+    
+    if opts[:gzip]
+      @f =  Zlib::GzipReader.new(@f)
+    end
+    @base_iterator = @f.each_with_index
+    @last_line_number = 0
+    @last_record_number = 0
+    @done = false
+    @buffer = []
+    @backup_buffer = []
+    @end_of_record_pattern = /\A\s*\n/
+  end
+  
+  
+  
+  # Override the normal enumerable #next to keep internal track
+  # of line numbers
+  def next
+    # Get the next line from the backup buffer or the stream
+    y = @backup_buffer.empty? ? @base_iterator.next : @backup_buffer.shift
+    
+    # Feed the buffer
+    @buffer.shift if @buffer.size ==  BUFFER_SIZE
+    @buffer.push y
+    
+    @last_line_number = y[1] + 1
+    return y[0].chomp
+  end
+  
+  alias_method :next_line, :next
+  
+  # Provide a #peek that takes into account the backup buffer
+  
+  def peek
+    @backup_buffer.empty? ? @base_iterator.peek : @backup_buffer[0]
+  end
+  
+  
+  # Skip n lines (default: 1). Just calls next over and over again,
+  # but will *never* throw StopIteration error
+  def skip(n = 1)
+    if n == 0
+      return;
+    elsif n > 0
+      skip_forward(n)
+    elsif n < 0
+      skip_backwards(-n)
+    else
+      raise "Tried to skip backwards more than size of buffer (#{BUFFER_SIZE})"
+    end
+  end
+
+  def skip_forward(n)
+    begin
+      n.times do
+        self.next
+      end
+    rescue StopIteration
+      @done = true
+    end
+  end
+  
+  def skip_backwards(n)    
+    # can we back up?
+    raise IndexError.new, "Tried to skip backwards too far", nil if n > @buffer.size
+    n.times { @backup_buffer.unshift @buffer.pop }
+    @last_line_number = @backup_buffer[0][1]
+  end
+  
+  # Override normal #each to track last_line_nunber
+  def each
+    unless block_given?
+      return enum_for :each
+    end
+    begin
+      while true
+        yield self.next
+      end
+    rescue StopIteration
+      @done = true
+    end
+  end
+  
+  alias_method :each_line, :each
+  
+  # Like #each_with_index, but track line numbers
+  # This allows you to call next/skip and still get the correct
+  # line number out
+  def each_with_line_number
+    unless block_given? 
+      return enum_for :each_with_line_number
+    end
+    
+    begin
+      while true
+        yield [self.next, self.last_line_number]
+      end
+    rescue StopIteration
+      @done = true
+    end
+  end
+  
+  # Detect the end_of_record for a line-based file, and 
+  # do whatever you need to do 
+
+  # This default implementation just checks for blank lines 
+  # and eats them, but you can override this in a subclass 
+  # (perhaps using the contents of the buffer to determine
+  # EOR status)
+  
+  def end_of_record(buff)
+    y = peek
+    if  end_of_record_pattern.match(y[0])
+      self.next # eat the next line
+      return true
+    else 
+      return false
+    end
+  end
+    
+  # Get the next record and return (as an array of chomp'ed lines)
+  def next_record
+    raise StopIteration if self.done
+    buff = []
+    begin
+      while true do
+        if end_of_record(buff) and not buff.empty?
+          @last_record_number += 1
+          return buff
+        else
+          buff << self.next
+        end
+      end
+    rescue StopIteration
+      @last_record_number += 1
+      @done = true
+      return buff
+    end
+  end
+  
+  # iterate over records
+  def each_record
+    unless block_given?
+      return enum_for(:each_record)
+    end
+    
+    begin
+      while !self.done
+        yield self.next_record
+      end
+    end
+  end
+  
+  # Skip over records (forward only!!!!)
+  def skip_record(n = 1)
+    begin
+      n.times {self.next_record}
+    rescue StopIteration
+      @done = true
+    end
+  end
+  
+  alias_method :skip_records, :skip_record
+  
+  
+end

data/lib/line_iterator/version.rb

@@ -0,0 +1,4 @@
+require 'delegate'
+class LineIterator < SimpleDelegator
+  VERSION = "0.1.0"
+end

data/line_iterator.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'line_iterator/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "line_iterator"
+  spec.version       = LineIterator::VERSION
+  spec.authors       = ["Bill Dueber"]
+  spec.email         = ["bill@dueber.com"]
+  spec.description   = %q{Provides methods to more easily work with line-oriented text file and records within those files}
+  spec.summary       = %q{Easily work with line-oriented text files}
+  spec.homepage      = "http://github.com/billdueber/line_iterator"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+end

data/test/minitest_helper.rb

@@ -0,0 +1,11 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'line_iterator'
+
+require 'minitest'
+require 'minitest/spec'
+require 'minitest/autorun'
+
+
+def test_data(relative_path)
+  return File.expand_path(File.join("test_data", relative_path), File.dirname(__FILE__))
+end

data/test/test_data/addresses.txt

@@ -0,0 +1,7 @@
+Bill Dueber
+1234 Sample st.
+Ann Arbor, MI 4813
+---
+Mike Dueber
+1350 N. Nowhere
+St. Paul, MN 55117

data/test/test_data/numbers.txt

@@ -0,0 +1,12 @@
+One
+Two
+Three
+Four
+Five
+Six
+Seven
+Eight
+Nine
+Ten
+Eleven
+Twelve

data/test/test_data/poetry.txt

@@ -0,0 +1,37 @@
+Out of burlap sacks, out of bearing butter,
+Out of black bean and wet slate bread,
+Out of the acids of rage, the candor of tar,
+Out of creosote, gasoline, drive shafts, wooden dollies,
+They Lion grow.
+
+Out of the gray hills
+Of industrial barns, out of rain, out of bus ride,
+West Virginia to Kiss My Ass, out of buried aunties,
+Mothers hardening like pounded stumps, out of stumps,
+Out of the bones' need to sharpen and the muscles' to stretch,
+They Lion grow.
+
+Earth is eating trees, fence posts,
+Gutted cars, earth is calling in her little ones,
+"Come home, Come home!" From pig balls,
+From the ferocity of pig driven to holiness,
+From the furred ear and the full jowl come
+The repose of the hung belly, from the purpose
+They Lion grow.
+
+From the sweet glues of the trotters
+Come the sweet kinks of the fist, from the full flower
+Of the hams the thorax of caves,
+From "Bow Down" come "Rise Up,"
+Come they Lion from the reeds of shovels,
+The grained arm that pulls the hands,
+They Lion grow.
+
+From my five arms and all my hands,
+From all my white sins forgiven, they feed,
+From my car passing under the stars,
+They Lion, from my children inherit,
+From the oak turned to a wall, they Lion,
+From they sack and they belly opened
+And all that was hidden burning on the oil-stained earth
+They feed they Lion and he comes.

data/test/test_data/prefix_based_record.txt

@@ -0,0 +1,10 @@
+001 One
+001 Two
+001 Three
+005 Five
+005 Six
+005 Seven
+005 Eight
+005 Nine
+010 Ten
+010 Eleven

data/test/test_line_iterator.rb

@@ -0,0 +1,269 @@
+require 'minitest_helper'
+
+describe "creating a new iterator" do
+  it "opens a real file" do
+    i = LineIterator.new(test_data('numbers.txt'))
+    assert_equal "One", i.next
+  end
+  
+  it "opens a gzipped file" do
+    i = LineIterator.new(test_data('numbers.txt.gz'))
+    assert_equal "One", i.next
+  end
+  
+  it "deals with a file object" do
+    f = File.open(test_data('numbers.txt'))
+    i = LineIterator.new(f)
+    assert_equal 'One', i.next
+  end
+  
+  it "deals with a gzipped file object" do
+    f = File.open(test_data('numbers.txt.gz'))
+    i = LineIterator.new(f, :gzip=>true)
+    assert_equal 'One', i.next
+  end    
+  
+  it "doesn't error on a zero-length file" do
+    f = File.open(test_data('zero_length_file.txt'))
+    f.each do |l|
+      assert(false) # should never be called
+    end
+  end
+    
+end
+
+describe "skip forward" do
+  before do
+    @i = LineIterator.new(test_data('numbers.txt'))
+  end
+    
+  it "skips forward one" do
+    @i.skip
+    assert_equal 'Two', @i.next
+  end
+  
+  it "skips forward N" do
+    @i.skip(5)
+    assert_equal 5, @i.last_line_number
+    assert_equal 'Six', @i.next
+  end
+  
+  it "skips to the end and will then raise stopiteration on next call" do
+    @i.skip(100)
+    assert_equal 12, @i.last_line_number
+    assert_raises(StopIteration) { @i.next }
+  end
+  
+end
+
+describe "skip backwards" do
+  before do
+    @i = LineIterator.new(test_data('numbers.txt'))
+  end
+  
+  it "does basic backwards skip" do
+    @i.skip(5)
+    @i.skip(-1)
+    assert_equal "Five", @i.next
+  end
+  
+  it "errors out if you back up past the beginning of the file" do
+    @i.skip 2
+    assert_raises(IndexError) {@i.skip(-3)}
+  end
+  
+  it "can exactly rewind" do
+    @i.skip 10
+    @i.skip -10
+    assert_equal "One", @i.next
+    assert_equal "Two", @i.next
+  end
+  
+  it "errors out if you back up past the buffer" do
+    @i.skip 10
+    @i.skip(-5)
+    assert_raises(IndexError) {@i.skip(-6)}
+  end  
+end
+
+describe "throwing Errors" do
+  it "throws StopIteration when #next is called too many times" do
+    @i = LineIterator.new(test_data('numbers.txt'))
+    assert_raises(StopIteration) { 20.times{@i.next}}
+  end
+end
+
+
+
+describe "maintain line number" do
+  before do
+    @i = LineIterator.new(test_data('numbers.txt'))
+  end
+  
+  it "starts with a zero last_line_number" do
+    assert_equal 0, @i.last_line_number   
+  end
+  
+  it "advances when using next" do
+    @i.next
+    @i.next
+    assert_equal 2, @i.last_line_number
+  end
+  
+  it "advances when using each" do
+    cnt = 0
+    @i.each do |y|
+      cnt += 1
+      break if cnt == 3
+    end
+    assert_equal 3, @i.last_line_number
+  end
+  
+  it "advances when using skip" do
+    @i.skip(3)
+    assert_equal 3, @i.last_line_number
+  end
+    
+  it "stays correct on mixed usage" do
+    @i.next
+    @i.next
+    assert_equal 2, @i.last_line_number
+    @i.each do |line|
+      assert_equal 3, @i.last_line_number
+      break
+    end 
+  end
+  
+  it "correctly resets when skipping backwards" do
+    @i.skip(3)
+    assert_equal 3, @i.last_line_number
+    @i.skip(-2)
+    assert_equal 1, @i.last_line_number
+    line = @i.next
+    assert_equal 'Two', line
+    assert_equal 2, @i.last_line_number
+  end
+    
+    
+end
+
+
+describe "working with pattern-delimited records" do
+  before do
+    @i = LineIterator.new(test_data('poetry.txt'))
+  end
+  
+  it "works with each_record()" do
+    cnt = 0
+    @i.each_record do |r|
+      cnt += 1
+    end
+    
+    assert_equal 5, cnt
+    assert_equal 5, @i.last_record_number
+  end
+  
+  it "gets the first record" do
+    r = @i.next_record
+    assert_equal 5, r.size
+  end
+    
+  
+  it "correctly deals with next_record" do
+    r = @i.next_record
+    r = @i.next_record
+    assert_equal "Out of the bones' need to sharpen and the muscles' to stretch,", r[4]
+  end
+  
+  it "throws StopIterator when it should stop" do
+    begin
+      while true
+        @i.next_record
+      end
+    rescue StopIteration
+      assert_equal 5, @i.last_record_number
+    end
+  end
+  
+  it "breaks out of each_record correctly" do
+    rec = nil
+    @i.each_record do |r|
+      rec = r
+      break if @i.last_record_number == 2
+    end
+    assert_equal "Out of the bones' need to sharpen and the muscles' to stretch,", rec[4]
+  end
+
+  it "works with a backup buffer" do
+    r1 = @i.next_record
+    @i.skip(-(@i.last_line_number))
+    r2 = @i.next_record
+    assert_equal r1, r2
+  end
+
+  it "can use a custom pattern" do
+    iter = LineIterator.new(test_data('addresses.txt'))
+    iter.end_of_record_pattern = /\A--+\s*\Z/
+    recs = []
+    iter.each_record do |rec|
+      recs << rec
+    end
+    assert_equal 2, recs.size
+    assert_equal 'Bill Dueber', recs[0][0]
+    assert_equal 'Mike Dueber', recs[1][0]
+  end
+    
+
+end
+
+
+# Subclass to override end_of_record(buff)
+# We introduce a new instance variable to track the most recent prefix
+class PrefixBasedRecordIterator < LineIterator
+  PREFIXP = /^(\d+)\s+/
+  def prefix(line)
+    (PREFIXP.match(line))[1]
+  end
+  
+  def end_of_record(buff)
+    return true if self.done
+    line, line_no = peek
+    p = prefix(line)
+    if p != @previous_prefix
+      @previous_prefix = p
+      return !buff.empty?
+    else
+      return false
+    end
+  end
+end
+
+describe "subclass to do prefix-based records" do
+  before do
+    @i = PrefixBasedRecordIterator.new(test_data('prefix_based_record.txt'))
+  end
+  
+  it "finds all the records with each_record" do
+    cnt = 0
+    @i.each_record do |rec|
+      cnt += 1
+    end
+    assert_equal 3, cnt
+  end
+  
+  it "gets the prefixed records with next_record" do
+    rec = @i.next_record
+    assert_equal 3, rec.size
+
+    rec = @i.next_record
+    assert_equal 5, rec.size
+
+    rec = @i.next_record
+    assert_equal 2, rec.size
+
+  end
+end
+  
+
+    
+  

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: line_iterator
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Bill Dueber
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Provides methods to more easily work with line-oriented text file and
+  records within those files
+email:
+- bill@dueber.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/line_iterator.rb
+- lib/line_iterator/version.rb
+- line_iterator.gemspec
+- test/minitest_helper.rb
+- test/test_data/addresses.txt
+- test/test_data/numbers.txt
+- test/test_data/numbers.txt.gz
+- test/test_data/poetry.txt
+- test/test_data/prefix_based_record.txt
+- test/test_data/zero_length_file.txt
+- test/test_line_iterator.rb
+homepage: http://github.com/billdueber/line_iterator
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.1
+signing_key: 
+specification_version: 4
+summary: Easily work with line-oriented text files
+test_files:
+- test/minitest_helper.rb
+- test/test_data/addresses.txt
+- test/test_data/numbers.txt
+- test/test_data/numbers.txt.gz
+- test/test_data/poetry.txt
+- test/test_data/prefix_based_record.txt
+- test/test_data/zero_length_file.txt
+- test/test_line_iterator.rb
+has_rdoc: