hflr2 1.0.0

data/History.txt

@@ -0,0 +1,14 @@
+== 1.0.1 / 2010-01-21
+
+* Fixed warnings 
+* Added better exception handling for badly formatted output data
+
+
+== 0.11.0 / 2009-08-04
+
+*  Removed useless files 
+  * Corrected version number 
+
+== 1.0.1 / 2011-02-09
+   * Added ability to specify file format in Ruby with Range class (see example.rb in /test.)
+   

data/README.txt

@@ -0,0 +1,45 @@
+= HFLR
+
+* http://ruff.rubyforge.org
+
+== Description:
+
+HFLR -- Hierarchical Fixed Length Records
+
+Allows you to read and write files of fixed width records when the file contains one or more
+than one type of record.  
+
+Install with 'gem install hflr'
+
+
+
+
+See the tests and examples bundled with this gem.
+
+
+
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009  Colin C. Davis 
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/hflr2.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{hflr2}
+  s.version = "1.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Colin Davis", "Bozydar Sobczak"]
+  s.date = %q{2011-10-24}
+  s.description = %q{HFLR2 -- Hierarchical Fixed Length Records
+
+NOTE: This gem is a modification of the hflr gem. It can be not compatible with it.
+
+Allows you to read and write files of fixed width records when the file contains one or more
+than one type of record.  
+
+Install with 'gem install hflr2'
+
+See the tests and examples bundled with this gem.}
+  s.email = %q{colin.c.davis@gmail.com}
+  s.extra_rdoc_files = %w(History.txt README.txt)
+  s.files = %w(History.txt README.txt hflr2.gemspec lib/hflr.rb lib/hflr/fl_record_file.rb lib/hflr/hflr.rb lib/hflr/record_template.rb test/customer_orders.dat test/customers.dat test/examples.rb test/flrfile_test.rb test/record_template_test.rb test/sample.dat test/sample2_out.dat test/sample_activities.dat test/sample_out.dat test/test_helper.rb test/test_hflr.rb)
+  s.require_paths = %w(lib)
+
+  s.rubygems_version = %q{1.3.4}
+  s.summary = %q{HFLR2 -- Hierarchical Fixed Length Records 2 Allows you to read and write files of fixed width records when the file contains one or more than one type of record}
+  s.test_files = %w(test/test_hflr.rb test/test_helper.rb)
+end

data/lib/hflr/fl_record_file.rb

@@ -0,0 +1,262 @@
+class FLRFile
+
+  include Enumerable
+
+  attr_reader :line_number, :record_template
+
+  def initialize(source, record_types, record_layouts, logical_first_column=0, extra_columns = nil)
+    # Allow record layouts like 
+    # {:type1=>[:var1=>1..5,:var2=>7..8],:type2=>[:var1=>1..1,:var2=>3..4]}
+    if record_layouts.values.first.is_a? Hash
+      record_layouts = create_layouts(record_layouts)
+    end
+    @line_number = 0
+    @file = source
+    @record_type_labels = record_types
+    @record_type_symbols = record_types.is_a?(Hash) ? record_types : :none
+    if extra_columns then
+      @record_template = HFLR::RecordTemplate.create(record_layouts, @record_type_symbols, logical_first_column, extra_columns)
+    else
+      @record_template = HFLR::RecordTemplate.create(record_layouts, @record_type_symbols, logical_first_column)
+    end
+  end
+
+  def set_fast
+    @fast = !@record_type_labels.is_a?(Hash)
+    unless @fast
+      raise "Cannot set fast mode with more than one record type."
+    end
+    if @fast
+      @width = get_record_width_from_file
+
+
+      records_to_take = 100000000 / @width
+
+      @buffer_size = @width * records_to_take
+
+
+      @position=0
+      @current_buffer=nil
+    end
+  end
+
+  def ranges=(ranges)
+    @fast or raise "Cannot read selected ranges because input file has multiple record types #{@record_type_labels.to_s}"
+    unless ranges.first.is_a?(Range)
+      raise "You specified a #{ranges.first.class.to_s} instead of a range in the list of ranges.  Use (a..b) to specify a range."
+    end
+
+    @offsets =offsets_to_read(ranges, @width)
+
+    @ranges = ranges
+  end
+
+
+  def in_range?(line_number)
+    @ranges ? !!(@ranges.detect { |r| r.member?(line_number) }) : true
+  end
+
+  def finished?
+    if @fast
+      @offsets.empty? && @current_buffer.nil?
+    else
+      @file.eof?
+    end
+  end
+
+  def close
+    @file.close
+  end
+
+# If multiple record types,  extract it from the string, otherwise just return the type of this file
+  def get_record_type(line)
+    return nil if line.nil?
+    return nil if line.strip.empty?
+    if @record_type_labels.is_a?(Hash)
+      matching_pair = @record_type_labels.find do |_, value|
+        discriminator = value[:discriminator]
+        position = value[:position] - 1
+        line[position..-1].start_with?(discriminator)
+      end
+      if matching_pair
+        matching_pair[0]
+      else
+        nil
+      end
+    else
+      @record_type_labels
+    end
+  end
+
+  def build_record(line)
+    return nil if line.nil?
+    record_type = line_type(line)
+    raise "Unknown record type at line #{@line_number.to_s}" if record_type == :unknown
+    @record_template[record_type].build_record(line.chomp)
+  end
+
+  def next_record
+    build_record(get_next_known_line_type)
+  end
+
+  def line_type(line)
+    record_type = get_record_type(line)
+    record_type ? record_type : :unknown
+  end
+
+  def get_next_known_line_type
+    @fast ? fast_get_next_known_line_type : sequential_get_next_known_line_type
+  end
+
+  def get_next_line
+    line = @file.gets
+    @line_number+=1
+    line
+  end
+
+  def next
+    build_record(get_next_line)
+  end
+
+  def fast_get_next_known_line_type
+    if @current_buffer.nil? && (@offsets.nil? || @offsets.empty?)
+      nil
+    else
+      if @current_buffer.nil?
+        chunk = @offsets.shift
+
+
+        @file.pos = chunk.pos
+        @current_buffer=@file.read(chunk.width)
+
+        record= @current_buffer.slice(@position, @width)
+
+
+        @position += @width
+
+        if @position >= @current_buffer.size
+
+          @current_buffer = nil
+          @position=0
+        end
+        return record
+      else
+        record= @current_buffer.slice(@position, @width)
+
+        @position += @width
+        if @position>=@current_buffer.size
+          @position=0
+          @current_buffer=nil
+        end
+        return record
+      end
+
+    end
+  end
+
+  def sequential_get_next_known_line_type
+    line = @file.gets
+    @line_number+=1
+    record_type = line_type(line)
+    while !finished? && (!in_range?(@line_number) || record_type == :unknown)
+      line = @file.gets
+      @line_number+=1
+      record_type = line_type(line)
+    end
+    record_type == :unknown ? nil : line
+  end
+
+
+  def each
+    @line_number = 1
+    if @fast
+      yield(next_record) until finished?
+    else
+      @file.each_line do |line|
+        unless line_type(line) == :unknown || !in_range?(@line_number)
+          data = build_record(line)
+          yield data
+        end
+      end
+    end
+  end
+
+  # This will take a Hash or Struct orArray;  if an Array the record type must be the last element when
+  # the record layout has more than one record type.
+  def <<(record)
+    record_type =
+        if record.is_a? Array
+          @record_type_symbols == :none ? @record_template.keys.first : record.last
+        else
+          if @record_template[record[:record_type]] == nil then
+            raise "Record type problem in output: #{record[:record_type].to_s} type on record, #{@record_template.keys.join(",")} types of templates"
+          end
+          @record_type_symbols == :none ? @record_template.keys.first : record[:record_type]
+        end
+    line = @record_template[record_type].build_line(record)
+    if @cr_before_line
+      line = "\n" + line
+    else
+      @cr_before_line = true
+    end
+    @file.write(line)
+  end
+
+# Use when creating a new HFLR file
+  def self.open(path, mode, record_types, record_layouts, logical_first_column=0)
+    file = File.open(path, mode)
+    begin
+      hflr_file = new(file, record_types, record_layouts, logical_first_column)
+      yield hflr_file
+    ensure
+      file.close
+    end
+  end
+
+  private
+
+  def offsets_to_read(ranges, width)
+    #ranges.map{|r| r.map{|o| o * width}}.flatten.uniq
+    chunk = Struct.new(:pos, :width)
+    chunks = []
+    ranges.each do |range|
+      offsets = range.map { |offset| offset }
+      taken = []
+
+      until offsets.empty?
+        taken << offsets.shift
+        if taken.size * width == @buffer_size || offsets.empty?
+
+          chunks << chunk.new(taken.first * @width, taken.size * width)
+          taken = []
+        end # if
+      end # while
+    end # each
+
+    chunks
+  end
+
+  def get_record_width_from_file
+    width = @file.gets.size
+    @file.rewind
+    width
+  end
+
+
+# If the layout is given in the convenient Ruby form
+  def create_layouts(layout)
+    var_class = Struct.new(:name, :start, :len)
+    new_layout = {}
+    layout.each_pair do |record_type, vars|
+
+      new_layout[record_type] = []
+      vars.each_pair do |var_name, range|
+        new_layout[record_type] << var_class.new(var_name.to_s, range.first, range.last - range.first + 1)
+      end
+      new_layout[record_type].sort! { |a, b| a.start<=>b.start }
+    end
+    new_layout
+  end
+
+end
+

data/lib/hflr/hflr.rb

@@ -0,0 +1 @@
+require File.expand_path(File.dirname(__FILE__) + "/fl_record_file")

data/lib/hflr/record_template.rb

@@ -0,0 +1,155 @@
+module HFLR
+
+  class RecordTemplate
+    UNFILLED_CHAR = ' '
+    MISSING_OUTPUT = "ZZZZZZZZZZZZZZZZZZZZZ"
+
+    attr_reader :record_structure, :field_pattern, :record_type, :record_type_label
+    attr_accessor :strip_whitespace
+
+    def initialize(record_type, record_type_label, record_structure, field_pattern, field_widths)
+      @record_type = record_type
+      @record_type_label = record_type_label
+      @record_structure = record_structure
+      @field_pattern = field_pattern
+      @field_widths = field_widths
+    end
+
+# Layouts is a hash of variables by record type
+# record_type_symbols maps record type names to their labels in the data {:household=>"H",:person=>"P"}
+# Returns a set of record templates, one for each record type  
+    def self.create(record_layouts, record_type_symbols, first_column_location, extra_columns=[])
+      extra_columns = empty_extra_columns(record_layouts.keys) if  extra_columns.is_a? Array
+      templates = {}
+      self.check_record_layouts(record_layouts)
+
+      record_layouts.each_pair do |record_type, vars|
+        record_label = record_type_symbols == :none ? :none : record_type_symbols[record_type]
+        templates[record_type] =
+            self.create_template_class(record_type,
+                                       record_label,
+                                       vars,
+                                       first_column_location,
+                                       extra_columns[record_type])
+      end
+      return templates
+    end
+
+    private
+# If the name exists already do not replace it, but add extra columns not to be mapped by the unpack field patterns
+# and ensure the record_type variable is added.
+# Since 'record_type' may not be in the metadata we don't want to map it to a
+# specific column location but do want it included always.
+    def self.add_extra_columns(names, extra)
+      new_names = names.dup
+      # names are not case sensitive
+      extra.each { |n| new_names << n unless names.map { |m| m.to_s.upcase }.include? n.to_s.upcase }
+
+      # No matter what, include 'record_type'
+      unless new_names.map { |n| n.to_s.upcase }.include?("RECORD_TYPE")
+        new_names << :record_type
+      end
+      return new_names
+    end
+
+    def self.get_pattern(layout, first_column_location=0)
+      layout.map { |l| '@' + (l.start - first_column_location).to_s + 'a' + l.len.to_s }.to_s
+    end
+
+    public
+
+    def build_record(line)
+      rec = line.unpack(@field_pattern)
+      rec.map { |f| f.strip! } if @strip_whitespace
+      begin
+        data = self.record_structure.new(*rec)
+        data[:record_type] = @record_type
+      rescue Exception => msg
+        raise "On record type #{self.record_type} problem with structure " + msg.to_s
+      end
+      data
+    end
+
+    def build_line(record)
+      line = format_fields(record).pack(@field_pattern)
+      line.tr!("\0", UNFILLED_CHAR)
+      line
+    end
+
+    private
+
+    def self.empty_extra_columns(record_types)
+      extra = {}
+      record_types.map { |rt| extra[rt] = [] }
+      extra
+    end
+
+# All starting columns must be in order
+    def self.check_record_layouts(layouts)
+      layouts.values.each do |layout|
+        last_v = layout.first
+        layout.each do |v|
+          if v.respond_to?(:rectype) then
+            if last_v.rectype != v.rectype
+              raise "record type mismatch between #{v.name} and #{last_v.name}"
+            end
+          end
+          if last_v.start<= v.start then
+            last_v = v
+          else
+            raise "Problem with start columns #{last_v.name} start #{last_v.start.to_s}  out of sequence with #{v.name} starting at #{v.start.to_s}"
+          end
+        end
+      end
+    end
+
+    def self.create_template_class(record_type, record_type_label, layout, first_column_location, extra_columns = nil)
+      names = layout.map { |l| l.name.to_sym }
+      names = add_extra_columns(names, extra_columns)
+      structure = Struct.new(*names)
+      return new(record_type,
+                 record_type_label,
+                 structure,
+                 self.get_pattern(layout, first_column_location),
+                 layout.map { |v| v.len })
+    end
+
+
+    def format_fields(record)
+      if record.is_a?(Array) or record.is_a?(Struct)
+        fields = []
+        @field_widths.each_with_index do |width, i|
+          begin
+            fields << right_format(record[i], width)
+          rescue Exception => msg
+            if  record.is_a?(Struct)
+              raise "Output format problem for #{record.members[i].to_s} #{msg.to_s}"
+            else
+              raise "Output format problem for column #{i.to_s} with value #{record[i].to_s} #{msg.to_s}"
+            end
+          end
+        end
+        fields
+      else
+        raise "Record to format must be a Struct or Array"
+      end
+    end
+
+    def right_format(data, len)
+      if data.is_a? String
+        data_str = data.ljust(len)
+      elsif data.is_a? Symbol
+        data_str = data.to_s.ljust(len)
+      else
+        data_str = sprintf("%0#{len.to_s}d", data)
+        data_str = MISSING_OUTPUT[0..len-1] if data == -999998
+      end
+      raise "Data too large for allocated columns #{data_str}" if data_str.size > len
+      data_str
+    end
+
+
+  end # RecordTemplate class
+
+end # HFLR module
+

data/lib/hflr.rb

@@ -0,0 +1,49 @@
+
+module Hflr
+
+  # :stopdoc:
+  VERSION = '0.11.0'
+  LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
+  PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
+  # :startdoc:
+
+  # Returns the version string for the library.
+  #
+  def self.version
+    VERSION
+  end
+
+  # Returns the library path for the module. If any arguments are given,
+  # they will be joined to the end of the libray path using
+  # <tt>File.join</tt>.
+  #
+  def self.libpath( *args )
+    args.empty? ? LIBPATH : ::File.join(LIBPATH, args.flatten)
+  end
+
+  # Returns the lpath for the module. If any arguments are given,
+  # they will be joined to the end of the path using
+  # <tt>File.join</tt>.
+  #
+  def self.path( *args )
+    args.empty? ? PATH : ::File.join(PATH, args.flatten)
+  end
+
+  # Utility method used to require all files ending in .rb that lie in the
+  # directory below this file that has the same name as the filename passed
+  # in. Optionally, a specific _directory_ name can be passed in such that
+  # the _filename_ does not have to be equivalent to the directory.
+  #
+  def self.require_all_libs_relative_to( fname, dir = nil )
+    dir ||= ::File.basename(fname, '.*')
+    search_me = ::File.expand_path(
+        ::File.join(::File.dirname(fname), dir, '**', '*.rb'))
+
+    Dir.glob(search_me).sort.each {|rb| require rb}
+  end
+
+end  # module Hflr
+
+Hflr.require_all_libs_relative_to(__FILE__)
+
+# EOF

data/test/customer_orders.dat

@@ -0,0 +1,5 @@
+CJoe Smith                 55455025.53 
+O0005233110-10-2008
+CJane Smith                55404015.25
+O0054933310-11-2008
+O0075789110-12-2008

data/test/customers.dat

@@ -0,0 +1,3 @@
+Joe Smith                 55455025.53 
+Jane Smith                55404015.25 
+John Smith                55404015.25 

data/test/examples.rb

@@ -0,0 +1,67 @@
+require File.dirname(__FILE__) + "/../lib/hflr"
+
+
+# Read a file with only one record type (no record type markers)
+
+# metadata for customer file
+Column = Struct.new(:name, :start, :len)
+columns = {:customer => [
+    Column.new("name", 1, 25),
+    Column.new("zip", 26, 5),
+    Column.new("balance", 31, 5)]}
+
+customer_file = FLRFile.new(File.new("customers.dat"), :customer, columns, 1, [:line_number])
+
+
+# You can read through the file and access the fields with methods named after the columns:
+customer_file.each do |record|
+  puts "Customer #{customer_file.line_number.to_s} #{record.name} #{record.zip} "
+end
+
+
+# You can get the values by attribute name like a hash
+def show(record)
+  print record.members.map { |m| m.to_s + ": " + record[m].to_s }.join(", ") + "\n"
+end
+
+# metadata for customer_orders file
+layouts = {:customer => [
+    Column.new("name", 1, 25),
+    Column.new("zip", 26, 5),
+    Column.new("balance", 31, 5)],
+           :order => [
+               Column.new("order_num", 1, 8),
+               Column.new("date", 9, 10),]}
+
+
+customer_orders_file = FLRFile.new(
+    File.new("customer_orders.dat"),
+    {"C" => :customer, "O" => :order}, # Use these characters as record type markers
+    layouts,
+    0, # shift parsed string 0 columns to the left of the indicated start column
+    {:customer => [:line_number, :record_type], :order => [:line_number, :record_type]}) # Add these columns to the indicated record types post read
+
+
+customer_orders_file.each do |record|
+  show record
+end
+
+
+puts " ----- You can also use metadata in Ruby -----"
+# Use Ruby metadata
+
+layout = {:customer => {
+    :name => 1..25,
+    :zip => 26..30,
+    :balance => 31..35
+
+}
+}
+customer_file = FLRFile.new(File.new("customers.dat"), :customer, layout, 1, [:line_number])
+
+
+# You can read through the file and access the fields with methods named after the columns:
+customer_file.each do |record|
+  puts "Customer #{customer_file.line_number.to_s} #{record.name} #{record.zip} "
+end
+

data/test/flrfile_test.rb

@@ -0,0 +1,301 @@
+require 'test_hflr'
+
+class FLRFileTest < Test::Unit::TestCase
+
+  def setup
+    var_type = Struct.new(:name, :start, :len)
+
+    # Split up your metadata by record type.
+
+    @layouts =
+        {:household => [var_type.new(:rectypeh, 1, 1), var_type.new(:phone, 2, 1), var_type.new("mortgage", 3, 1)],
+         :person => [var_type.new(:rectypep, 1, 1), var_type.new("age", 2, 3), var_type.new("sex", 5, 1), var_type.new("marst", 6, 1)]}
+
+    # Give the values used in the data for each record type  
+    @record_types = {:household => {:discriminator => 'H', :position => 1}, :person => {:discriminator => 'P', :position => 1}}
+  end
+
+  def teardown
+  end
+
+  def test_initialize
+    sample_data_path = File.dirname(__FILE__)
+    fwf = FLRFile.new(
+        File.new("#{sample_data_path}/sample.dat"),
+        @record_types, # Record types to read from the file, all others will be ignored
+        @layouts, # metadata for all record types
+        1, # column  0 starts at logical location 1
+        {:household => [:people], :person => [:household_id, :pserial]} # extra columns by record type
+    )
+
+    # Extra columns + record_type accessors should have been created
+    hh_struct = fwf.record_template[:household].record_structure.new
+    assert hh_struct.respond_to?(:record_type), "household record should have record_type method"
+    p_struct = fwf.record_template[:person].record_structure.new
+    assert p_struct.respond_to?(:household_id), "Should have household_id as an extra column"
+    assert p_struct.respond_to?(:record_type), "Should have record_type method"
+
+
+    fwf = FLRFile.new(
+        File.new("#{sample_data_path}/sample.dat"),
+        @record_types, # Record types to read from the file, all others will be ignored
+        @layouts, # metadata for all record types
+        1)
+
+    # Should still have added the record_type method but none of the others
+    hh_struct = fwf.record_template[:household].record_structure.new
+    assert hh_struct.respond_to?(:record_type), "Should have record_type method"
+    assert !hh_struct.respond_to?(:people)
+  end
+
+  def test_get_record_type
+    sample_data_path = File.dirname(__FILE__)
+    fwf = FLRFile.new(
+        File.new("#{sample_data_path}/sample.dat"),
+        @record_types, # Record types to read from the file, all others will be ignored
+        @layouts, # metadata for all record types
+        1, # column  0 starts at logical location 1
+        {:household => [:people], :person => [:household_id, :pserial]} # extra columns by record type
+    )
+    assert_nil fwf.get_record_type(nil)
+    assert_equal :household, fwf.get_record_type("H123")
+    assert_equal :person, fwf.get_record_type("P1234")
+    assert_equal nil, fwf.get_record_type("C123")
+
+  end
+
+
+  def test_build_record
+    sample_data_path = File.dirname(__FILE__)
+    fwf = FLRFile.new(
+        File.new("#{sample_data_path}/sample.dat"),
+        @record_types, # Record types to read from the file, all others will be ignored
+        @layouts, # metadata for all record types
+        1, # column  0 starts at logical location 1
+        {:household => [:people], :person => [:household_id, :pserial]} # extra columns by record type
+    )
+
+    assert_equal nil, fwf.build_record(nil)
+    rec = fwf.build_record("H012345666665555444333")
+    assert_equal :household, rec[:record_type]
+
+    assert_raise RuntimeError do
+      fwf.build_record("c23abbbc")
+    end
+
+  end
+
+  def test_fast_next_record
+    sample_data_path = File.dirname(__FILE__)
+
+    layout = {:customer => {
+        :name => 1..25,
+        :zip => 26..30,
+        :balance => 31..35
+    }
+    }
+
+    customer_file = FLRFile.new(File.new(sample_data_path + "/customers.dat"), :customer, layout, 1)
+    customer_file.set_fast
+    customer_file.ranges=[(0..2)]
+
+    records = []
+    until customer_file.finished?
+      record = customer_file.next_record
+
+      puts record.inspect
+      records << record
+    end
+
+    assert_equal 3, records.size
+
+# Check that the records aren't off by one
+    assert_equal "Jane Smith", records[1].name.strip
+    assert_equal "John Smith", records.last.name.strip
+
+  end
+
+  def test_partial_fast_next_line
+    sample_data_path = File.dirname(__FILE__)
+
+    layout = {:customer => {
+        :name => 1..25,
+        :zip => 26..30,
+        :balance => 31..35
+    }
+    }
+
+    customer_file = FLRFile.new(File.new(sample_data_path + "/customers.dat"), :customer, layout, 1)
+    customer_file.set_fast
+    customer_file.ranges=[(0..1)]
+
+    records = []
+    until customer_file.finished?
+      record = customer_file.next_record
+      puts record.inspect
+      records << record
+    end
+
+    assert_equal 2, records.size
+
+# Check that the records aren't off by one
+    assert_equal "Jane Smith", records[1].name.strip
+
+  end
+
+  def test_fast_each
+    sample_data_path = File.dirname(__FILE__)
+
+    layout = {:customer => {
+        :name => 1..25,
+        :zip => 26..30,
+        :balance => 31..35
+    }
+    }
+
+    customer_file = FLRFile.new(File.new(sample_data_path + "/customers.dat"), :customer, layout, 1)
+    customer_file.set_fast
+    customer_file.ranges=[(0..1)]
+
+    records = []
+    customer_file.each do |record|
+
+      puts record.inspect
+      records << record
+    end
+
+    assert_equal 2, records.size
+
+# Check that the records aren't off by one
+    assert_equal "Jane Smith               ", records[1].name
+
+  end
+
+
+  def test_each
+    sample_data_path = File.dirname(__FILE__)
+    fwf = FLRFile.new(
+        File.new("#{sample_data_path}/sample.dat"),
+        @record_types,
+        @layouts,
+        1,
+        {:household => [:record_type, :people], :person => [:household_id, :pserial, :record_type]})
+
+    records = []
+
+    fwf.each do |record|
+      records << record
+    end
+    assert records.first.respond_to?(:record_type)
+    assert_equal :person, records.last.record_type
+    assert_equal :household, records[0].record_type
+    assert_equal :person, records[1].record_type
+
+  end
+
+  def test_next_record
+    sample_data_path = File.dirname(__FILE__)
+    fwf = FLRFile.new(
+        File.new("#{sample_data_path}/sample.dat"), # data is in this file
+        @record_types, # Records of different types have these labels
+        @layouts, # metadata for creating record structs
+        1, # All metadata starting column locations are to be shifted 1 left
+        {:household => [:people], :person => [:household_id, :pserial]}) # Extra columns not to come from metadata
+
+    records = []
+    while (rec = fwf.next_record) do
+      records << rec
+
+    end
+
+    assert_equal :household, records.first.record_type
+
+    assert records.first.respond_to?(:record_type)
+
+    # The last record is a person type and should not have a 'people' accessor
+    assert !records.last.respond_to?(:people)
+
+    # Should have added these accessors from the extra_columns argument above
+    assert records.first.respond_to?(:people)
+    assert records.last.respond_to?(:household_id)
+
+    assert_equal :household, records[0].record_type
+    assert_equal :person, records[1].record_type
+
+  end
+
+  def test_open
+    record = Struct.new(:rectypeh, :phone, :mortgage, :record_type)
+
+    sample_data_path = File.dirname(__FILE__)
+    FLRFile.open(
+        "#{sample_data_path}/sample_out.dat", # data is in this file
+        "w", # open file for writing
+        @record_types, # Records of different types have these labels
+        @layouts, # metadata for creating record structs
+        1) do |fwf| # All metadata starting column locations are to be shifted 1 left
+      assert_equal FLRFile, fwf.class
+      fwf << record.new("H", 1, 2, :household)
+      fwf << ["H", 1, 3, :household]
+    end
+
+    assert File.exists?("#{sample_data_path}/sample.dat") # data is in this file
+
+    var = Struct.new(:name, :start, :len)
+    l = {:customer => [var.new("name", 1, 25), var.new("age", 26, 3)]}
+    fwf = FLRFile.new(File.new("sample2_out.dat", "w"),
+                      :customer, l, 1)
+
+    fwf << ["joe", 25, :customer]
+    fwf.close
+
+  end
+
+
+  def test_line_type
+    sample_data_path = File.dirname(__FILE__)
+    fwf = FLRFile.new(
+        File.new("#{sample_data_path}/sample.dat"),
+        @record_types, # Record types to read from the file, all others will be ignored 
+        @layouts, # metadata for all record types
+        1, # column  0 starts at logical location 1 
+        {:household => [:people], :person => [:household_id, :pserial]} # extra columns by record type
+    )
+
+    assert_equal :unknown, fwf.line_type(nil)
+    assert_equal :household, fwf.line_type("H123")
+    assert_equal :person, fwf.line_type("P123")
+    assert_equal :unknown, fwf.line_type("C123")
+  end
+
+  def test_ranges
+  end
+
+  def test_in_range
+  end
+
+
+  def test_get_next_known_line_type
+    sample_data_path = File.dirname(__FILE__)
+    fwf = FLRFile.new(
+        File.new("#{sample_data_path}/sample_activities.dat"),
+        @record_types, # Record types to read from the file, all others will be ignored
+        @layouts, # metadata for all record types
+        1, # column  0 starts at logical location 1
+        {:household => [:people], :person => [:household_id, :pserial]} # extra columns by record type
+    )
+    # By reading the sample_activities file with only the household and person record types know
+    # we should get the activity and who records to be skipped.
+    while (rec=fwf.get_next_known_line_type)
+
+      unless rec.strip.empty?
+
+        assert %w(P H).include?(rec[0..0])
+
+      end
+    end
+
+  end
+end
+
+

data/test/record_template_test.rb

@@ -0,0 +1,169 @@
+require 'test_hflr'
+
+class RecordTemplateTest < Test::Unit::TestCase
+
+  Var = Struct.new(:name, :start, :len)
+
+  # Some real world metadata. This is only layout for the starting portion of the ATUS / CPS household record
+  # which actually extends to beyond column 117
+  AtusHH =[[:RECTYPEH, 1, 1,],
+           [:CASEID, 2, 14],
+           [:AGEYCHILD, 16, 3],
+           [:SERIAL, 19, 7],
+           [:HH_NUMADULTS, 26, 2],
+           [:FAMBUS_RESP, 28, 2],
+           [:FAMBUS_OTHER, 30, 2],
+           [:FAMBUS_SPOUSE, 32, 2],
+           [:FAMBUS, 34, 2],
+           [:HH_CHILD, 36, 2],
+           [:HH_NUMKIDS, 38, 2],
+           [:HH_SIZE, 40, 3],
+           [:HH_NUMEMPLD, 43, 3],
+           [:FAMINCOME, 46, 3]]
+
+
+  def setup
+
+    hh_vars = AtusHH.map { |v| Var.new(v[0], v[1], v[2]) }
+
+    @vars = {:household => hh_vars,
+             :person => [Var.new("age", 2, 3), Var.new("sex", 5, 1)],
+             :activity => [Var.new("where", 1, 5), Var.new("activity", 6, 5)],
+             :who => [Var.new("relatew", 1, 2)]}
+    @record_types = {"H" => :household, "P" => :person, "A" => :activity, "W" => :who}
+    @record_type_symbols = @record_types.invert
+  end
+
+  def test_create
+    record_layouts = @vars # variables by record type
+    templates = HFLR::RecordTemplate.create(record_layouts, @record_type_symbols, 1)
+    assert_equal @vars.keys, templates.keys
+
+    household_field_pattern = templates[:household].field_pattern
+    person_field_pattern =templates[:person].field_pattern
+    activity_field_pattern = templates[:activity].field_pattern
+    who_field_pattern = templates[:who].field_pattern
+
+
+    assert household_field_pattern.is_a?(String)
+    assert person_field_pattern.size>2, "field pattern should have at least one variable"
+    assert activity_field_pattern.size>2, "field pattern for activity should have at least one variable"
+    assert who_field_pattern.size>2, "field pattern for who should have at least one variable"
+
+
+    household_record_struct = templates[:household].record_structure.new
+    assert household_record_struct.respond_to?(:HH_SIZE)
+
+  end
+
+  def test_create_template_class
+    template = HFLR::RecordTemplate.create_template_class(:person, @record_type_symbols[:person], @vars[:person], 1, {})
+    assert template.respond_to?(:field_pattern)
+    assert template.respond_to?(:record_structure)
+    record_structure = template.record_structure
+    assert record_structure.new.is_a?(Struct)
+    assert record_structure.new.respond_to?(:record_type)
+
+  end
+
+
+  def test_get_pattern
+    household_layout = @vars[:household]
+
+    pattern = HFLR::RecordTemplate.get_pattern(household_layout)
+    assert_equal "@1a1", pattern[0..3]
+
+    # Adjust the location ('@') leftward (the metadata refers to the 0th column as column 1.)
+    pattern = HFLR::RecordTemplate.get_pattern(household_layout, 1)
+    assert_equal "@0a1", pattern[0..3]
+
+    vars_in_pattern = pattern.scan("a").size
+    assert_equal household_layout.size, vars_in_pattern
+
+  end
+
+  def test_build_record
+    templates = HFLR::RecordTemplate.create(@vars, @record_type_symbols, 1)
+    hh_str = "1200501010500069980000001020000000000000020009960200009999999999999999991330299902305030201034300000000037110550997797000000000007700100000000200411000000000"
+
+    # from a string to a record struct
+    household_rec = templates[:household].build_record(hh_str)
+
+    assert household_rec.is_a?(Struct)
+    assert household_rec.values.size> @vars[:household].size, "Should be values for the extra columns"
+
+    # Check a few things...
+    assert_equal 1, household_rec[:SERIAL].to_i
+
+
+    assert_equal 1, household_rec.SERIAL.to_i
+
+    assert_equal "1", household_rec[0]
+    assert_equal "1", household_rec.RECTYPEH
+    assert_equal "1", household_rec[:RECTYPEH]
+
+
+  end
+
+  def test_build_line
+    templates = HFLR::RecordTemplate.create(@vars, @record_type_symbols, 1)
+
+    hh_str = "H200501010500069980000001020000000000000020009960200009999999999999999991330299902305030201034300000000037110550997797000000000007700100000000200411000000000"
+
+    # from a string to a record struct
+    household_rec = templates[:household].build_record(hh_str)
+    assert_equal "002", hh_str[39..41]
+    assert_equal "002", household_rec.HH_SIZE
+
+
+    # back to a string
+    new_hh_str = templates[:household].build_line(household_rec)
+
+
+    assert_equal "002", new_hh_str[39..41]
+
+    # Some of the values in hh_str won't be in new_hh_str because not all data
+    # in hh_str is mapped by household_layout, but the mapped variables should have
+    # the same values.
+
+    @vars[:household].each do |v|
+      format_str = "@#{(v.start-1).to_s}a#{v.len.to_s}"
+      orig_data = hh_str.unpack(format_str)
+      new_data = new_hh_str.unpack(format_str)
+
+
+      assert_equal new_data, orig_data, "Comparing #{v.name} #{format_str}"
+    end
+  end
+
+
+  def test_format_fields
+    templates = HFLR::RecordTemplate.create(@vars, @record_type_symbols, 1)
+
+    formatted_fields = templates[:who].send(:format_fields, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])
+
+    widths = @vars[:who].map { |v| v.len }
+    formatted_fields.size.times do |i|
+      assert_equal formatted_fields[i].size, widths[i], "Width of #{@vars[:who][i].name} should have been #{widths[i].to_s}"
+    end
+
+  end
+
+  def test_write_format
+    templates = HFLR::RecordTemplate.create(@vars, @record_type_symbols, 1)
+
+    assert_equal "abc", templates[:activity].send(:right_format, "abc", 3)
+    assert_equal "abc   ", templates[:activity].send(:right_format, "abc", 6)
+    assert_equal "3", templates[:activity].send(:right_format, 3, 1)
+    assert_equal "005", templates[:activity].send(:right_format, 5, 3)
+    assert_equal "ZZZ", templates[:activity].send(:right_format, -999998, 3)
+  end
+
+
+end
+ 
+ 
+ 
+ 
+ 
+

data/test/sample.dat

@@ -0,0 +1,7 @@
+H010599999999
+P05525
+P02326
+P01916
+H010599999999
+P02516
+

data/test/sample2_out.dat

@@ -0,0 +1 @@
+joe                      025

data/test/sample_activities.dat

@@ -0,0 +1,12 @@
+H010599999999
+P05525
+P02326
+A12344001201
+W230
+P01916
+H010599999999
+P02516
+A2212320601
+A2232320602
+W100
+

data/test/sample_out.dat

@@ -0,0 +1,2 @@
+H12
+H13

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'stringio'
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/hflr'

data/test/test_hflr.rb

@@ -0,0 +1,4 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+require File.dirname(__FILE__) + "/record_template_test"
+require File.dirname(__FILE__) + "/flrfile_test"
+

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification 
+name: hflr2
+version: !ruby/object:Gem::Version 
+  hash: 23
+  prerelease: 
+  segments: 
+  - 1
+  - 0
+  - 0
+  version: 1.0.0
+platform: ruby
+authors: 
+- Colin Davis
+- Bozydar Sobczak
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-10-24 00:00:00 Z
+dependencies: []
+
+description: |-
+  HFLR2 -- Hierarchical Fixed Length Records
+  
+  NOTE: This gem is a modification of the hflr gem. It can be not compatible with it.
+  
+  Allows you to read and write files of fixed width records when the file contains one or more
+  than one type of record.  
+  
+  Install with 'gem install hflr2'
+  
+  See the tests and examples bundled with this gem.
+email: colin.c.davis@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- README.txt
+files: 
+- History.txt
+- README.txt
+- hflr2.gemspec
+- lib/hflr.rb
+- lib/hflr/fl_record_file.rb
+- lib/hflr/hflr.rb
+- lib/hflr/record_template.rb
+- test/customer_orders.dat
+- test/customers.dat
+- test/examples.rb
+- test/flrfile_test.rb
+- test/record_template_test.rb
+- test/sample.dat
+- test/sample2_out.dat
+- test/sample_activities.dat
+- test/sample_out.dat
+- test/test_helper.rb
+- test/test_hflr.rb
+homepage: 
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: HFLR2 -- Hierarchical Fixed Length Records 2 Allows you to read and write files of fixed width records when the file contains one or more than one type of record
+test_files: 
+- test/test_hflr.rb
+- test/test_helper.rb