hflr 1.0.2 → 1.1.0

data/hflr.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{hflr}
-  s.version = "1.0.2"
+  s.version = "1.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Colin Davis"]

data/lib/hflr/fl_record_file.rb

@@ -23,9 +23,34 @@ class FLRFile
         @record_template = HFLR::RecordTemplate.create(record_layouts, @record_type_symbols, logical_first_column)   
     end        
   end
+  
+  def set_fast
+  @fast = !@record_type_labels.is_a?(Hash)                    
+  unless @fast
+    raise "Cannot set fast mode with more than one record type."
+  end
+      if @fast
+       @width = get_record_width_from_file
+      end        
+  end
+  
+  def ranges=(ranges)
+    @fast or raise "Cannot read selected ranges because input file has multiple record types #{@record_type_labels.to_s}"               
+    unless ranges.first.is_a?(Range) 
+     raise "You specified a #{ranges.first.class.to_s} instead of a range in the list of ranges.  Use (a..b) to specify a range."
+     end
+     
+      @offsets =offsets_to_read(ranges, @width)
+      @ranges = ranges
+  end
+
+  
+  def in_range?(line_number)
+    @ranges ? !!(@ranges.detect{|r| r.member?(line_number)}) : true
+  end
 
 def finished?
-  @file.eof?
+  @file.eof? ||  (@fast && @offsets.empty?)
 end
 
 def close
@@ -46,8 +71,7 @@ def  build_record(line)
   return @record_template[record_type].build_record(line.chomp)                 
 end
 
-def next_record
-  @line_number += 1
+def next_record  
   build_record(get_next_known_line_type) 
 end
 
@@ -57,23 +81,44 @@ def line_type(line)
 end
 
 def get_next_known_line_type  
+  @fast ? fast_get_next_known_line_type   : sequential_get_next_known_line_type  
+end
+
+def fast_get_next_known_line_type  
+  unless @offsets.nil? ||  @offsets.empty?
+    @file.pos = @offsets.shift
+    @file.read(@width)    
+  else    
+    nil
+  end
+end
+
+def sequential_get_next_known_line_type  
   line = @file.gets 
+  @line_number+=1
   record_type = line_type(line)
-  while record_type == :unknown and (not finished?)
+  while !finished? && (!in_range?(@line_number) || record_type == :unknown)
     line = @file.gets
+    @line_number+=1
     record_type = line_type(line)
   end
   record_type == :unknown ? nil : line   
 end
 
+
+
 def each
-  @file.each_line do |line|    
-    @line_number += 1
-    unless line_type(line) == :unknown       
+  @line_number = 1
+  if @fast
+    yield(next_record) until finished?
+  else
+  @file.each_line do |line|        
+    unless line_type(line) == :unknown        || !in_range?(@line_number)
       data = build_record(line)
       yield data 
     end
   end 
+end
 end
 
  # This will take a Hash or Struct orArray;  if an Array the record type must be the last element when 
@@ -105,6 +150,17 @@ end
 
 private
 
+def offsets_to_read(ranges, width)
+  ranges.map{|r| r.map{|o| o * width}}.flatten.uniq
+end
+
+def get_record_width_from_file  
+  width = @file.gets.size 
+  @file.rewind
+  width
+end
+
+
 # If the layout is given in the convenient Ruby form
 def create_layouts(layout)
   var_class = Struct.new(:name,:start,:len)
@@ -121,3 +177,4 @@ new_layout = {}
 end
  
 end
+

data/test/customers.dat

@@ -1,2 +1,3 @@
 Joe Smith                 55455025.53 
-Jane Smith                55404015.25
+Jane Smith                55404015.25 
+John Smith                55404015.25 

data/test/flrfile_test.rb

@@ -23,14 +23,14 @@ class FLRFileTest < Test::Unit::TestCase
   
   
   def test_initialize
-  sample_data_path = File.dirname(__FILE__) 
-  fwf = FLRFile.new(
-        File.new("#{sample_data_path}/sample.dat"),
-        @record_types, # Record types to read from the file, all others will be ignored 
-        @layouts,# metadata for all record types
-        1, # column  0 starts at logical location 1 
-        {:household=>[:people],:person=>[:household_id,:pserial]} # extra columns by record type
-        )
+      sample_data_path = File.dirname(__FILE__) 
+      fwf = FLRFile.new(
+          File.new("#{sample_data_path}/sample.dat"),
+          @record_types, # Record types to read from the file, all others will be ignored 
+          @layouts,# metadata for all record types
+          1, # column  0 starts at logical location 1 
+          {:household=>[:people],:person=>[:household_id,:pserial]} # extra columns by record type
+          )
   
    # Extra columns + record_type accessors should have been created
      hh_struct = fwf.record_template[:household].record_structure.new
@@ -72,7 +72,7 @@ class FLRFileTest < Test::Unit::TestCase
   
   
   def test_build_record
-    sample_data_path = File.dirname(__FILE__) 
+      sample_data_path = File.dirname(__FILE__) 
       fwf = FLRFile.new(
       File.new("#{sample_data_path}/sample.dat"),
       @record_types, # Record types to read from the file, all others will be ignored 
@@ -90,7 +90,93 @@ class FLRFileTest < Test::Unit::TestCase
       end
            
   end
+
+def test_faast_next_record
+sample_data_path = File.dirname(__FILE__) 
+    
+  layout = {:customer=>{
+    :name=>1..25,
+    :zip=>26..30,
+    :balance=>31..35   
+  }
+  }
+
+customer_file = FLRFile.new(File.new(sample_data_path + "/customers.dat"), :customer, layout, 1)
+customer_file.set_fast
+customer_file.ranges=[(0..2)]
+
+records = []
+while !customer_file.finished?
+  record = customer_file.next_record 
+  puts record.inspect
+  records << record
+  end
+  
+assert_equal 3, records.size
+
+# Check that the records aren't off by one
+assert_equal "Jane Smith",records[1].name
+assert_equal "John Smith",records.last.name
+
+end
+
+def test_partial_fast_next_line
+sample_data_path = File.dirname(__FILE__) 
+    
+  layout = {:customer=>{
+    :name=>1..25,
+    :zip=>26..30,
+    :balance=>31..35   
+  }
+  }
+
+customer_file = FLRFile.new(File.new(sample_data_path + "/customers.dat"), :customer, layout, 1)
+customer_file.set_fast
+customer_file.ranges=[(0..1)]
+
+records = []
+while !customer_file.finished?
+  record = customer_file.next_record 
+  puts record.inspect
+  records << record
+  end
+  
+assert_equal 2, records.size
+
+# Check that the records aren't off by one
+assert_equal "Jane Smith",records[1].name
+
+end
+
+def test_fast_each
+sample_data_path = File.dirname(__FILE__) 
+    
+  layout = {:customer=>{
+    :name=>1..25,
+    :zip=>26..30,
+    :balance=>31..35   
+  }
+  }
+
+customer_file = FLRFile.new(File.new(sample_data_path + "/customers.dat"), :customer, layout, 1)
+customer_file.set_fast
+customer_file.ranges=[(0..1)]
+
+records = []
+customer_file.each do |record|
   
+  puts record.inspect
+  records << record
+  end
+  
+assert_equal 2, records.size
+
+# Check that the records aren't off by one
+assert_equal "Jane Smith",records[1].name
+
+end
+
+
   
   def test_each
     sample_data_path = File.dirname(__FILE__) 
@@ -187,6 +273,14 @@ def test_line_type
   assert_equal :person,fwf.line_type("P123")
   assert_equal :unknown, fwf.line_type("C123")
 end
+
+def test_ranges
+end
+
+def test_in_range
+end
+
+
   
   def test_get_next_known_line_type
     sample_data_path = File.dirname(__FILE__) 
@@ -211,3 +305,5 @@ end
   end
   
 end
+
+

metadata

@@ -2,7 +2,7 @@
 name: hflr
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors: 
   - Colin Davis