hflr 0.10.1

data/tasks/test.rake

@@ -0,0 +1,40 @@
+
+if test(?e, PROJ.test.file) or not PROJ.test.files.to_a.empty?
+require 'rake/testtask'
+
+namespace :test do
+
+  Rake::TestTask.new(:run) do |t|
+    t.libs = PROJ.libs
+    t.test_files = if test(?f, PROJ.test.file) then [PROJ.test.file]
+                   else PROJ.test.files end
+    t.ruby_opts += PROJ.ruby_opts
+    t.ruby_opts += PROJ.test.opts
+  end
+
+  if HAVE_RCOV
+    desc 'Run rcov on the unit tests'
+    task :rcov => :clobber_rcov do
+      opts = PROJ.rcov.opts.dup << '-o' << PROJ.rcov.dir
+      opts = opts.join(' ')
+      files = if test(?f, PROJ.test.file) then [PROJ.test.file]
+              else PROJ.test.files end
+      files = files.join(' ')
+      sh "#{RCOV} #{files} #{opts}"
+    end
+
+    task :clobber_rcov do
+      rm_r 'coverage' rescue nil
+    end
+  end
+
+end  # namespace :test
+
+desc 'Alias to test:run'
+task :test => 'test:run'
+
+task :clobber => 'test:clobber_rcov' if HAVE_RCOV
+
+end
+
+# EOF

data/tasks/zentest.rake

@@ -0,0 +1,36 @@
+if HAVE_ZENTEST
+
+# --------------------------------------------------------------------------
+if test(?e, PROJ.test.file) or not PROJ.test.files.to_a.empty?
+require 'autotest'
+
+namespace :test do
+  task :autotest do
+    Autotest.run
+  end
+end
+
+desc "Run the autotest loop"
+task :autotest => 'test:autotest'
+
+end  # if test
+
+# --------------------------------------------------------------------------
+if HAVE_SPEC_RAKE_SPECTASK and not PROJ.spec.files.to_a.empty?
+require 'autotest/rspec'
+
+namespace :spec do
+  task :autotest do
+    load '.autotest' if test(?f, '.autotest')
+    Autotest::Rspec.run
+  end
+end
+
+desc "Run the autotest loop"
+task :autotest => 'spec:autotest'
+
+end  # if rspec
+
+end  # if HAVE_ZENTEST
+
+# EOF

data/test/customer_orders.dat

@@ -0,0 +1,5 @@
+CJoe Smith                 55455025.53 
+O0005233110-10-2008
+CJane Smith                55404015.25
+O0054933310-11-2008
+O0075789110-12-2008

data/test/customers.dat

@@ -0,0 +1,2 @@
+Joe Smith                 55455025.53 
+Jane Smith                55404015.25

data/test/examples.rb

@@ -0,0 +1,48 @@
+require '../lib/fl_record_file'
+
+
+# Read a file with only one record type (no record type markers)
+
+# metadata for customer file
+Column = Struct.new(:name,:start,:len)
+columns = {:customer=>[
+Column.new("name",1,25),
+Column.new("zip",26,5),
+Column.new("balance",31,5)]}
+  
+customer_file = FixedLengthRecordFile.new(File.new("customers.dat"), :customer, columns, 1, [:line_number])
+
+customer_file.each do |record|
+  puts "Customer #{customer_file.line_number.to_s} #{record.name} #{record.zip} "
+end
+
+
+# You can get the values by attribute name like a hash
+def show(record)
+  print record.members.map{|m| m.to_s + ": " + record[m].to_s}.join(", ") + "\n" 
+  end
+
+# metadata for customer_orders file
+layouts = {:customer=>[
+  Column.new("name",1,25),
+  Column.new("zip",26,5),
+  Column.new("balance",31,5)],  
+:order=>[
+  Column.new("order_num",1,8),
+  Column.new("date",9,10),]}
+  
+  
+customer_orders_file = FixedLengthRecordFile.new(
+  File.new("customer_orders.dat"), 
+  {"C"=>:customer,"O"=>:order},# Use these characters as record type markers 
+  layouts, 
+  0, # shift parsed string 0 columns to the left of the indicated start column 
+  {:customer=>[:line_number,:record_type],:order=>[:line_number,:record_type]}) # Add these columns to the indicated record types post read
+  
+  
+  customer_orders_file.each do |record|
+    show record
+  end
+  
+  
+  

data/test/fixed_length_record_file_test.rb

@@ -0,0 +1,212 @@
+
+
+
+class FixedLengthRecordFileTest < Test::Unit::TestCase 
+  
+  
+
+  def setup
+    var_type = Struct.new(:name,:start,:len)
+    
+    # Split up your metadata by record type.
+    
+    @layouts = 
+      {:household=>[var_type.new(:rectypeh,1,1), var_type.new(:phone,2,1), var_type.new("mortgage",3,1)],
+        :person=>[var_type.new(:rectypep,1,1), var_type.new("age",2,3), var_type.new("sex",5,1), var_type.new("marst",6,1)]}
+    
+    # Give the values used in the data for each record type  
+    @record_types ={"H"=>:household,"P"=>:person} 
+  end
+  
+  def teardown
+  end
+  
+  
+  def test_initialize
+  sample_data_path = File.dirname(__FILE__) 
+  fwf = FixedLengthRecordFile.new(
+        File.new("#{sample_data_path}/sample.dat"),
+        @record_types, # Record types to read from the file, all others will be ignored 
+        @layouts,# metadata for all record types
+        1, # column  0 starts at logical location 1 
+        {:household=>[:people],:person=>[:household_id,:pserial]} # extra columns by record type
+        )
+  
+   # Extra columns + record_type accessors should have been created
+     hh_struct = fwf.record_template[:household].record_structure.new
+     assert hh_struct.respond_to?(:record_type),"household record should have record_type method"
+     p_struct = fwf.record_template[:person].record_structure.new
+     assert p_struct.respond_to?(:household_id),"Should have household_id as an extra column"
+     assert p_struct.respond_to?(:record_type),"Should have record_type method"
+   
+  
+  fwf = FixedLengthRecordFile.new(
+          File.new("#{sample_data_path}/sample.dat"),
+          @record_types, # Record types to read from the file, all others will be ignored 
+          @layouts,# metadata for all record types
+          1          )
+          
+          # Should still have added the record_type method but none of the others
+          hh_struct = fwf.record_template[:household].record_structure.new
+          assert hh_struct.respond_to?(:record_type),"Should have record_type method"
+          assert !hh_struct.respond_to?(:people)  
+  end
+  
+  def test_get_record_type
+      sample_data_path = File.dirname(__FILE__) 
+        fwf = FixedLengthRecordFile.new(
+          File.new("#{sample_data_path}/sample.dat"),
+          @record_types, # Record types to read from the file, all others will be ignored 
+          @layouts,# metadata for all record types
+          1, # column  0 starts at logical location 1 
+          {:household=>[:people],:person=>[:household_id,:pserial]} # extra columns by record type
+          )
+      assert_nil fwf.get_record_type(nil)
+      assert_equal :household,fwf.get_record_type("H123")
+      assert_equal :person,fwf.get_record_type("P1234")
+      assert_equal nil, fwf.get_record_type("C123") 
+      
+  end
+  
+  
+  
+  
+  def test_build_record
+    sample_data_path = File.dirname(__FILE__) 
+      fwf = FixedLengthRecordFile.new(
+      File.new("#{sample_data_path}/sample.dat"),
+      @record_types, # Record types to read from the file, all others will be ignored 
+      @layouts,# metadata for all record types
+      1, # column  0 starts at logical location 1 
+      {:household=>[:people],:person=>[:household_id,:pserial]} # extra columns by record type
+      )
+      
+      assert_equal nil, fwf.build_record(nil)
+      rec = fwf.build_record("H012345666665555444333")
+      assert_equal :household,rec[:record_type]
+      
+      assert_raise RuntimeError do 
+        fwf.build_record("c23abbbc")
+      end
+           
+  end
+  
+  
+  def test_each
+    sample_data_path = File.dirname(__FILE__) 
+     fwf = FixedLengthRecordFile.new(
+       File.new("#{sample_data_path}/sample.dat"),
+        @record_types, 
+        @layouts,
+        1,
+        {:household=>[:record_type,:people],:person=>[:household_id,:pserial,:record_type]})
+  
+    records = []
+  
+  fwf.each do |record|
+    records << record    
+  end  
+  assert records.first.respond_to?(:record_type)
+  assert_equal :person, records.last.record_type  
+  assert_equal :household,records[0].record_type
+  assert_equal :person, records[1].record_type 
+    
+  end
+  
+  def test_next_record
+    sample_data_path = File.dirname(__FILE__) 
+    fwf = FixedLengthRecordFile.new(
+      File.new("#{sample_data_path}/sample.dat"), # data is in this file
+      @record_types, # Records of different types have these labels 
+      @layouts, # metadata for creating record structs
+      1, # All metadata starting column locations are to be shifted 1 left
+      {:household=>[:people],:person=>[:household_id,:pserial]}) # Extra columns not to come from metadata
+      
+      records = []
+      while rec = fwf.next_record do 
+        records << rec
+      end
+      
+      assert_equal :household, records.first.record_type 
+      
+  assert records.first.respond_to?(:record_type)
+  
+  # The last record is a person type and should not have a 'people' accessor
+  assert !records.last.respond_to?(:people)
+  
+  # Should have added these accessors from the extra_columns argument above
+  assert records.first.respond_to?(:people)
+  assert records.last.respond_to?(:household_id)
+  
+  assert_equal :household,records[0].record_type
+  assert_equal :person, records[1].record_type 
+          
+  end
+  
+  def test_open      
+    record = Struct.new(:rectypeh,:phone,:mortgage,:record_type)
+    
+    sample_data_path = File.dirname(__FILE__) 
+     FixedLengthRecordFile.open(
+    "#{sample_data_path}/sample_out.dat", # data is in this file
+    "w",# open file for writing
+          @record_types, # Records of different types have these labels 
+          @layouts, # metadata for creating record structs
+          1)  do |fwf|# All metadata starting column locations are to be shifted 1 left            
+          assert_equal FixedLengthRecordFile, fwf.class
+          fwf << record.new("H",1,2,:household)
+          fwf << ["H",1,3,:household]
+          end
+                           
+     assert File.exists?("#{sample_data_path}/sample.dat") # data is in this file
+     
+     var = Struct.new(:name,:start,:len)
+     l = {:customer=>[var.new("name",1,25),var.new("age",26,3)]}
+     fwf = FixedLengthRecordFile.new(File.new("sample2_out.dat","w"),
+         :customer, l,1) 
+     
+     fwf << ["joe",25,:customer]
+     fwf.close
+     
+  end
+  
+
+def test_line_type
+      sample_data_path = File.dirname(__FILE__) 
+      fwf = FixedLengthRecordFile.new(
+        File.new("#{sample_data_path}/sample.dat"),
+        @record_types, # Record types to read from the file, all others will be ignored 
+        @layouts,# metadata for all record types
+        1, # column  0 starts at logical location 1 
+        {:household=>[:people],:person=>[:household_id,:pserial]} # extra columns by record type
+        )
+  
+  assert_equal :unknown, fwf.line_type(nil)
+  assert_equal :household,fwf.line_type("H123")
+  assert_equal :person,fwf.line_type("P123")
+  assert_equal :unknown, fwf.line_type("C123")
+end
+  
+  def test_get_next_known_line_type
+    sample_data_path = File.dirname(__FILE__) 
+      fwf = FixedLengthRecordFile.new(
+          File.new("#{sample_data_path}/sample_activities.dat"),
+          @record_types, # Record types to read from the file, all others will be ignored 
+          @layouts,# metadata for all record types
+          1, # column  0 starts at logical location 1 
+          {:household=>[:people],:person=>[:household_id,:pserial]} # extra columns by record type
+          )
+  # By reading the sample_activities file with only the household and person record types know
+  # we should get the activity and who records to be skipped.
+  while rec=fwf.get_next_known_line_type 
+
+    unless rec.strip.empty?
+
+    assert ["P","H"].include?(rec[0..0])
+
+      end
+  end
+  
+  end
+  
+end

data/test/record_template_test.rb

@@ -0,0 +1,174 @@
+		
+
+
+class RecordTemplateTest < Test::Unit::TestCase
+   
+  
+	  Var = Struct.new(:name,:start,:len)
+  
+  # Some real world metadata. This is only layout for the starting portion of the ATUS / CPS household record
+  # which actually extends to beyond column 117
+  AtusHH =[[:RECTYPEH,	1,	1,],
+[:CASEID,	2,	14],
+[:AGEYCHILD,	16,	3],
+[:SERIAL,	19,	7],
+[:HH_NUMADULTS,	26,	2],
+[:FAMBUS_RESP,	28,	2],
+[:FAMBUS_OTHER,	30,	2],
+[:FAMBUS_SPOUSE,	32,	2],
+[:FAMBUS,	34,	2],
+[:HH_CHILD,	36,	2],
+[:HH_NUMKIDS,	38,	2],
+[:HH_SIZE,	40,	3],
+[:HH_NUMEMPLD,	43,	3],
+[:FAMINCOME,	46,	3]]
+
+
+  
+  def setup      
+    
+    hh_vars = AtusHH.map{|v| Var.new(v[0],v[1],v[2])}
+
+  @vars = {:household=>hh_vars,
+  :person=>[Var.new("age",2,3),Var.new("sex",5,1)],
+  :activity=>[Var.new("where",1,5),Var.new("activity",6,5)],
+  :who=>[Var.new("relatew",1,2)]}
+  @record_types = {"H"=>:household,"P"=>:person,"A"=>:activity,"W"=>:who} 
+  @record_type_symbols = @record_types.invert
+  end  
+  
+ def  test_create
+   record_layouts = @vars # variables by record type
+   templates = HFLR::RecordTemplate.create(record_layouts, @record_type_symbols, 1)
+   assert_equal @vars.keys, templates.keys
+   
+   household_field_pattern =  templates[:household].field_pattern
+   person_field_pattern =templates[:person].field_pattern
+   activity_field_pattern = templates[:activity].field_pattern
+   who_field_pattern = templates[:who].field_pattern
+   
+   
+   assert household_field_pattern.is_a?(String)
+   assert person_field_pattern.size>2,"field pattern should have at least one variable"
+   assert activity_field_pattern.size>2,"field pattern for activity should have at least one variable"
+   assert who_field_pattern.size>2,"field pattern for who should have at least one variable"
+   
+   
+   household_record_struct = templates[:household].record_structure.new
+   assert household_record_struct.respond_to?(:HH_SIZE)
+  
+ end
+ 
+ def test_create_template_class
+   template = HFLR::RecordTemplate.create_template_class(:person,@record_type_symbols[:person], @vars[:person],1,{})
+   assert template.respond_to?(:field_pattern)
+   assert template.respond_to?(:record_structure)
+   record_structure = template.record_structure
+  assert record_structure.new.is_a?(Struct)
+  assert record_structure.new.respond_to?(:record_type)
+   
+ end
+ 
+ 
+ def test_get_pattern
+    household_layout = @vars[:household]
+    
+    pattern = HFLR::RecordTemplate.get_pattern(household_layout)
+    assert_equal "@1A1",pattern[0..3]
+    
+    # Adjust the location ('@') leftward (the metadata refers to the 0th column as column 1.)
+    pattern = HFLR::RecordTemplate.get_pattern(household_layout,1)
+    assert_equal "@0A1",pattern[0..3]
+    
+    vars_in_pattern = pattern.scan("A").size 
+    assert_equal household_layout.size, vars_in_pattern
+         
+ end
+
+def test_build_record
+  templates = HFLR::RecordTemplate.create(@vars,@record_type_symbols,1)
+  hh_str = "1200501010500069980000001020000000000000020009960200009999999999999999991330299902305030201034300000000037110550997797000000000007700100000000200411000000000"  
+  
+  # from a string to a record struct
+  household_rec = templates[:household].build_record(hh_str)
+    
+  assert household_rec.is_a?(Struct)
+  assert household_rec.values.size> @vars[:household].size,"Should be values for the extra columns"
+  
+  # Check a few things...
+  assert_equal 1,household_rec[:SERIAL].to_i
+  
+
+  assert_equal 1,household_rec.SERIAL.to_i
+  
+  assert_equal "1",household_rec[0]
+  assert_equal "1",household_rec.RECTYPEH
+  assert_equal "1",household_rec[:RECTYPEH]
+  
+  
+
+  end
+  
+  def test_build_line
+  templates = HFLR::RecordTemplate.create(@vars,@record_type_symbols,1)
+
+    hh_str = "H200501010500069980000001020000000000000020009960200009999999999999999991330299902305030201034300000000037110550997797000000000007700100000000200411000000000"  
+    
+    # from a string to a record struct
+    household_rec = templates[:household].build_record(hh_str)
+    assert_equal "002",hh_str[39..41]    
+    assert_equal "002",household_rec.HH_SIZE
+
+    
+    # back to a string
+    new_hh_str = templates[:household].build_line(household_rec)
+    
+       
+    assert_equal "002",new_hh_str[39..41]
+    
+    # Some of the values in hh_str won't be in new_hh_str because not all data
+    # in hh_str is mapped by household_layout, but the mapped variables should have
+    # the same values.
+    
+    @vars[:household].each do |v|
+      format_str = "@#{(v.start-1).to_s}a#{v.len.to_s}" 
+      orig_data = hh_str.unpack(format_str) 
+      new_data = new_hh_str.unpack(format_str)
+    
+   
+      assert_equal new_data, orig_data,"Comparing #{v.name} #{format_str}" 
+    end       
+  end
+  
+  
+  def test_format_fields
+    templates = HFLR::RecordTemplate.create(@vars,@record_type_symbols,1)
+    
+    formatted_fields = templates[:who].send(:format_fields,[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15])
+    
+    widths = @vars[:who].map{|v| v.len}
+    formatted_fields.size.times do |i|
+      assert_equal formatted_fields[i].size, widths[i],"Width of #{@vars[:who][i].name} should have been #{widths[i].to_s}" 
+    end
+    
+  end
+  
+  def test_write_format    
+   templates = HFLR::RecordTemplate.create(@vars,@record_type_symbols,1)
+   
+   assert_equal "abc",templates[:activity].send(:right_format,"abc",3)
+   assert_equal "abc   ",templates[:activity].send(:right_format,"abc",6)
+    assert_equal "3",templates[:activity].send(:right_format,3,1)
+    assert_equal "005",templates[:activity].send(:right_format,5,3)
+    assert_equal "ZZZ",templates[:activity].send(:right_format,-999998,3)
+  end
+  
+  
+   
+end
+ 
+ 
+ 
+ 
+ 
+