hflr 0.10.1

data/History.txt

@@ -0,0 +1,4 @@
+== 1.0.0 / 2009-08-03
+
+* 1 major enhancement
+  * Birthday!

data/README.txt

@@ -0,0 +1,45 @@
+= HFLR
+
+* http://ruff.rubyforge.org
+
+== Description:
+
+HFLR -- Hierarchical Fixed Length Records
+
+Allows you to read and write files of fixed width records when the file contains one or more
+than one type of record.  
+
+Install with 'gem install hflr'
+
+
+
+
+See the tests and examples bundled with this gem.
+
+
+
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009  Colin C. Davis 
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,30 @@
+# Look in the tasks/setup.rb file for the various options that can be
+# configured in this Rakefile. The .rake files in the tasks directory
+# are where the options are used.
+
+begin
+  require 'bones'
+  Bones.setup
+rescue LoadError
+  begin
+    load 'tasks/setup.rb'
+  rescue LoadError
+    raise RuntimeError, '### please install the "bones" gem ###'
+  end
+end
+
+ensure_in_path 'lib'
+require 'hflr'
+
+task :default => 'spec:run'
+
+PROJ.name = 'hflr'
+PROJ.authors = 'Colin Davis'
+PROJ.email = 'colin.c.davis@gmail.com'
+PROJ.url = 'http://ruff.rubyforge.org'
+PROJ.version = Hflr::VERSION
+PROJ.rubyforge.name = 'hflr'
+
+PROJ.spec.opts << '--color'
+
+# EOF

data/bin/hflr

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require File.expand_path(
+    File.join(File.dirname(__FILE__), %w[.. lib hflr]))
+
+# Put your code here
+
+# EOF

data/hflr.gemspec

@@ -0,0 +1,43 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{hflr}
+  s.version = "1.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Colin Davis"]
+  s.date = %q{2009-08-03}
+  s.default_executable = %q{hflr}
+  s.description = %q{HFLR -- Hierarchical Fixed Length Records
+
+Allows you to read and write files of fixed width records when the file contains one or more
+than one type of record.  
+
+Install with 'gem install hflr'
+
+See the tests and examples bundled with this gem.}
+  s.email = %q{colin.c.davis@gmail.com}
+  s.executables = ["hflr"]
+  s.extra_rdoc_files = ["History.txt", "README.txt", "bin/hflr"]
+  s.files = ["History.txt", "README.txt", "Rakefile", "bin/hflr", "lib/hflr.rb", "lib/hflr/fl_record_file.rb", "lib/hflr/hflr.rb", "lib/hflr/record_template.rb", "sample2_out.dat", "spec/hflr_spec.rb", "spec/spec_helper.rb", "test/customer_orders.dat", "test/customers.dat", "test/examples.rb", "test/fixed_length_record_file_test.rb", "test/record_template_test.rb", "test/sample.dat", "test/sample2_out.dat", "test/sample_activities.dat", "test/sample_out.dat", "test/test_helper.rb", "test/test_hflr.rb"]
+  s.homepage = %q{http://ruff.rubyforge.org}
+  s.rdoc_options = ["--main", "README.txt"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{hflr}
+  s.rubygems_version = %q{1.3.4}
+  s.summary = %q{HFLR -- Hierarchical Fixed Length Records  Allows you to read and write files of fixed width records when the file contains one or more than one type of record}
+  s.test_files = ["test/test_hflr.rb", "test/test_helper.rb"]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<bones>, [">= 2.5.1"])
+    else
+      s.add_dependency(%q<bones>, [">= 2.5.1"])
+    end
+  else
+    s.add_dependency(%q<bones>, [">= 2.5.1"])
+  end
+end

data/lib/hflr.rb

@@ -0,0 +1,49 @@
+
+module Hflr
+
+  # :stopdoc:
+  VERSION = '0.10.1'
+  LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
+  PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
+  # :startdoc:
+
+  # Returns the version string for the library.
+  #
+  def self.version
+    VERSION
+  end
+
+  # Returns the library path for the module. If any arguments are given,
+  # they will be joined to the end of the libray path using
+  # <tt>File.join</tt>.
+  #
+  def self.libpath( *args )
+    args.empty? ? LIBPATH : ::File.join(LIBPATH, args.flatten)
+  end
+
+  # Returns the lpath for the module. If any arguments are given,
+  # they will be joined to the end of the path using
+  # <tt>File.join</tt>.
+  #
+  def self.path( *args )
+    args.empty? ? PATH : ::File.join(PATH, args.flatten)
+  end
+
+  # Utility method used to require all files ending in .rb that lie in the
+  # directory below this file that has the same name as the filename passed
+  # in. Optionally, a specific _directory_ name can be passed in such that
+  # the _filename_ does not have to be equivalent to the directory.
+  #
+  def self.require_all_libs_relative_to( fname, dir = nil )
+    dir ||= ::File.basename(fname, '.*')
+    search_me = ::File.expand_path(
+        ::File.join(::File.dirname(fname), dir, '**', '*.rb'))
+
+    Dir.glob(search_me).sort.each {|rb| require rb}
+  end
+
+end  # module Hflr
+
+Hflr.require_all_libs_relative_to(__FILE__)
+
+# EOF

data/lib/hflr/fl_record_file.rb

@@ -0,0 +1,104 @@
+
+
+require File.expand_path(File.dirname(__FILE__) + "/record_template")
+
+
+
+class FixedLengthRecordFile  
+
+  include Enumerable
+  
+  attr_reader :line_number, :record_template
+  
+  def initialize(source, record_types, record_layouts, logical_first_column=0, extra_columns = nil)
+    # Allow record layouts like 
+    # {:type1=>[:var1=>1..5,:var2=>7..8],:type2=>[:var1=>1..1,:var2=>3..4]}
+    # ... todo
+    @line_number = 0
+    @file = source    
+    @record_type_labels=record_types
+    @record_type_symbols = record_types.is_a?(Hash) ? record_types.invert : :none
+    if extra_columns  then
+      @record_template = HFLR::RecordTemplate.create(record_layouts, @record_type_symbols, logical_first_column, extra_columns)   
+    else
+        @record_template = HFLR::RecordTemplate.create(record_layouts, @record_type_symbols, logical_first_column)   
+    end
+        
+  end
+
+def finished?
+  @file.eof?
+end
+
+def close
+  @file.close
+end
+
+# If multiple record types,  extract it from the string, otherwise just return the type of this file
+def get_record_type(line)
+  return nil if line.nil?  
+  return nil if line.strip.empty?
+  @record_type_labels.is_a?(Hash) ? @record_type_labels[line[0..0]] : @record_type_labels       
+end
+
+def  build_record(line)    
+  return nil if line.nil?        
+
+  record_type = line_type(line)
+  raise "Unknown record type at line #{@line_number.to_s}" if record_type == :unknown
+  return @record_template[record_type].build_record(line.chomp)                 
+end
+
+def next_record
+  @line_number += 1
+  build_record(get_next_known_line_type) 
+end
+
+def line_type(line)
+  record_type = get_record_type(line)
+  return record_type ? record_type : :unknown
+end
+
+def get_next_known_line_type
+  line = @file.gets       
+  while line_type(line) == :unknown and (not finished?)  
+    line = @file.gets 
+    end
+return line
+end
+
+def each
+  @file.each_line do |line|    
+    @line_number += 1
+    unless line_type(line) == :unknown       
+      data = build_record(line)
+      yield data 
+    end
+  end 
+end
+
+ # This will take a Hash or Struct orArray;  if an Array the record type must be the last element
+  def <<(record)         
+    if record.is_a? Array
+      @file.puts @record_template[record.last].build_line(record)
+    else
+      if @record_template[record[:record_type]] == nil then
+        raise "Record type problem in output: #{record[:record_type].to_s} type on record, #{@record_template.keys.join(",")} types of templates"
+      end
+      
+      @file.puts @record_template[record[:record_type]].build_line(record)      
+    end
+  end
+
+# Use when creating a new HFLR file
+def self.open(path, mode, record_types, record_layouts, logical_first_column=0)  
+  file = File.open(path, mode)
+  begin
+    hflr_file = new(file, record_types, record_layouts, logical_first_column) 
+    yield hflr_file
+  ensure
+    file.close
+  end
+end
+  
+end

data/lib/hflr/hflr.rb

@@ -0,0 +1,2 @@
+
+require File.expand_path(File.dirname(__FILE__) + "/fl_record_file")

data/lib/hflr/record_template.rb

@@ -0,0 +1,150 @@
+
+module HFLR
+
+class RecordTemplate
+  UnfilledChar = ' '
+  MissingOutput = "ZZZZZZZZZZZZZZZZZZZZZ"
+  
+  attr_reader :record_structure, :field_pattern,:record_type,:record_type_label
+  attr_accessor :strip_whitespace
+    
+  def initialize(record_type, record_type_label, record_structure,field_pattern, field_widths)
+    @record_type = record_type    
+    @record_type_label = record_type_label
+    @record_structure = record_structure
+    @field_pattern = field_pattern
+    @field_widths = field_widths
+  end
+  
+# Layouts is a hash of variables by record type
+# record_type_symbols maps record type names to their labels in the data {:household=>"H",:person=>"P"}
+# Returns a set of record templates, one for each record type  
+def self.create(record_layouts, record_type_symbols, first_column_location, extra_columns=[])    
+   extra_columns = empty_extra_columns(record_layouts.keys) if  extra_columns.is_a? Array    
+  templates = {}
+  self.check_record_layouts(record_layouts)
+  
+  record_layouts.keys.each do |record_type|    
+    record_label = record_type_symbols == :none ? :none : record_type_symbols[record_type]
+    templates[record_type] = 
+    self.create_template_class(record_type,
+      record_label,
+    record_layouts[record_type], 
+    first_column_location, 
+    extra_columns[record_type]) 
+  end
+  return templates  
+end
+
+
+  # If the name exists already do not replace it, but add extra columns not to be mapped by the unpack field patterns
+  # and ensure the record_type variable is added.
+  # Since 'record_type' may not be in the metadata we don't want to map it to a 
+  # specific column location but do want it included always.
+  def self.add_extra_columns(names, extra)
+    new_names = names.dup
+    # names are not case sensitive
+    extra.each{|n|new_names << n unless names.map{|m| m.to_s.upcase}.include? n.to_s.upcase}
+    	    
+    # No matter what, include 'record_type'
+    unless new_names.map{|n| n.to_s.upcase}.include?("RECORD_TYPE")
+      new_names << :record_type
+    end   
+    return new_names
+  end
+  
+  def self.get_pattern(layout, first_column_location=0)
+      
+
+    layout.map {|l| '@' + (l.start - first_column_location).to_s + 'A' + l.len.to_s}.to_s
+  end
+    
+  def build_record(line)    
+    rec = line.unpack(@field_pattern)            
+    rec.map{|f| f.strip!} if @strip_whitespace
+    begin
+      data = self.record_structure.new(*rec)              
+      data[:record_type] = @record_type
+    rescue Exception=>msg
+        raise "On record type #{self.record_type} problem with structure " + msg.to_s
+    end
+    return data
+  end
+    
+  def build_line(record)            
+    line = format_fields(record).pack(@field_pattern)
+    line[0] = @record_type_label unless @record_type_label == :none
+    line.tr!("\0",UnfilledChar)  
+    return line
+  end
+
+private
+
+def self.empty_extra_columns(record_types)
+  extra = {}
+  record_types.map{|rt| extra[rt] = []}
+  extra
+end
+
+# All starting columns must be in order
+def self.check_record_layouts(layouts)
+  layouts.values.each do |layout|
+    last_v = layout.first
+    layout.each do |v|
+      if v.respond_to?(:rectype) then
+      if last_v.rectype != v.rectype
+        raise "record type mismatch between #{v.name} and #{last_v.name}"
+      end
+    end  
+      if last_v.start<= v.start then
+        last_v = v
+      else
+        raise "Problem with start columns #{last_v.name} start #{last_v.start.to_s}  out of sequence with #{v.name} starting at #{v.start.to_s}"
+      end
+    end
+  end
+end
+
+def self.create_template_class(record_type, record_type_label, layout, first_column_location, extra_columns = nil)
+  names = layout.map {|l| l.name.to_sym}
+  names = add_extra_columns(names, extra_columns) 
+  structure = Struct.new(*names)
+  return new(record_type, 
+    record_type_label,
+    structure,
+    self.get_pattern(layout, first_column_location),
+    layout.map{|v| v.len})
+  end
+  
+    
+  def format_fields(record)
+   if record.is_a?(Array) or record.is_a?(Struct) then
+     fields = []    
+     @field_widths.each_with_index do |width, i|  
+    fields << right_format(record[i], width)
+   end
+  return fields
+ else
+   raise "Record to format must be a Struct or Array"
+  end
+end
+           
+  def right_format(data, len)
+    data_str = ""
+    if data.is_a? String
+      data_str = data.ljust(len)
+    elsif data.is_a? Symbol 
+     data_str = data.to_s.ljust(len)
+    else
+     data_str = sprintf("%0#{len.to_s}d",data)
+     data_str = MissingOutput[0..len-1] if data == -999998
+  end
+    raise "Data too large for allocated columns #{data_str}" if data_str.size > len
+    data_str
+  end
+      
+
+end # RecordTemplate class
+
+end # HFLR module
+

data/sample2_out.dat

@@ -0,0 +1 @@
+joe                      025