hflr 0.10.1
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +4 -0
- data/README.txt +45 -0
- data/Rakefile +30 -0
- data/bin/hflr +8 -0
- data/hflr.gemspec +43 -0
- data/lib/hflr.rb +49 -0
- data/lib/hflr/fl_record_file.rb +104 -0
- data/lib/hflr/hflr.rb +2 -0
- data/lib/hflr/record_template.rb +150 -0
- data/sample2_out.dat +1 -0
- data/spec/hflr_spec.rb +7 -0
- data/spec/spec_helper.rb +16 -0
- data/tasks/ann.rake +80 -0
- data/tasks/bones.rake +20 -0
- data/tasks/gem.rake +201 -0
- data/tasks/git.rake +40 -0
- data/tasks/notes.rake +27 -0
- data/tasks/post_load.rake +34 -0
- data/tasks/rdoc.rake +51 -0
- data/tasks/rubyforge.rake +55 -0
- data/tasks/setup.rb +292 -0
- data/tasks/spec.rake +54 -0
- data/tasks/svn.rake +47 -0
- data/tasks/test.rake +40 -0
- data/tasks/zentest.rake +36 -0
- data/test/customer_orders.dat +5 -0
- data/test/customers.dat +2 -0
- data/test/examples.rb +48 -0
- data/test/fixed_length_record_file_test.rb +212 -0
- data/test/record_template_test.rb +174 -0
- data/test/sample.dat +6 -0
- data/test/sample2_out.dat +1 -0
- data/test/sample_activities.dat +12 -0
- data/test/sample_out.dat +2 -0
- data/test/test_helper.rb +3 -0
- data/test/test_hflr.rb +4 -0
- metadata +111 -0
data/History.txt
ADDED
data/README.txt
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
= HFLR
|
2
|
+
|
3
|
+
* http://ruff.rubyforge.org
|
4
|
+
|
5
|
+
== Description:
|
6
|
+
|
7
|
+
HFLR -- Hierarchical Fixed Length Records
|
8
|
+
|
9
|
+
Allows you to read and write files of fixed width records when the file contains one or more
|
10
|
+
than one type of record.
|
11
|
+
|
12
|
+
Install with 'gem install hflr'
|
13
|
+
|
14
|
+
|
15
|
+
|
16
|
+
|
17
|
+
See the tests and examples bundled with this gem.
|
18
|
+
|
19
|
+
|
20
|
+
|
21
|
+
|
22
|
+
== LICENSE:
|
23
|
+
|
24
|
+
(The MIT License)
|
25
|
+
|
26
|
+
Copyright (c) 2009 Colin C. Davis
|
27
|
+
|
28
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
29
|
+
a copy of this software and associated documentation files (the
|
30
|
+
'Software'), to deal in the Software without restriction, including
|
31
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
32
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
33
|
+
permit persons to whom the Software is furnished to do so, subject to
|
34
|
+
the following conditions:
|
35
|
+
|
36
|
+
The above copyright notice and this permission notice shall be
|
37
|
+
included in all copies or substantial portions of the Software.
|
38
|
+
|
39
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
40
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
41
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
42
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
43
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
44
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
45
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# Look in the tasks/setup.rb file for the various options that can be
|
2
|
+
# configured in this Rakefile. The .rake files in the tasks directory
|
3
|
+
# are where the options are used.
|
4
|
+
|
5
|
+
begin
|
6
|
+
require 'bones'
|
7
|
+
Bones.setup
|
8
|
+
rescue LoadError
|
9
|
+
begin
|
10
|
+
load 'tasks/setup.rb'
|
11
|
+
rescue LoadError
|
12
|
+
raise RuntimeError, '### please install the "bones" gem ###'
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
ensure_in_path 'lib'
|
17
|
+
require 'hflr'
|
18
|
+
|
19
|
+
task :default => 'spec:run'
|
20
|
+
|
21
|
+
PROJ.name = 'hflr'
|
22
|
+
PROJ.authors = 'Colin Davis'
|
23
|
+
PROJ.email = 'colin.c.davis@gmail.com'
|
24
|
+
PROJ.url = 'http://ruff.rubyforge.org'
|
25
|
+
PROJ.version = Hflr::VERSION
|
26
|
+
PROJ.rubyforge.name = 'hflr'
|
27
|
+
|
28
|
+
PROJ.spec.opts << '--color'
|
29
|
+
|
30
|
+
# EOF
|
data/bin/hflr
ADDED
data/hflr.gemspec
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = %q{hflr}
|
5
|
+
s.version = "1.0.0"
|
6
|
+
|
7
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
|
+
s.authors = ["Colin Davis"]
|
9
|
+
s.date = %q{2009-08-03}
|
10
|
+
s.default_executable = %q{hflr}
|
11
|
+
s.description = %q{HFLR -- Hierarchical Fixed Length Records
|
12
|
+
|
13
|
+
Allows you to read and write files of fixed width records when the file contains one or more
|
14
|
+
than one type of record.
|
15
|
+
|
16
|
+
Install with 'gem install hflr'
|
17
|
+
|
18
|
+
See the tests and examples bundled with this gem.}
|
19
|
+
s.email = %q{colin.c.davis@gmail.com}
|
20
|
+
s.executables = ["hflr"]
|
21
|
+
s.extra_rdoc_files = ["History.txt", "README.txt", "bin/hflr"]
|
22
|
+
s.files = ["History.txt", "README.txt", "Rakefile", "bin/hflr", "lib/hflr.rb", "lib/hflr/fl_record_file.rb", "lib/hflr/hflr.rb", "lib/hflr/record_template.rb", "sample2_out.dat", "spec/hflr_spec.rb", "spec/spec_helper.rb", "test/customer_orders.dat", "test/customers.dat", "test/examples.rb", "test/fixed_length_record_file_test.rb", "test/record_template_test.rb", "test/sample.dat", "test/sample2_out.dat", "test/sample_activities.dat", "test/sample_out.dat", "test/test_helper.rb", "test/test_hflr.rb"]
|
23
|
+
s.homepage = %q{http://ruff.rubyforge.org}
|
24
|
+
s.rdoc_options = ["--main", "README.txt"]
|
25
|
+
s.require_paths = ["lib"]
|
26
|
+
s.rubyforge_project = %q{hflr}
|
27
|
+
s.rubygems_version = %q{1.3.4}
|
28
|
+
s.summary = %q{HFLR -- Hierarchical Fixed Length Records Allows you to read and write files of fixed width records when the file contains one or more than one type of record}
|
29
|
+
s.test_files = ["test/test_hflr.rb", "test/test_helper.rb"]
|
30
|
+
|
31
|
+
if s.respond_to? :specification_version then
|
32
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
33
|
+
s.specification_version = 3
|
34
|
+
|
35
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
36
|
+
s.add_development_dependency(%q<bones>, [">= 2.5.1"])
|
37
|
+
else
|
38
|
+
s.add_dependency(%q<bones>, [">= 2.5.1"])
|
39
|
+
end
|
40
|
+
else
|
41
|
+
s.add_dependency(%q<bones>, [">= 2.5.1"])
|
42
|
+
end
|
43
|
+
end
|
data/lib/hflr.rb
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
|
2
|
+
module Hflr
|
3
|
+
|
4
|
+
# :stopdoc:
|
5
|
+
VERSION = '0.10.1'
|
6
|
+
LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
|
7
|
+
PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
|
8
|
+
# :startdoc:
|
9
|
+
|
10
|
+
# Returns the version string for the library.
|
11
|
+
#
|
12
|
+
def self.version
|
13
|
+
VERSION
|
14
|
+
end
|
15
|
+
|
16
|
+
# Returns the library path for the module. If any arguments are given,
|
17
|
+
# they will be joined to the end of the libray path using
|
18
|
+
# <tt>File.join</tt>.
|
19
|
+
#
|
20
|
+
def self.libpath( *args )
|
21
|
+
args.empty? ? LIBPATH : ::File.join(LIBPATH, args.flatten)
|
22
|
+
end
|
23
|
+
|
24
|
+
# Returns the lpath for the module. If any arguments are given,
|
25
|
+
# they will be joined to the end of the path using
|
26
|
+
# <tt>File.join</tt>.
|
27
|
+
#
|
28
|
+
def self.path( *args )
|
29
|
+
args.empty? ? PATH : ::File.join(PATH, args.flatten)
|
30
|
+
end
|
31
|
+
|
32
|
+
# Utility method used to require all files ending in .rb that lie in the
|
33
|
+
# directory below this file that has the same name as the filename passed
|
34
|
+
# in. Optionally, a specific _directory_ name can be passed in such that
|
35
|
+
# the _filename_ does not have to be equivalent to the directory.
|
36
|
+
#
|
37
|
+
def self.require_all_libs_relative_to( fname, dir = nil )
|
38
|
+
dir ||= ::File.basename(fname, '.*')
|
39
|
+
search_me = ::File.expand_path(
|
40
|
+
::File.join(::File.dirname(fname), dir, '**', '*.rb'))
|
41
|
+
|
42
|
+
Dir.glob(search_me).sort.each {|rb| require rb}
|
43
|
+
end
|
44
|
+
|
45
|
+
end # module Hflr
|
46
|
+
|
47
|
+
Hflr.require_all_libs_relative_to(__FILE__)
|
48
|
+
|
49
|
+
# EOF
|
@@ -0,0 +1,104 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
require File.expand_path(File.dirname(__FILE__) + "/record_template")
|
4
|
+
|
5
|
+
|
6
|
+
|
7
|
+
class FixedLengthRecordFile
|
8
|
+
|
9
|
+
include Enumerable
|
10
|
+
|
11
|
+
attr_reader :line_number, :record_template
|
12
|
+
|
13
|
+
def initialize(source, record_types, record_layouts, logical_first_column=0, extra_columns = nil)
|
14
|
+
# Allow record layouts like
|
15
|
+
# {:type1=>[:var1=>1..5,:var2=>7..8],:type2=>[:var1=>1..1,:var2=>3..4]}
|
16
|
+
# ... todo
|
17
|
+
@line_number = 0
|
18
|
+
@file = source
|
19
|
+
@record_type_labels=record_types
|
20
|
+
@record_type_symbols = record_types.is_a?(Hash) ? record_types.invert : :none
|
21
|
+
if extra_columns then
|
22
|
+
@record_template = HFLR::RecordTemplate.create(record_layouts, @record_type_symbols, logical_first_column, extra_columns)
|
23
|
+
else
|
24
|
+
@record_template = HFLR::RecordTemplate.create(record_layouts, @record_type_symbols, logical_first_column)
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
def finished?
|
30
|
+
@file.eof?
|
31
|
+
end
|
32
|
+
|
33
|
+
def close
|
34
|
+
@file.close
|
35
|
+
end
|
36
|
+
|
37
|
+
# If multiple record types, extract it from the string, otherwise just return the type of this file
|
38
|
+
def get_record_type(line)
|
39
|
+
return nil if line.nil?
|
40
|
+
return nil if line.strip.empty?
|
41
|
+
@record_type_labels.is_a?(Hash) ? @record_type_labels[line[0..0]] : @record_type_labels
|
42
|
+
end
|
43
|
+
|
44
|
+
def build_record(line)
|
45
|
+
return nil if line.nil?
|
46
|
+
|
47
|
+
record_type = line_type(line)
|
48
|
+
raise "Unknown record type at line #{@line_number.to_s}" if record_type == :unknown
|
49
|
+
return @record_template[record_type].build_record(line.chomp)
|
50
|
+
end
|
51
|
+
|
52
|
+
def next_record
|
53
|
+
@line_number += 1
|
54
|
+
build_record(get_next_known_line_type)
|
55
|
+
end
|
56
|
+
|
57
|
+
def line_type(line)
|
58
|
+
record_type = get_record_type(line)
|
59
|
+
return record_type ? record_type : :unknown
|
60
|
+
end
|
61
|
+
|
62
|
+
def get_next_known_line_type
|
63
|
+
line = @file.gets
|
64
|
+
while line_type(line) == :unknown and (not finished?)
|
65
|
+
line = @file.gets
|
66
|
+
end
|
67
|
+
return line
|
68
|
+
end
|
69
|
+
|
70
|
+
def each
|
71
|
+
@file.each_line do |line|
|
72
|
+
@line_number += 1
|
73
|
+
unless line_type(line) == :unknown
|
74
|
+
data = build_record(line)
|
75
|
+
yield data
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
# This will take a Hash or Struct orArray; if an Array the record type must be the last element
|
81
|
+
def <<(record)
|
82
|
+
if record.is_a? Array
|
83
|
+
@file.puts @record_template[record.last].build_line(record)
|
84
|
+
else
|
85
|
+
if @record_template[record[:record_type]] == nil then
|
86
|
+
raise "Record type problem in output: #{record[:record_type].to_s} type on record, #{@record_template.keys.join(",")} types of templates"
|
87
|
+
end
|
88
|
+
|
89
|
+
@file.puts @record_template[record[:record_type]].build_line(record)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
# Use when creating a new HFLR file
|
94
|
+
def self.open(path, mode, record_types, record_layouts, logical_first_column=0)
|
95
|
+
file = File.open(path, mode)
|
96
|
+
begin
|
97
|
+
hflr_file = new(file, record_types, record_layouts, logical_first_column)
|
98
|
+
yield hflr_file
|
99
|
+
ensure
|
100
|
+
file.close
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
end
|
data/lib/hflr/hflr.rb
ADDED
@@ -0,0 +1,150 @@
|
|
1
|
+
|
2
|
+
module HFLR
|
3
|
+
|
4
|
+
class RecordTemplate
|
5
|
+
UnfilledChar = ' '
|
6
|
+
MissingOutput = "ZZZZZZZZZZZZZZZZZZZZZ"
|
7
|
+
|
8
|
+
attr_reader :record_structure, :field_pattern,:record_type,:record_type_label
|
9
|
+
attr_accessor :strip_whitespace
|
10
|
+
|
11
|
+
def initialize(record_type, record_type_label, record_structure,field_pattern, field_widths)
|
12
|
+
@record_type = record_type
|
13
|
+
@record_type_label = record_type_label
|
14
|
+
@record_structure = record_structure
|
15
|
+
@field_pattern = field_pattern
|
16
|
+
@field_widths = field_widths
|
17
|
+
end
|
18
|
+
|
19
|
+
# Layouts is a hash of variables by record type
|
20
|
+
# record_type_symbols maps record type names to their labels in the data {:household=>"H",:person=>"P"}
|
21
|
+
# Returns a set of record templates, one for each record type
|
22
|
+
def self.create(record_layouts, record_type_symbols, first_column_location, extra_columns=[])
|
23
|
+
extra_columns = empty_extra_columns(record_layouts.keys) if extra_columns.is_a? Array
|
24
|
+
templates = {}
|
25
|
+
self.check_record_layouts(record_layouts)
|
26
|
+
|
27
|
+
record_layouts.keys.each do |record_type|
|
28
|
+
record_label = record_type_symbols == :none ? :none : record_type_symbols[record_type]
|
29
|
+
templates[record_type] =
|
30
|
+
self.create_template_class(record_type,
|
31
|
+
record_label,
|
32
|
+
record_layouts[record_type],
|
33
|
+
first_column_location,
|
34
|
+
extra_columns[record_type])
|
35
|
+
end
|
36
|
+
return templates
|
37
|
+
end
|
38
|
+
|
39
|
+
|
40
|
+
# If the name exists already do not replace it, but add extra columns not to be mapped by the unpack field patterns
|
41
|
+
# and ensure the record_type variable is added.
|
42
|
+
# Since 'record_type' may not be in the metadata we don't want to map it to a
|
43
|
+
# specific column location but do want it included always.
|
44
|
+
def self.add_extra_columns(names, extra)
|
45
|
+
new_names = names.dup
|
46
|
+
# names are not case sensitive
|
47
|
+
extra.each{|n|new_names << n unless names.map{|m| m.to_s.upcase}.include? n.to_s.upcase}
|
48
|
+
|
49
|
+
# No matter what, include 'record_type'
|
50
|
+
unless new_names.map{|n| n.to_s.upcase}.include?("RECORD_TYPE")
|
51
|
+
new_names << :record_type
|
52
|
+
end
|
53
|
+
return new_names
|
54
|
+
end
|
55
|
+
|
56
|
+
def self.get_pattern(layout, first_column_location=0)
|
57
|
+
|
58
|
+
|
59
|
+
layout.map {|l| '@' + (l.start - first_column_location).to_s + 'A' + l.len.to_s}.to_s
|
60
|
+
end
|
61
|
+
|
62
|
+
def build_record(line)
|
63
|
+
rec = line.unpack(@field_pattern)
|
64
|
+
rec.map{|f| f.strip!} if @strip_whitespace
|
65
|
+
begin
|
66
|
+
data = self.record_structure.new(*rec)
|
67
|
+
data[:record_type] = @record_type
|
68
|
+
rescue Exception=>msg
|
69
|
+
raise "On record type #{self.record_type} problem with structure " + msg.to_s
|
70
|
+
end
|
71
|
+
return data
|
72
|
+
end
|
73
|
+
|
74
|
+
def build_line(record)
|
75
|
+
line = format_fields(record).pack(@field_pattern)
|
76
|
+
line[0] = @record_type_label unless @record_type_label == :none
|
77
|
+
line.tr!("\0",UnfilledChar)
|
78
|
+
return line
|
79
|
+
end
|
80
|
+
|
81
|
+
private
|
82
|
+
|
83
|
+
def self.empty_extra_columns(record_types)
|
84
|
+
extra = {}
|
85
|
+
record_types.map{|rt| extra[rt] = []}
|
86
|
+
extra
|
87
|
+
end
|
88
|
+
|
89
|
+
# All starting columns must be in order
|
90
|
+
def self.check_record_layouts(layouts)
|
91
|
+
layouts.values.each do |layout|
|
92
|
+
last_v = layout.first
|
93
|
+
layout.each do |v|
|
94
|
+
if v.respond_to?(:rectype) then
|
95
|
+
if last_v.rectype != v.rectype
|
96
|
+
raise "record type mismatch between #{v.name} and #{last_v.name}"
|
97
|
+
end
|
98
|
+
end
|
99
|
+
if last_v.start<= v.start then
|
100
|
+
last_v = v
|
101
|
+
else
|
102
|
+
raise "Problem with start columns #{last_v.name} start #{last_v.start.to_s} out of sequence with #{v.name} starting at #{v.start.to_s}"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def self.create_template_class(record_type, record_type_label, layout, first_column_location, extra_columns = nil)
|
109
|
+
names = layout.map {|l| l.name.to_sym}
|
110
|
+
names = add_extra_columns(names, extra_columns)
|
111
|
+
structure = Struct.new(*names)
|
112
|
+
return new(record_type,
|
113
|
+
record_type_label,
|
114
|
+
structure,
|
115
|
+
self.get_pattern(layout, first_column_location),
|
116
|
+
layout.map{|v| v.len})
|
117
|
+
end
|
118
|
+
|
119
|
+
|
120
|
+
def format_fields(record)
|
121
|
+
if record.is_a?(Array) or record.is_a?(Struct) then
|
122
|
+
fields = []
|
123
|
+
@field_widths.each_with_index do |width, i|
|
124
|
+
fields << right_format(record[i], width)
|
125
|
+
end
|
126
|
+
return fields
|
127
|
+
else
|
128
|
+
raise "Record to format must be a Struct or Array"
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def right_format(data, len)
|
133
|
+
data_str = ""
|
134
|
+
if data.is_a? String
|
135
|
+
data_str = data.ljust(len)
|
136
|
+
elsif data.is_a? Symbol
|
137
|
+
data_str = data.to_s.ljust(len)
|
138
|
+
else
|
139
|
+
data_str = sprintf("%0#{len.to_s}d",data)
|
140
|
+
data_str = MissingOutput[0..len-1] if data == -999998
|
141
|
+
end
|
142
|
+
raise "Data too large for allocated columns #{data_str}" if data_str.size > len
|
143
|
+
data_str
|
144
|
+
end
|
145
|
+
|
146
|
+
|
147
|
+
end # RecordTemplate class
|
148
|
+
|
149
|
+
end # HFLR module
|
150
|
+
|
data/sample2_out.dat
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
joe 025
|