flat_filer 0.0.18 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/flat_file.rb +203 -410
- data/lib/flat_file/field_def.rb +116 -0
- data/lib/flat_file/record.rb +97 -0
- data/lib/flat_filer.rb +7 -0
- metadata +66 -34
- data/lib/core_extensions.rb +0 -20
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: aea5aa325cee0fd5897274b7783bd5d13c22390f
|
4
|
+
data.tar.gz: 11f6771b6caf2522a0955d4a938e9b48bc5416bb
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ee9378907c97fc2c5925581d65d8b03373f45bb2ff8f7491a99a5059bd93259b90a44047addd0b32bca9787cbdd71bb45993f0b421ec5f1d3e34043778c52a25
|
7
|
+
data.tar.gz: a378695c4b64871187d04ffac73c71e90ab2958599604b456def05864c172b4e777a85140134ac310c9fe1cc377588aa60ad8f163be6de820dd2aa9b05b7477b
|
data/lib/flat_file.rb
CHANGED
@@ -25,13 +25,13 @@
|
|
25
25
|
# puts "First Name: #{ person.first_name }"
|
26
26
|
# puts "Last Name : #{ person.last_name}"
|
27
27
|
# puts "Birthday : #{ person.birthday}"
|
28
|
-
#
|
28
|
+
#
|
29
29
|
# puts person.to_s
|
30
30
|
# end
|
31
|
-
#
|
32
|
-
#
|
33
31
|
#
|
34
|
-
#
|
32
|
+
#
|
33
|
+
#
|
34
|
+
# An alternative method for adding fields is to pass a block to the
|
35
35
|
# add_field method. The name is optional, but needs to be set either
|
36
36
|
# by passing the name parameter, or in the block that's passed. When
|
37
37
|
# you pass a block the first parameter is the FieldDef for the field
|
@@ -45,7 +45,7 @@
|
|
45
45
|
# fd.add_formatter { |v| v.trim }
|
46
46
|
# .
|
47
47
|
# .
|
48
|
-
# }
|
48
|
+
# }
|
49
49
|
# end
|
50
50
|
#
|
51
51
|
# Filters and Formatters
|
@@ -59,10 +59,10 @@
|
|
59
59
|
# Structurally, filters and formatters can be lambdas, code
|
60
60
|
# blocks, or symbols referencing methods.
|
61
61
|
#
|
62
|
-
# There's an expectaiton on the part of formatters of the
|
62
|
+
# There's an expectaiton on the part of formatters of the
|
63
63
|
# type of a field value. This means that the programmer
|
64
64
|
# needs to set the value of a field as a type that the formatter
|
65
|
-
# won't bork on.
|
65
|
+
# won't bork on.
|
66
66
|
#
|
67
67
|
# A good argument can be made to change filtering to happen any
|
68
68
|
# time a field value is assigned. I've decided to not take this
|
@@ -102,432 +102,225 @@
|
|
102
102
|
#
|
103
103
|
# add_field also adds to a variable that olds a pack format. This is
|
104
104
|
# how the records parsed and assembeled.
|
105
|
-
require File.dirname(__FILE__) + "/../lib/core_extensions"
|
106
|
-
class FlatFile
|
107
105
|
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
# :width, specify the field width, :formatter, specify a formatter, :filter, specify a
|
138
|
-
# filter.
|
139
|
-
def initialize(name=null,options={},klass={})
|
140
|
-
@name = name
|
141
|
-
@width = 10
|
142
|
-
@filters = Array.new
|
143
|
-
@formatters = Array.new
|
144
|
-
@file_klass = klass
|
145
|
-
@padding = options.delete(:padding)
|
146
|
-
@default = options.has_key?(:default) ? options.delete(:default) : ""
|
147
|
-
|
148
|
-
add_filter(options[:filter]) if options.has_key?(:filter)
|
149
|
-
add_formatter(options[:formatter]) if options.has_key?(:formatter)
|
150
|
-
@map_in_proc = options[:map_in_proc]
|
151
|
-
@width = options[:width] if options.has_key?(:width)
|
152
|
-
@aggressive = options[:aggressive] || false
|
153
|
-
end
|
154
|
-
|
155
|
-
# Will return true if the field is a padding field. Padding fields are ignored
|
156
|
-
# when doing various things. For example, when you're populating an ActiveRecord
|
157
|
-
# model with a record, padding fields are ignored.
|
158
|
-
def is_padding?
|
159
|
-
@padding
|
160
|
-
end
|
161
|
-
|
162
|
-
# Add a filter. Filters are used for processing field data when a flat file is being
|
163
|
-
# processed. For fomratting the data when writing a flat file, see add_formatter
|
164
|
-
def add_filter(filter=nil,&block) #:nodoc:
|
165
|
-
@filters.push(filter) unless filter.nil?
|
166
|
-
@filters.push(block) if block_given?
|
167
|
-
end
|
168
|
-
|
169
|
-
# Add a formatter. Formatters are used for formatting a field
|
170
|
-
# for rendering a record, or writing it to a file in the desired format.
|
171
|
-
def add_formatter(formatter=nil,&block) #:nodoc:
|
172
|
-
@formatters.push(formatter) if formatter
|
173
|
-
@formatters.push(block) if block_given?
|
174
|
-
end
|
175
|
-
|
176
|
-
# Filters a value based on teh filters associated with a
|
177
|
-
# FieldDef.
|
178
|
-
def pass_through_filters(v) #:nodoc:
|
179
|
-
pass_through(@filters,v)
|
180
|
-
end
|
181
|
-
|
182
|
-
# Filters a value based on the filters associated with a
|
183
|
-
# FieldDef.
|
184
|
-
def pass_through_formatters(v) #:nodoc:
|
185
|
-
pass_through(@formatters,v)
|
186
|
-
end
|
187
|
-
|
188
|
-
#protected
|
189
|
-
|
190
|
-
def pass_through(what,value) #:nodoc:
|
191
|
-
#puts "PASS THROUGH #{what.inspect} => #{value}"
|
192
|
-
what.each do |filter|
|
193
|
-
value = case
|
194
|
-
when filter.is_a?(Symbol)
|
195
|
-
#puts "filter is a symbol"
|
196
|
-
@file_klass.send(filter,value)
|
197
|
-
when filter_block?(filter)
|
198
|
-
#puts "filter is a block"
|
199
|
-
filter.call(value)
|
200
|
-
when filter_class?(filter)
|
201
|
-
#puts "filter is a class"
|
202
|
-
filter.filter(value)
|
203
|
-
else
|
204
|
-
#puts "filter not valid, preserving"
|
205
|
-
value
|
206
|
-
end
|
207
|
-
end
|
208
|
-
value
|
209
|
-
end
|
210
|
-
|
211
|
-
# Test to see if filter is a filter block. A filter block
|
212
|
-
# can be called (using call) and takes one parameter
|
213
|
-
def filter_block?(filter) #:nodoc:
|
214
|
-
filter.respond_to?('call') && ( filter.arity >= 1 || filter.arity <= -1 )
|
215
|
-
end
|
216
|
-
|
217
|
-
# Test to see if a class is a filter class. A filter class responds
|
218
|
-
# to the filter signal (you can call filter on it).
|
219
|
-
def filter_class?(filter) #:nodoc:
|
220
|
-
filter.respond_to?('filter')
|
221
|
-
end
|
106
|
+
class FlatFile
|
107
|
+
class FlatFileException < Exception; end
|
108
|
+
class ShortRecordError < FlatFileException; end
|
109
|
+
class LongRecordError < FlatFileException; end
|
110
|
+
class RecordLengthError < FlatFileException; end
|
111
|
+
|
112
|
+
|
113
|
+
# A hash of data stored on behalf of subclasses. One hash
|
114
|
+
# key for each subclass.
|
115
|
+
@@subclass_data = Hash.new(nil)
|
116
|
+
|
117
|
+
# Used to generate unique names for pad fields which use :auto_name.
|
118
|
+
@@unique_id = 0
|
119
|
+
|
120
|
+
def next_record(io,&block)
|
121
|
+
return nil if io.eof?
|
122
|
+
required_line_length = self.class.get_subclass_variable 'width'
|
123
|
+
line = io.readline
|
124
|
+
line.chop!
|
125
|
+
return nil if line.length == 0
|
126
|
+
difference = required_line_length - line.length
|
127
|
+
raise RecordLengthError.new(
|
128
|
+
"length is #{line.length} but should be #{required_line_length}"
|
129
|
+
) unless(difference == 0)
|
130
|
+
|
131
|
+
if block_given?
|
132
|
+
yield(create_record(line, io.lineno), line)
|
133
|
+
else
|
134
|
+
create_record(line,io.lineno)
|
222
135
|
end
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
klass_fields = klass.get_subclass_variable('fields')
|
246
|
-
|
247
|
-
klass_fields.each do |f|
|
248
|
-
@fields.store(f.name, "")
|
249
|
-
end
|
250
|
-
|
251
|
-
@fields.merge!(fields)
|
252
|
-
|
253
|
-
@fields.each_key do |k|
|
254
|
-
@fields.delete(k) unless klass.has_field?(k)
|
255
|
-
end
|
256
|
-
|
257
|
-
yield(block, self)if block_given?
|
258
|
-
|
259
|
-
self
|
260
|
-
end
|
261
|
-
|
262
|
-
def map_in(model)
|
263
|
-
@klass.non_pad_fields.each do |f|
|
264
|
-
next unless(model.respond_to? "#{f.name}=")
|
265
|
-
if f.map_in_proc
|
266
|
-
f.map_in_proc.call(model,self)
|
267
|
-
else
|
268
|
-
model.send("#{f.name}=", send(f.name)) if f.aggressive or model.send(f.name).blank?
|
269
|
-
model.send("#{f.name}=", f.default) if model.send(f.name).blank?
|
270
|
-
end
|
271
|
-
end
|
272
|
-
end
|
273
|
-
|
274
|
-
# Catches method calls and returns field values or raises an Error.
|
275
|
-
def method_missing(method,params=nil)
|
276
|
-
if(method.to_s.match(/^(.*)=$/))
|
277
|
-
if(fields.has_key?($1.to_sym))
|
278
|
-
@fields.store($1.to_sym,params)
|
279
|
-
else
|
280
|
-
raise Exception.new("Unknown method: #{ method }")
|
281
|
-
end
|
282
|
-
else
|
283
|
-
if(fields.has_key? method)
|
284
|
-
@fields.fetch(method)
|
285
|
-
else
|
286
|
-
raise Exception.new("Unknown method: #{ method }")
|
287
|
-
end
|
288
|
-
end
|
289
|
-
end
|
290
|
-
|
291
|
-
# Returns a string representation of the record suitable for writing to a flat
|
292
|
-
# file on disk or other media. The fields are parepared according to the file
|
293
|
-
# definition, and any formatters attached to the field definitions.
|
294
|
-
def to_s
|
295
|
-
klass.fields.map { |field_def|
|
296
|
-
field_name = field_def.name.to_s
|
297
|
-
v = @fields[ field_name.to_sym ].to_s
|
298
|
-
|
299
|
-
field_def.pass_through_formatters(
|
300
|
-
field_def.is_padding? ? "" : v
|
301
|
-
)
|
302
|
-
}.pack(klass.pack_format)
|
303
|
-
end
|
304
|
-
|
305
|
-
# Produces a multiline string, one field per line suitable for debugging purposes.
|
306
|
-
def debug_string
|
307
|
-
str = ""
|
308
|
-
klass.fields.each do |f|
|
309
|
-
if f.is_padding?
|
310
|
-
str << "#{f.name}: \n"
|
311
|
-
else
|
312
|
-
str << "#{f.name}: #{send(f.name.to_sym)}\n"
|
313
|
-
end
|
314
|
-
end
|
315
|
-
|
316
|
-
str
|
317
|
-
end
|
136
|
+
end
|
137
|
+
|
138
|
+
# Iterate through each record (each line of the data file). The passed
|
139
|
+
# block is passed a new Record representing the line.
|
140
|
+
#
|
141
|
+
# s = SomeFile.new
|
142
|
+
# s.each_record(open('/path/to/file')) do |r|
|
143
|
+
# puts r.first_name
|
144
|
+
# end
|
145
|
+
#
|
146
|
+
def each_record(io,&block)
|
147
|
+
io.each_line do |line|
|
148
|
+
required_line_length = self.class.get_subclass_variable 'width'
|
149
|
+
#line = io.readline
|
150
|
+
line.chop!
|
151
|
+
next if line.length == 0
|
152
|
+
difference = required_line_length - line.length
|
153
|
+
raise RecordLengthError.new(
|
154
|
+
"length is #{line.length} but should be #{required_line_length}"
|
155
|
+
) unless(difference == 0)
|
156
|
+
yield(create_record(line, io.lineno), line)
|
318
157
|
end
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
158
|
+
end
|
159
|
+
|
160
|
+
# create a record from line. The line is one line (or record) read from the
|
161
|
+
# text file. The resulting record is an object which. The object takes signals
|
162
|
+
# for each field according to the various fields defined with add_field or
|
163
|
+
# varients of it.
|
164
|
+
#
|
165
|
+
# line_number is an optional line number of the line in a file of records.
|
166
|
+
# If line is not in a series of records (lines), omit and it'll be -1 in the
|
167
|
+
# resulting record objects. Just make sure you realize this when reporting
|
168
|
+
# errors.
|
169
|
+
#
|
170
|
+
# Both a getter (field_name), and setter (field_name=) are available to the
|
171
|
+
# user.
|
172
|
+
def create_record(line, line_number = -1) #:nodoc:
|
173
|
+
h = Hash.new
|
174
|
+
|
175
|
+
pack_format = self.class.get_subclass_variable 'pack_format'
|
176
|
+
fields = self.class.get_subclass_variable 'fields'
|
177
|
+
|
178
|
+
f = line.unpack(pack_format)
|
179
|
+
(0..(fields.size-1)).map do |index|
|
180
|
+
unless fields[index].is_padding?
|
181
|
+
h.store fields[index].name, fields[index].pass_through_filters(f[index])
|
182
|
+
end
|
343
183
|
end
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
184
|
+
Record.new(self.class, h, line_number)
|
185
|
+
end
|
186
|
+
|
187
|
+
# Add a field to the FlatFile subclass. Options can include
|
188
|
+
#
|
189
|
+
# :width - number of characters in field (default 10)
|
190
|
+
# :filter - callack, lambda or code block for processing during reading
|
191
|
+
# :formatter - callback, lambda, or code block for processing during writing
|
192
|
+
#
|
193
|
+
# class SomeFile < FlatFile
|
194
|
+
# add_field :some_field_name, :width => 35
|
195
|
+
# end
|
196
|
+
#
|
197
|
+
def self.add_field(name=nil, options={},&block)
|
198
|
+
options[:width] ||= 10;
|
199
|
+
|
200
|
+
fields = get_subclass_variable 'fields'
|
201
|
+
width = get_subclass_variable 'width'
|
202
|
+
pack_format = get_subclass_variable 'pack_format'
|
203
|
+
|
204
|
+
|
205
|
+
fd = FieldDef.new(name,options,self)
|
206
|
+
yield(fd) if block_given?
|
207
|
+
|
208
|
+
fields << fd
|
209
|
+
width += fd.width
|
210
|
+
pack_format << "A#{fd.width}"
|
211
|
+
set_subclass_variable 'width', width
|
212
|
+
fd
|
213
|
+
end
|
214
|
+
|
215
|
+
# Add a pad field. To have the name auto generated, use :auto_name for
|
216
|
+
# the name parameter. For options see add_field.
|
217
|
+
def self.pad(name, options = {})
|
218
|
+
fd = self.add_field(
|
219
|
+
name.eql?(:auto_name) ? self.new_pad_name : name,
|
220
|
+
options
|
221
|
+
)
|
222
|
+
fd.padding = true
|
223
|
+
end
|
224
|
+
|
225
|
+
def self.new_pad_name #:nodoc:
|
226
|
+
"pad_#{ @@unique_id+=1 }".to_sym
|
227
|
+
end
|
228
|
+
|
229
|
+
|
230
|
+
# Create a new empty record object conforming to this file.
|
231
|
+
#
|
232
|
+
#
|
233
|
+
def self.new_record(model = nil, &block)
|
234
|
+
fields = get_subclass_variable 'fields'
|
235
|
+
|
236
|
+
record = Record.new(self)
|
237
|
+
|
238
|
+
fields.map do |f|
|
239
|
+
assign_method = "#{f.name}="
|
240
|
+
value = model.respond_to?(f.name.to_sym) ? model.send(f.name.to_sym) : ""
|
241
|
+
record.send(assign_method, value)
|
365
242
|
end
|
366
243
|
|
367
|
-
|
368
|
-
|
369
|
-
# for each field according to the various fields defined with add_field or
|
370
|
-
# varients of it.
|
371
|
-
#
|
372
|
-
# line_number is an optional line number of the line in a file of records.
|
373
|
-
# If line is not in a series of records (lines), omit and it'll be -1 in the
|
374
|
-
# resulting record objects. Just make sure you realize this when reporting
|
375
|
-
# errors.
|
376
|
-
#
|
377
|
-
# Both a getter (field_name), and setter (field_name=) are available to the
|
378
|
-
# user.
|
379
|
-
def create_record(line, line_number = -1) #:nodoc:
|
380
|
-
h = Hash.new
|
381
|
-
|
382
|
-
pack_format = self.class.get_subclass_variable 'pack_format'
|
383
|
-
fields = self.class.get_subclass_variable 'fields'
|
384
|
-
|
385
|
-
f = line.unpack(pack_format)
|
386
|
-
(0..(fields.size-1)).map do |index|
|
387
|
-
unless fields[index].is_padding?
|
388
|
-
h.store fields[index].name, fields[index].pass_through_filters(f[index])
|
389
|
-
end
|
390
|
-
end
|
391
|
-
Record.new(self.class, h, line_number)
|
392
|
-
end
|
393
|
-
|
394
|
-
# Add a field to the FlatFile subclass. Options can include
|
395
|
-
#
|
396
|
-
# :width - number of characters in field (default 10)
|
397
|
-
# :filter - callack, lambda or code block for processing during reading
|
398
|
-
# :formatter - callback, lambda, or code block for processing during writing
|
399
|
-
#
|
400
|
-
# class SomeFile < FlatFile
|
401
|
-
# add_field :some_field_name, :width => 35
|
402
|
-
# end
|
403
|
-
#
|
404
|
-
def self.add_field(name=nil, options={},&block)
|
405
|
-
options[:width] ||= 10;
|
406
|
-
|
407
|
-
fields = get_subclass_variable 'fields'
|
408
|
-
width = get_subclass_variable 'width'
|
409
|
-
pack_format = get_subclass_variable 'pack_format'
|
410
|
-
|
411
|
-
|
412
|
-
fd = FieldDef.new(name,options,self)
|
413
|
-
yield(fd) if block_given?
|
414
|
-
|
415
|
-
fields << fd
|
416
|
-
width += fd.width
|
417
|
-
pack_format << "A#{fd.width}"
|
418
|
-
set_subclass_variable 'width', width
|
419
|
-
fd
|
244
|
+
if block_given?
|
245
|
+
yield block, record
|
420
246
|
end
|
421
247
|
|
422
|
-
|
423
|
-
|
424
|
-
def self.pad(name, options = {})
|
425
|
-
fd = self.add_field(
|
426
|
-
name.eql?(:auto_name) ? self.new_pad_name : name,
|
427
|
-
options
|
428
|
-
)
|
429
|
-
fd.padding = true
|
430
|
-
end
|
248
|
+
record
|
249
|
+
end
|
431
250
|
|
432
|
-
|
433
|
-
|
434
|
-
|
251
|
+
# Return a lsit of fields for the FlatFile subclass
|
252
|
+
def fields
|
253
|
+
self.class.fields
|
254
|
+
end
|
435
255
|
|
256
|
+
def self.non_pad_fields
|
257
|
+
self.fields.select { |f| not f.is_padding? }
|
258
|
+
end
|
436
259
|
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
def self.new_record(model = nil, &block)
|
441
|
-
fields = get_subclass_variable 'fields'
|
442
|
-
|
443
|
-
record = Record.new(self)
|
444
|
-
|
445
|
-
fields.map do |f|
|
446
|
-
assign_method = "#{f.name}="
|
447
|
-
value = model.respond_to?(f.name.to_sym) ? model.send(f.name.to_sym) : ""
|
448
|
-
record.send(assign_method, value)
|
449
|
-
end
|
450
|
-
|
451
|
-
if block_given?
|
452
|
-
yield block, record
|
453
|
-
end
|
454
|
-
|
455
|
-
record
|
456
|
-
end
|
260
|
+
def non_pad_fields
|
261
|
+
self.non_pad_fields
|
262
|
+
end
|
457
263
|
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
end
|
462
|
-
|
463
|
-
def self.non_pad_fields
|
464
|
-
self.fields.select { |f| not f.is_padding? }
|
465
|
-
end
|
466
|
-
|
467
|
-
def non_pad_fields
|
468
|
-
self.non_pad_fields
|
469
|
-
end
|
264
|
+
def self.fields
|
265
|
+
self.get_subclass_variable 'fields'
|
266
|
+
end
|
470
267
|
|
471
|
-
|
472
|
-
self.get_subclass_variable 'fields'
|
473
|
-
end
|
474
|
-
|
475
|
-
def self.has_field?(field_name)
|
476
|
-
|
477
|
-
if self.fields.select { |f| f.name == field_name.to_sym }.length > 0
|
478
|
-
true
|
479
|
-
else
|
480
|
-
false
|
481
|
-
end
|
482
|
-
end
|
268
|
+
def self.has_field?(field_name)
|
483
269
|
|
484
|
-
|
485
|
-
|
270
|
+
if self.fields.select { |f| f.name == field_name.to_sym }.length > 0
|
271
|
+
true
|
272
|
+
else
|
273
|
+
false
|
486
274
|
end
|
275
|
+
end
|
487
276
|
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
end
|
492
|
-
|
493
|
-
# Returns the pack format which is generated from add_field
|
494
|
-
# calls. This format is used to unpack each line and create Records.
|
495
|
-
def pack_format
|
496
|
-
self.class.get_pack_format
|
497
|
-
end
|
277
|
+
def self.width
|
278
|
+
get_subclass_variable 'width'
|
279
|
+
end
|
498
280
|
|
499
|
-
|
500
|
-
|
501
|
-
|
281
|
+
# Return the record length for the FlatFile subclass
|
282
|
+
def width
|
283
|
+
self.class.width
|
284
|
+
end
|
502
285
|
|
503
|
-
|
286
|
+
# Returns the pack format which is generated from add_field
|
287
|
+
# calls. This format is used to unpack each line and create Records.
|
288
|
+
def pack_format
|
289
|
+
self.class.get_pack_format
|
290
|
+
end
|
504
291
|
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
@@subclass_data.store(self, Hash.new)
|
509
|
-
end
|
292
|
+
def self.pack_format
|
293
|
+
get_subclass_variable 'pack_format'
|
294
|
+
end
|
510
295
|
|
511
|
-
|
512
|
-
end
|
296
|
+
protected
|
513
297
|
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
end
|
298
|
+
# Retrieve the subclass data hash for the current class
|
299
|
+
def self.subclass_data #:nodoc:
|
300
|
+
unless(@@subclass_data.has_key?(self))
|
301
|
+
@@subclass_data.store(self, Hash.new)
|
519
302
|
end
|
520
303
|
|
521
|
-
|
522
|
-
|
523
|
-
subclass_data.store name, value
|
524
|
-
end
|
304
|
+
@@subclass_data.fetch(self)
|
305
|
+
end
|
525
306
|
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
s.set_subclass_variable('pack_format',"")
|
531
|
-
s.set_subclass_variable('fields',Array.new)
|
307
|
+
# Retrieve a particular subclass variable for this class by it's name.
|
308
|
+
def self.get_subclass_variable(name) #:nodoc:
|
309
|
+
if subclass_data.has_key? name
|
310
|
+
subclass_data.fetch name
|
532
311
|
end
|
312
|
+
end
|
313
|
+
|
314
|
+
# Set a subclass variable of 'name' to 'value'
|
315
|
+
def self.set_subclass_variable(name,value) #:nodoc:
|
316
|
+
subclass_data.store name, value
|
317
|
+
end
|
318
|
+
|
319
|
+
# Setup subclass class variables. This initializes the
|
320
|
+
# record width, pack format, and fields array
|
321
|
+
def self.inherited(s) #:nodoc:
|
322
|
+
s.set_subclass_variable('width',0)
|
323
|
+
s.set_subclass_variable('pack_format',"")
|
324
|
+
s.set_subclass_variable('fields',Array.new)
|
325
|
+
end
|
533
326
|
end
|
@@ -0,0 +1,116 @@
|
|
1
|
+
class FlatFile
|
2
|
+
# A field definition tracks infomration that's necessary for
|
3
|
+
# FlatFile to process a particular field. This is typically
|
4
|
+
# added to a subclass of FlatFile like so:
|
5
|
+
#
|
6
|
+
# class SomeFile < FlatFile
|
7
|
+
# add_field :some_field_name, :width => 35
|
8
|
+
# end
|
9
|
+
#
|
10
|
+
class FieldDef
|
11
|
+
attr :name, true
|
12
|
+
attr :width, true
|
13
|
+
attr :filters, true
|
14
|
+
attr :formatters, true
|
15
|
+
attr :file_klass, true
|
16
|
+
attr :padding, true
|
17
|
+
attr :map_in_proc, true
|
18
|
+
attr :aggressive, true
|
19
|
+
attr :default, true
|
20
|
+
|
21
|
+
# Create a new FeildDef, having name and width. klass is a reference to the FlatFile
|
22
|
+
# subclass that contains this field definition. This reference is needed when calling
|
23
|
+
# filters if they are specified using a symbol.
|
24
|
+
#
|
25
|
+
# Options can be :padding (if present and a true value, field is marked as a pad field),
|
26
|
+
# :width, specify the field width, :formatter, specify a formatter, :filter, specify a
|
27
|
+
# filter.
|
28
|
+
#
|
29
|
+
# The option :default => 'value' may be used to specify a default value to be mapped
|
30
|
+
# into a model field provided the flat filer record is empty.
|
31
|
+
|
32
|
+
def initialize(name=null,options={},klass={})
|
33
|
+
@name = name
|
34
|
+
@width = 10
|
35
|
+
@filters = Array.new
|
36
|
+
@formatters = Array.new
|
37
|
+
@file_klass = klass
|
38
|
+
@padding = options.delete(:padding)
|
39
|
+
@default = options.has_key?(:default) ? options.delete(:default) : ""
|
40
|
+
|
41
|
+
add_filter(options[:filter]) if options.has_key?(:filter)
|
42
|
+
add_formatter(options[:formatter]) if options.has_key?(:formatter)
|
43
|
+
@map_in_proc = options[:map_in_proc]
|
44
|
+
@width = options[:width] if options.has_key?(:width)
|
45
|
+
@aggressive = options[:aggressive] || false
|
46
|
+
end
|
47
|
+
|
48
|
+
# Will return true if the field is a padding field. Padding fields are ignored
|
49
|
+
# when doing various things. For example, when you're populating an ActiveRecord
|
50
|
+
# model with a record, padding fields are ignored.
|
51
|
+
def is_padding?
|
52
|
+
@padding
|
53
|
+
end
|
54
|
+
|
55
|
+
# Add a filter. Filters are used for processing field data when a flat file is being
|
56
|
+
# processed. For fomratting the data when writing a flat file, see add_formatter
|
57
|
+
def add_filter(filter=nil,&block) #:nodoc:
|
58
|
+
@filters.push(filter) unless filter.nil?
|
59
|
+
@filters.push(block) if block_given?
|
60
|
+
end
|
61
|
+
|
62
|
+
# Add a formatter. Formatters are used for formatting a field
|
63
|
+
# for rendering a record, or writing it to a file in the desired format.
|
64
|
+
def add_formatter(formatter=nil,&block) #:nodoc:
|
65
|
+
@formatters.push(formatter) if formatter
|
66
|
+
@formatters.push(block) if block_given?
|
67
|
+
end
|
68
|
+
|
69
|
+
# Filters a value based on teh filters associated with a
|
70
|
+
# FieldDef.
|
71
|
+
def pass_through_filters(v) #:nodoc:
|
72
|
+
pass_through(@filters,v)
|
73
|
+
end
|
74
|
+
|
75
|
+
# Filters a value based on the filters associated with a
|
76
|
+
# FieldDef.
|
77
|
+
def pass_through_formatters(v) #:nodoc:
|
78
|
+
pass_through(@formatters,v)
|
79
|
+
end
|
80
|
+
|
81
|
+
#protected
|
82
|
+
|
83
|
+
def pass_through(what,value) #:nodoc:
|
84
|
+
#puts "PASS THROUGH #{what.inspect} => #{value}"
|
85
|
+
what.each do |filter|
|
86
|
+
value = case
|
87
|
+
when filter.is_a?(Symbol)
|
88
|
+
#puts "filter is a symbol"
|
89
|
+
@file_klass.send(filter,value)
|
90
|
+
when filter_block?(filter)
|
91
|
+
#puts "filter is a block"
|
92
|
+
filter.call(value)
|
93
|
+
when filter_class?(filter)
|
94
|
+
#puts "filter is a class"
|
95
|
+
filter.filter(value)
|
96
|
+
else
|
97
|
+
#puts "filter not valid, preserving"
|
98
|
+
value
|
99
|
+
end
|
100
|
+
end
|
101
|
+
value
|
102
|
+
end
|
103
|
+
|
104
|
+
# Test to see if filter is a filter block. A filter block
|
105
|
+
# can be called (using call) and takes one parameter
|
106
|
+
def filter_block?(filter) #:nodoc:
|
107
|
+
filter.respond_to?('call') && ( filter.arity >= 1 || filter.arity <= -1 )
|
108
|
+
end
|
109
|
+
|
110
|
+
# Test to see if a class is a filter class. A filter class responds
|
111
|
+
# to the filter signal (you can call filter on it).
|
112
|
+
def filter_class?(filter) #:nodoc:
|
113
|
+
filter.respond_to?('filter')
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
class FlatFile
|
2
|
+
# A record abstracts on line or 'record' of a fixed width field.
|
3
|
+
# The methods available are the kes of the hash passed to the constructor.
|
4
|
+
# For example the call:
|
5
|
+
#
|
6
|
+
# h = Hash['first_name','Andy','status','Supercool!']
|
7
|
+
# r = Record.new(h)
|
8
|
+
#
|
9
|
+
# would respond to r.first_name, and r.status yielding
|
10
|
+
# 'Andy' and 'Supercool!' respectively.
|
11
|
+
#
|
12
|
+
class Record
|
13
|
+
attr_reader :fields
|
14
|
+
attr_reader :klass
|
15
|
+
attr_reader :line_number
|
16
|
+
|
17
|
+
# Create a new Record from a hash of fields
|
18
|
+
def initialize(klass,fields=Hash.new,line_number = -1,&block)
|
19
|
+
@fields = Hash.new()
|
20
|
+
@klass = klass
|
21
|
+
@line_number = line_number
|
22
|
+
|
23
|
+
klass_fields = klass.get_subclass_variable('fields')
|
24
|
+
|
25
|
+
klass_fields.each do |f|
|
26
|
+
@fields.store(f.name, "")
|
27
|
+
end
|
28
|
+
|
29
|
+
@fields.merge!(fields)
|
30
|
+
|
31
|
+
@fields.each_key do |k|
|
32
|
+
@fields.delete(k) unless klass.has_field?(k)
|
33
|
+
end
|
34
|
+
|
35
|
+
yield(block, self)if block_given?
|
36
|
+
|
37
|
+
self
|
38
|
+
end
|
39
|
+
|
40
|
+
def map_in(model)
|
41
|
+
@klass.non_pad_fields.each do |f|
|
42
|
+
next unless(model.respond_to? "#{f.name}=")
|
43
|
+
if f.map_in_proc
|
44
|
+
f.map_in_proc.call(model,self)
|
45
|
+
else
|
46
|
+
model.send("#{f.name}=", send(f.name)) if f.aggressive or model.send(f.name).blank?
|
47
|
+
model.send("#{f.name}=", f.default) if model.send(f.name).blank?
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
# Catches method calls and returns field values or raises an Error.
|
53
|
+
def method_missing(method,params=nil)
|
54
|
+
if(method.to_s.match(/^(.*)=$/))
|
55
|
+
if(fields.has_key?($1.to_sym))
|
56
|
+
@fields.store($1.to_sym,params)
|
57
|
+
else
|
58
|
+
raise Exception.new("Unknown method: #{ method }")
|
59
|
+
end
|
60
|
+
else
|
61
|
+
if(fields.has_key? method)
|
62
|
+
@fields.fetch(method)
|
63
|
+
else
|
64
|
+
raise Exception.new("Unknown method: #{ method }")
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# Returns a string representation of the record suitable for writing to a flat
|
70
|
+
# file on disk or other media. The fields are parepared according to the file
|
71
|
+
# definition, and any formatters attached to the field definitions.
|
72
|
+
def to_s
|
73
|
+
klass.fields.map { |field_def|
|
74
|
+
field_name = field_def.name.to_s
|
75
|
+
v = @fields[ field_name.to_sym ].to_s
|
76
|
+
|
77
|
+
field_def.pass_through_formatters(
|
78
|
+
field_def.is_padding? ? "" : v
|
79
|
+
)
|
80
|
+
}.pack(klass.pack_format)
|
81
|
+
end
|
82
|
+
|
83
|
+
# Produces a multiline string, one field per line suitable for debugging purposes.
|
84
|
+
def debug_string
|
85
|
+
str = ""
|
86
|
+
klass.fields.each do |f|
|
87
|
+
if f.is_padding?
|
88
|
+
str << "#{f.name}: \n"
|
89
|
+
else
|
90
|
+
str << "#{f.name}: #{send(f.name.to_sym)}\n"
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
str
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
data/lib/flat_filer.rb
ADDED
metadata
CHANGED
@@ -1,56 +1,88 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: flat_filer
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
|
-
authors:
|
6
|
+
authors:
|
7
7
|
- Andrew Libby
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
11
|
+
date: 2018-02-09 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '5.1'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '5.1'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rspec
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '3.7'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '3.7'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: pry
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0.11'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0.11'
|
16
55
|
description:
|
17
|
-
email: alibby@
|
56
|
+
email: alibby@andylibby.org
|
18
57
|
executables: []
|
19
|
-
|
20
58
|
extensions: []
|
21
|
-
|
22
59
|
extra_rdoc_files: []
|
23
|
-
|
24
|
-
files:
|
25
|
-
- lib/core_extensions.rb
|
60
|
+
files:
|
26
61
|
- lib/flat_file.rb
|
27
|
-
|
28
|
-
|
62
|
+
- lib/flat_file/field_def.rb
|
63
|
+
- lib/flat_file/record.rb
|
64
|
+
- lib/flat_filer.rb
|
65
|
+
homepage: https://github.com/alibby/flat_filer
|
29
66
|
licenses: []
|
30
|
-
|
67
|
+
metadata: {}
|
31
68
|
post_install_message:
|
32
69
|
rdoc_options: []
|
33
|
-
|
34
|
-
require_paths:
|
70
|
+
require_paths:
|
35
71
|
- lib
|
36
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
72
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
38
74
|
- - ">="
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version:
|
41
|
-
|
42
|
-
|
43
|
-
requirements:
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
44
79
|
- - ">="
|
45
|
-
- !ruby/object:Gem::Version
|
46
|
-
version:
|
47
|
-
version:
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '0'
|
48
82
|
requirements: []
|
49
|
-
|
50
83
|
rubyforge_project:
|
51
|
-
rubygems_version:
|
84
|
+
rubygems_version: 2.6.13
|
52
85
|
signing_key:
|
53
|
-
specification_version:
|
86
|
+
specification_version: 4
|
54
87
|
summary: Library for processing flat files
|
55
88
|
test_files: []
|
56
|
-
|
data/lib/core_extensions.rb
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
class Object
|
2
|
-
# An object is blank if it's nil, empty, or a whitespace string.
|
3
|
-
# For example, "", " ", '0000', nil, [], 0, 0.0 and {} are blank.
|
4
|
-
#
|
5
|
-
# This simplifies
|
6
|
-
# if !address.nil? && !address.empty?
|
7
|
-
# to
|
8
|
-
# if !address.blank?
|
9
|
-
def blank?
|
10
|
-
if respond_to?(:empty?) && respond_to?(:strip)
|
11
|
-
empty? or strip.empty? or gsub('0', '').empty?
|
12
|
-
elsif respond_to?(:empty?)
|
13
|
-
empty?
|
14
|
-
elsif respond_to?(:zero?)
|
15
|
-
zero?
|
16
|
-
else
|
17
|
-
!self
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|