dbf 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +22 -0
- data/Gemfile +1 -4
- data/Gemfile.lock +83 -6
- data/README.md +20 -21
- data/Rakefile +9 -46
- data/bin/dbf +0 -0
- data/lib/dbf.rb +6 -11
- data/lib/dbf/column.rb +38 -74
- data/lib/dbf/memo.rb +88 -0
- data/lib/dbf/record.rb +29 -104
- data/lib/dbf/table.rb +54 -142
- data/lib/dbf/version.rb +1 -1
- data/spec/{unit → dbf}/column_spec.rb +37 -46
- data/spec/dbf/file_formats_spec.rb +178 -0
- data/spec/dbf/record_spec.rb +45 -0
- data/spec/{unit → dbf}/table_spec.rb +17 -104
- data/spec/spec_helper.rb +2 -4
- metadata +39 -27
- data/spec/functional/dbf_shared.rb +0 -54
- data/spec/functional/format_03_spec.rb +0 -23
- data/spec/functional/format_30_spec.rb +0 -23
- data/spec/functional/format_31_spec.rb +0 -23
- data/spec/functional/format_83_spec.rb +0 -23
- data/spec/functional/format_8b_spec.rb +0 -23
- data/spec/functional/format_f5_spec.rb +0 -23
- data/spec/unit/record_spec.rb +0 -111
data/lib/dbf/memo.rb
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
module DBF
|
2
|
+
class Memo
|
3
|
+
BLOCK_HEADER_SIZE = 8
|
4
|
+
FPT_HEADER_SIZE = 512
|
5
|
+
|
6
|
+
def initialize(data, version)
|
7
|
+
@data, @version = data, version
|
8
|
+
end
|
9
|
+
|
10
|
+
def get(start_block)
|
11
|
+
if start_block > 0
|
12
|
+
if format_fpt?
|
13
|
+
build_fpt_memo start_block
|
14
|
+
else
|
15
|
+
build_dbt_memo start_block
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def close
|
21
|
+
@data.close
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def format_fpt? #nodoc
|
27
|
+
File.extname(@data.path) =~ /fpt/i
|
28
|
+
end
|
29
|
+
|
30
|
+
def build_fpt_memo(start_block) #nodoc
|
31
|
+
@data.seek offset(start_block)
|
32
|
+
|
33
|
+
memo_type, memo_size, memo_string = @data.read(block_size).unpack("NNa*")
|
34
|
+
return nil unless memo_type == 1 && memo_size > 0
|
35
|
+
|
36
|
+
if memo_size > block_content_size
|
37
|
+
memo_string << @data.read(content_size(memo_size))
|
38
|
+
else
|
39
|
+
memo_string = memo_string[0, memo_size]
|
40
|
+
end
|
41
|
+
memo_string
|
42
|
+
end
|
43
|
+
|
44
|
+
def build_dbt_memo(start_block) #nodoc
|
45
|
+
case @version
|
46
|
+
when "83" # dbase iii
|
47
|
+
build_dbt_83_memo(start_block)
|
48
|
+
when "8b" # dbase iv
|
49
|
+
build_dbt_8b_memo(start_block)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def build_dbt_83_memo(start_block)
|
54
|
+
@data.seek offset(start_block)
|
55
|
+
memo_string = ""
|
56
|
+
begin
|
57
|
+
block = @data.read(block_size).gsub(/(\000|\032)/, '')
|
58
|
+
memo_string << block
|
59
|
+
end until block.size < block_size
|
60
|
+
memo_string
|
61
|
+
end
|
62
|
+
|
63
|
+
def build_dbt_8b_memo(start_block)
|
64
|
+
@data.seek offset(start_block)
|
65
|
+
@data.read(@data.read(BLOCK_HEADER_SIZE).unpack("x4L").first)
|
66
|
+
end
|
67
|
+
|
68
|
+
def offset(start_block) #nodoc
|
69
|
+
start_block * block_size
|
70
|
+
end
|
71
|
+
|
72
|
+
def content_size(memo_size) #nodoc
|
73
|
+
(memo_size - block_size) + BLOCK_HEADER_SIZE
|
74
|
+
end
|
75
|
+
|
76
|
+
def block_content_size #nodoc
|
77
|
+
@block_content_size ||= block_size - BLOCK_HEADER_SIZE
|
78
|
+
end
|
79
|
+
|
80
|
+
def block_size #nodoc
|
81
|
+
@block_size ||= begin
|
82
|
+
@data.rewind
|
83
|
+
format_fpt? ? @data.read(FPT_HEADER_SIZE).unpack('x6n').first || 0 : 512
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
end
|
88
|
+
end
|
data/lib/dbf/record.rb
CHANGED
@@ -1,21 +1,11 @@
|
|
1
1
|
module DBF
|
2
2
|
# An instance of DBF::Record represents a row in the DBF file
|
3
3
|
class Record
|
4
|
-
BLOCK_HEADER_SIZE = 8
|
5
|
-
|
6
|
-
attr_reader :table
|
7
|
-
attr_reader :attributes
|
8
|
-
attr_reader :memo_block_size
|
9
|
-
|
10
|
-
delegate :columns, :to => :table
|
11
|
-
|
12
4
|
# Initialize a new DBF::Record
|
13
5
|
#
|
14
6
|
# @param [DBF::Table] table
|
15
|
-
def initialize(
|
16
|
-
@
|
17
|
-
@memo_block_size = @table.memo_block_size
|
18
|
-
initialize_values
|
7
|
+
def initialize(data, columns, version, memo)
|
8
|
+
@data, @columns, @version, @memo = StringIO.new(data), columns, version, memo
|
19
9
|
define_accessors
|
20
10
|
end
|
21
11
|
|
@@ -31,7 +21,7 @@ module DBF
|
|
31
21
|
#
|
32
22
|
# @return [Array]
|
33
23
|
def to_a
|
34
|
-
columns.map { |column|
|
24
|
+
@columns.map { |column| attributes[column.name.underscore] }
|
35
25
|
end
|
36
26
|
|
37
27
|
# Do all search parameters match?
|
@@ -42,109 +32,44 @@ module DBF
|
|
42
32
|
options.all? {|key, value| attributes[key.to_s.underscore] == value}
|
43
33
|
end
|
44
34
|
|
35
|
+
def attributes
|
36
|
+
return @attributes if @attributes
|
37
|
+
|
38
|
+
@attributes = Attributes.new
|
39
|
+
@columns.each do |column|
|
40
|
+
@attributes[column.name] = init_attribute(column)
|
41
|
+
end
|
42
|
+
@attributes
|
43
|
+
end
|
44
|
+
|
45
45
|
private
|
46
46
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
47
|
+
def define_accessors #nodoc
|
48
|
+
@columns.each do |column|
|
49
|
+
unless self.class.method_defined? column.underscored_name
|
50
|
+
self.class.class_eval <<-END
|
51
|
+
def #{column.underscored_name}
|
52
|
+
@#{column.underscored_name} ||= attributes['#{column.underscored_name}']
|
53
|
+
end
|
54
|
+
END
|
54
55
|
end
|
55
56
|
end
|
56
57
|
end
|
57
58
|
|
58
|
-
|
59
|
-
|
60
|
-
@attributes = columns.inject(Attributes.new) do |hash, column|
|
61
|
-
if column.memo?
|
62
|
-
hash[column.name] = read_memo(get_starting_block(column))
|
63
|
-
else
|
64
|
-
hash[column.name] = column.type_cast(unpack_data(column.length))
|
65
|
-
end
|
66
|
-
hash
|
67
|
-
end
|
59
|
+
def init_attribute(column) #nodoc
|
60
|
+
column.memo? ? @memo.get(get_memo_start_block(column)) : column.type_cast(unpack_data(column))
|
68
61
|
end
|
69
62
|
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
def get_starting_block(column)
|
74
|
-
if %w(30 31).include?(@table.version)
|
75
|
-
@data.read(column.length).unpack('V')[0]
|
63
|
+
def get_memo_start_block(column) #nodoc
|
64
|
+
if %w(30 31).include?(@version)
|
65
|
+
@data.read(column.length).unpack('V').first
|
76
66
|
else
|
77
|
-
unpack_data(column
|
67
|
+
unpack_data(column).to_i
|
78
68
|
end
|
79
69
|
end
|
80
70
|
|
81
|
-
|
82
|
-
|
83
|
-
# @param [Fixnum] length
|
84
|
-
def unpack_data(length)
|
85
|
-
@data.read(length).unpack("a#{length}").first
|
86
|
-
end
|
87
|
-
|
88
|
-
# Reads a memo from the memo file
|
89
|
-
#
|
90
|
-
# @param [Fixnum] start_block
|
91
|
-
def read_memo(start_block)
|
92
|
-
return nil if !@table.has_memo_file? || start_block < 1
|
93
|
-
send "build_#{@table.memo_file_format}_memo", start_block
|
94
|
-
end
|
95
|
-
|
96
|
-
# Reconstructs a memo from an FPT memo file
|
97
|
-
#
|
98
|
-
# @param [Fixnum] start_block
|
99
|
-
# @return [String]
|
100
|
-
def build_fpt_memo(start_block)
|
101
|
-
@memo.seek(start_block * memo_block_size)
|
102
|
-
|
103
|
-
memo_type, memo_size, memo_string = @memo.read(memo_block_size).unpack("NNa*")
|
104
|
-
return nil unless memo_type == 1 && memo_size > 0
|
105
|
-
|
106
|
-
if memo_size > memo_block_content_size
|
107
|
-
memo_string << @memo.read(memo_content_size(memo_size))
|
108
|
-
else
|
109
|
-
memo_string = memo_string[0, memo_size]
|
110
|
-
end
|
111
|
-
memo_string.strip
|
112
|
-
end
|
113
|
-
|
114
|
-
# Reconstucts a memo from an DBT memo file
|
115
|
-
#
|
116
|
-
# @param [Fixnum] start_block
|
117
|
-
# @return [String]
|
118
|
-
def build_dbt_memo(start_block)
|
119
|
-
@memo.seek(start_block * memo_block_size)
|
120
|
-
|
121
|
-
case @table.version
|
122
|
-
when "83" # dbase iii
|
123
|
-
memo_string = ""
|
124
|
-
loop do
|
125
|
-
block = @memo.read(memo_block_size)
|
126
|
-
memo_string << block
|
127
|
-
break if block.tr("\000",'').size < memo_block_size
|
128
|
-
end
|
129
|
-
when "8b" # dbase iv
|
130
|
-
memo_type, memo_size = @memo.read(BLOCK_HEADER_SIZE).unpack("LL")
|
131
|
-
memo_string = @memo.read(memo_size)
|
132
|
-
end
|
133
|
-
memo_string
|
134
|
-
end
|
135
|
-
|
136
|
-
# The size in bytes of the content for each memo block
|
137
|
-
#
|
138
|
-
# @return [Fixnum]
|
139
|
-
def memo_block_content_size
|
140
|
-
memo_block_size - BLOCK_HEADER_SIZE
|
141
|
-
end
|
142
|
-
|
143
|
-
# The size in bytes of the entire memo
|
144
|
-
#
|
145
|
-
# @return [Fixnum]
|
146
|
-
def memo_content_size(memo_size)
|
147
|
-
(memo_size - memo_block_size) + BLOCK_HEADER_SIZE
|
71
|
+
def unpack_data(column) #nodoc
|
72
|
+
@data.read(column.length).unpack("a#{column.length}").first
|
148
73
|
end
|
149
74
|
|
150
75
|
end
|
data/lib/dbf/table.rb
CHANGED
@@ -2,9 +2,12 @@ module DBF
|
|
2
2
|
|
3
3
|
# DBF::Table is the primary interface to a single DBF file and provides
|
4
4
|
# methods for enumerating and searching the records.
|
5
|
+
|
6
|
+
# TODO set record_length to length of actual used column lengths
|
5
7
|
class Table
|
8
|
+
include Enumerable
|
9
|
+
|
6
10
|
DBF_HEADER_SIZE = 32
|
7
|
-
FPT_HEADER_SIZE = 512
|
8
11
|
|
9
12
|
VERSION_DESCRIPTIONS = {
|
10
13
|
"02" => "FoxBase",
|
@@ -21,15 +24,7 @@ module DBF
|
|
21
24
|
"fb" => "FoxPro without memo file"
|
22
25
|
}
|
23
26
|
|
24
|
-
attr_reader :column_count # The total number of columns
|
25
|
-
attr_reader :columns # An array of DBF::Column
|
26
27
|
attr_reader :version # Internal dBase version number
|
27
|
-
attr_reader :last_updated # Last updated datetime
|
28
|
-
attr_reader :memo_file_format # :fpt or :dpt
|
29
|
-
attr_reader :memo_block_size # The block size for memo records
|
30
|
-
attr_reader :options # The options hash used to initialize the table
|
31
|
-
attr_reader :data # DBF file handle
|
32
|
-
attr_reader :memo # Memo file handle
|
33
28
|
attr_reader :record_count # Total number of records
|
34
29
|
|
35
30
|
# Opens a DBF::Table
|
@@ -39,37 +34,14 @@ module DBF
|
|
39
34
|
# @param [String] path Path to the dbf file
|
40
35
|
def initialize(path)
|
41
36
|
@data = File.open(path, 'rb')
|
37
|
+
get_header_info
|
42
38
|
@memo = open_memo(path)
|
43
|
-
reload!
|
44
39
|
end
|
45
40
|
|
46
41
|
# Closes the table and memo file
|
47
42
|
def close
|
43
|
+
@memo && @memo.close
|
48
44
|
@data.close
|
49
|
-
@memo.close if @memo
|
50
|
-
end
|
51
|
-
|
52
|
-
# Reloads the database and memo files
|
53
|
-
def reload!
|
54
|
-
@records = nil
|
55
|
-
get_header_info
|
56
|
-
get_memo_header_info
|
57
|
-
get_column_descriptors
|
58
|
-
end
|
59
|
-
|
60
|
-
# Checks if there is a memo file
|
61
|
-
#
|
62
|
-
# @return [Boolean]
|
63
|
-
def has_memo_file?
|
64
|
-
@memo ? true : false
|
65
|
-
end
|
66
|
-
|
67
|
-
# Retrieve a Column by name
|
68
|
-
#
|
69
|
-
# @param [String, Symbol] column_name
|
70
|
-
# @return [DBF::Column]
|
71
|
-
def column(column_name)
|
72
|
-
@columns.detect {|f| f.name == column_name.to_s}
|
73
45
|
end
|
74
46
|
|
75
47
|
# Calls block once for each record in the table. The record may be nil
|
@@ -77,7 +49,7 @@ module DBF
|
|
77
49
|
#
|
78
50
|
# @yield [nil, DBF::Record]
|
79
51
|
def each
|
80
|
-
|
52
|
+
@record_count.times {|i| yield record(i)}
|
81
53
|
end
|
82
54
|
|
83
55
|
# Retrieve a record by index number.
|
@@ -120,40 +92,26 @@ module DBF
|
|
120
92
|
# t.column :notes, :text
|
121
93
|
# end
|
122
94
|
#
|
123
|
-
# @param [optional String] path
|
124
95
|
# @return [String]
|
125
|
-
def schema
|
96
|
+
def schema
|
126
97
|
s = "ActiveRecord::Schema.define do\n"
|
127
98
|
s << " create_table \"#{File.basename(@data.path, ".*")}\" do |t|\n"
|
128
99
|
columns.each do |column|
|
129
100
|
s << " t.column #{column.schema_definition}"
|
130
101
|
end
|
131
|
-
s << " end\nend"
|
132
|
-
|
133
|
-
if path
|
134
|
-
File.open(path, 'w') {|f| f.puts(s)}
|
135
|
-
end
|
136
|
-
|
102
|
+
s << " end\nend"
|
137
103
|
s
|
138
104
|
end
|
139
105
|
|
140
|
-
def to_a
|
141
|
-
records = []
|
142
|
-
each {|record| records << record if record}
|
143
|
-
records
|
144
|
-
end
|
145
|
-
|
146
106
|
# Dumps all records to a CSV file. If no filename is given then CSV is
|
147
107
|
# output to STDOUT.
|
148
108
|
#
|
149
109
|
# @param [optional String] path Defaults to basename of dbf file
|
150
110
|
def to_csv(path = nil)
|
151
|
-
path =
|
152
|
-
|
111
|
+
path = default_csv_path unless path
|
112
|
+
csv_class.open(path, 'w', :force_quotes => true) do |csv|
|
153
113
|
csv << columns.map {|c| c.name}
|
154
|
-
each
|
155
|
-
csv << record.to_a
|
156
|
-
end
|
114
|
+
each {|record| csv << record.to_a}
|
157
115
|
end
|
158
116
|
end
|
159
117
|
|
@@ -194,121 +152,75 @@ module DBF
|
|
194
152
|
end
|
195
153
|
end
|
196
154
|
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
if record.try(:match?, options)
|
208
|
-
if block_given?
|
209
|
-
yield record
|
210
|
-
else
|
211
|
-
results << record
|
212
|
-
end
|
213
|
-
end
|
155
|
+
# Retrieves column information from the database
|
156
|
+
def columns
|
157
|
+
return @columns if @columns
|
158
|
+
|
159
|
+
@data.seek(DBF_HEADER_SIZE)
|
160
|
+
@columns = []
|
161
|
+
column_count = (@header_length - DBF_HEADER_SIZE + 1) / DBF_HEADER_SIZE
|
162
|
+
column_count.times do
|
163
|
+
name, type, length, decimal = @data.read(32).unpack('a10 x a x4 C2')
|
164
|
+
@columns << Column.new(name.strip, type, length, decimal) if length > 0
|
214
165
|
end
|
215
|
-
|
166
|
+
@columns
|
216
167
|
end
|
217
168
|
|
218
|
-
|
219
|
-
#
|
220
|
-
# @param [Hash] options
|
221
|
-
# @return [DBF::Record, nil]
|
222
|
-
def find_first(options)
|
223
|
-
each do |record|
|
224
|
-
return record if record.try(:match?, options)
|
225
|
-
end
|
226
|
-
nil
|
227
|
-
end
|
169
|
+
private
|
228
170
|
|
229
|
-
|
230
|
-
#
|
231
|
-
# @params [String] path
|
232
|
-
# @return [File]
|
233
|
-
def open_memo(path)
|
171
|
+
def open_memo(path) #nodoc
|
234
172
|
%w(fpt FPT dbt DBT).each do |extname|
|
235
|
-
filename =
|
173
|
+
filename = path.sub(/#{File.extname(path)[1..-1]}$/, extname)
|
236
174
|
if File.exists?(filename)
|
237
|
-
|
238
|
-
return File.open(filename, 'rb')
|
175
|
+
return Memo.new(File.open(filename, 'rb'), version)
|
239
176
|
end
|
240
177
|
end
|
241
178
|
nil
|
242
179
|
end
|
243
180
|
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
181
|
+
def find_all(options) #nodoc
|
182
|
+
map do |record|
|
183
|
+
if record.try(:match?, options)
|
184
|
+
yield record if block_given?
|
185
|
+
record
|
186
|
+
end
|
187
|
+
end.compact
|
251
188
|
end
|
252
189
|
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
190
|
+
def find_first(options) #nodoc
|
191
|
+
each do |record|
|
192
|
+
return record if record.try(:match?, options)
|
193
|
+
end
|
194
|
+
nil
|
195
|
+
end
|
196
|
+
|
197
|
+
def deleted_record? #nodoc
|
257
198
|
@data.read(1).unpack('a') == ['*']
|
258
199
|
end
|
259
200
|
|
260
201
|
def current_record
|
261
|
-
deleted_record? ? nil : DBF::Record.new(
|
202
|
+
deleted_record? ? nil : DBF::Record.new(@data.read(@record_length), columns, version, @memo)
|
262
203
|
end
|
263
204
|
|
264
|
-
|
265
|
-
def get_header_info
|
205
|
+
def get_header_info #nodoc
|
266
206
|
@data.rewind
|
267
207
|
@version, @record_count, @header_length, @record_length = @data.read(DBF_HEADER_SIZE).unpack('H2 x3 V v2')
|
268
|
-
@column_count = (@header_length - DBF_HEADER_SIZE + 1) / DBF_HEADER_SIZE
|
269
208
|
end
|
270
209
|
|
271
|
-
|
272
|
-
|
273
|
-
@columns = []
|
274
|
-
@column_count.times do
|
275
|
-
name, type, length, decimal = @data.read(32).unpack('a10 x a x4 C2')
|
276
|
-
if length > 0
|
277
|
-
@columns << Column.new(name.strip, type, length, decimal)
|
278
|
-
end
|
279
|
-
end
|
280
|
-
# Reset the column count in case any were skipped
|
281
|
-
@column_count = @columns.size
|
282
|
-
|
283
|
-
@columns
|
210
|
+
def seek(offset) #nodoc
|
211
|
+
@data.seek @header_length + offset
|
284
212
|
end
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
if has_memo_file?
|
289
|
-
@memo.rewind
|
290
|
-
if @memo_file_format == :fpt
|
291
|
-
@memo_next_available_block, @memo_block_size = @memo.read(FPT_HEADER_SIZE).unpack('N x2 n')
|
292
|
-
@memo_block_size = 0 if @memo_block_size.nil?
|
293
|
-
else
|
294
|
-
@memo_block_size = 512
|
295
|
-
@memo_next_available_block = File.size(@memo.path) / @memo_block_size
|
296
|
-
end
|
297
|
-
end
|
213
|
+
|
214
|
+
def seek_to_record(index) #nodoc
|
215
|
+
seek index * @record_length
|
298
216
|
end
|
299
217
|
|
300
|
-
|
301
|
-
|
302
|
-
# @params [Fixnum] offset
|
303
|
-
def seek(offset)
|
304
|
-
@data.seek(@header_length + offset)
|
218
|
+
def csv_class #nodoc
|
219
|
+
CSV.const_defined?(:Reader) ? FCSV : CSV
|
305
220
|
end
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
# @param [Fixnum] index
|
310
|
-
def seek_to_record(index)
|
311
|
-
seek(index * @record_length)
|
221
|
+
|
222
|
+
def default_csv_path #nodoc
|
223
|
+
File.basename(@data.path, '.dbf') + '.csv'
|
312
224
|
end
|
313
225
|
|
314
226
|
end
|