dbf 5.4.0 → 5.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +77 -0
- data/LICENSE +1 -1
- data/README.md +91 -39
- data/dbf.gemspec +5 -1
- data/lib/dbf/cli.rb +82 -16
- data/lib/dbf/column.rb +28 -5
- data/lib/dbf/column_builder.rb +38 -2
- data/lib/dbf/column_type.rb +11 -4
- data/lib/dbf/database/foxpro.rb +23 -15
- data/lib/dbf/encoder.rb +60 -5
- data/lib/dbf/errors.rb +37 -0
- data/lib/dbf/file_handler.rb +62 -14
- data/lib/dbf/header.rb +1 -3
- data/lib/dbf/memo/base.rb +15 -4
- data/lib/dbf/memo/dbase3.rb +18 -4
- data/lib/dbf/memo/dbase4.rb +1 -1
- data/lib/dbf/memo/foxpro.rb +15 -14
- data/lib/dbf/record.rb +18 -5
- data/lib/dbf/record_context.rb +2 -1
- data/lib/dbf/record_iterator.rb +49 -20
- data/lib/dbf/schema.rb +3 -4
- data/lib/dbf/table.rb +64 -19
- data/lib/dbf/version.rb +1 -1
- data/lib/dbf.rb +1 -0
- metadata +11 -6
data/lib/dbf/database/foxpro.rb
CHANGED
|
@@ -20,13 +20,16 @@ module DBF
|
|
|
20
20
|
# contacts = db.contacts.record(0)
|
|
21
21
|
#
|
|
22
22
|
# @param path [String]
|
|
23
|
-
|
|
23
|
+
# @param max_memo_size [optional Integer] Bytes to read from each memo
|
|
24
|
+
# of the container and its tables; see DBF::Table.new
|
|
25
|
+
def initialize(path, max_memo_size: nil)
|
|
24
26
|
@path = path
|
|
25
27
|
@dirname = File.dirname(@path)
|
|
26
|
-
@
|
|
28
|
+
@max_memo_size = max_memo_size
|
|
29
|
+
@db = DBF::Table.new(@path, max_memo_size:)
|
|
27
30
|
@tables = extract_dbc_data
|
|
28
31
|
rescue Errno::ENOENT
|
|
29
|
-
raise
|
|
32
|
+
raise FileHandler.not_found(path)
|
|
30
33
|
end
|
|
31
34
|
|
|
32
35
|
def table_names
|
|
@@ -38,7 +41,7 @@ module DBF
|
|
|
38
41
|
# @param name [String]
|
|
39
42
|
# @return [DBF::Table]
|
|
40
43
|
def table(name)
|
|
41
|
-
Table.new(table_path(name), long_names: @tables[name])
|
|
44
|
+
Table.new(table_path(name), long_names: @tables[name], max_memo_size: @max_memo_size)
|
|
42
45
|
end
|
|
43
46
|
|
|
44
47
|
# Searches the database directory for the table's dbf file
|
|
@@ -71,15 +74,21 @@ module DBF
|
|
|
71
74
|
|
|
72
75
|
def method_missing(method, *args) # :nodoc:
|
|
73
76
|
name = method.to_s
|
|
74
|
-
|
|
77
|
+
table_method?(name) ? table(name) : super
|
|
75
78
|
end
|
|
76
79
|
|
|
77
80
|
def respond_to_missing?(method, *)
|
|
78
|
-
|
|
81
|
+
table_method?(method.to_s) || super
|
|
79
82
|
end
|
|
80
83
|
|
|
81
84
|
private
|
|
82
85
|
|
|
86
|
+
# Table names come from the container file, so implicit-conversion
|
|
87
|
+
# method names are never dispatched to a table (see Record).
|
|
88
|
+
def table_method?(name) # :nodoc:
|
|
89
|
+
!Record::RESERVED_METHODS.include?(name) && table_names.include?(name)
|
|
90
|
+
end
|
|
91
|
+
|
|
83
92
|
# This method extracts the data from the database container. This is
|
|
84
93
|
# just an ordinary table with a treelike structure. Field definitions
|
|
85
94
|
# are in the same order as in the linked tables but only the long name
|
|
@@ -108,19 +117,18 @@ module DBF
|
|
|
108
117
|
class Table < DBF::Table
|
|
109
118
|
attr_reader :long_names
|
|
110
119
|
|
|
111
|
-
def initialize(path, long_names:)
|
|
120
|
+
def initialize(path, long_names:, max_memo_size: nil)
|
|
112
121
|
@long_names = long_names
|
|
113
|
-
super(path)
|
|
122
|
+
super(path, max_memo_size:)
|
|
114
123
|
end
|
|
115
124
|
|
|
125
|
+
# Replaces each column's short name with its long name from the
|
|
126
|
+
# container. Long names were decoded when the .dbc was read, so they
|
|
127
|
+
# are not decoded again with this table's code page.
|
|
116
128
|
def build_columns # :nodoc:
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
# columnname property is readonly, recreate the column definitions
|
|
121
|
-
columns.map do |column|
|
|
122
|
-
long_name = long_names[columns.index(column)]
|
|
123
|
-
Column.new(self, long_name, column.type, column.length, column.decimal)
|
|
129
|
+
super.each_with_index.map do |column, index|
|
|
130
|
+
long_name = long_names[index]
|
|
131
|
+
long_name ? column.with_name(long_name) : column
|
|
124
132
|
end
|
|
125
133
|
end
|
|
126
134
|
end
|
data/lib/dbf/encoder.rb
CHANGED
|
@@ -4,12 +4,15 @@ module DBF
|
|
|
4
4
|
# Transcodes strings from a source encoding to a target encoding.
|
|
5
5
|
#
|
|
6
6
|
# Most DBF code pages map to encodings built into Ruby and are converted
|
|
7
|
-
# with String#encode. A few (Mazovia cp620, Kamenický cp895
|
|
8
|
-
#
|
|
7
|
+
# with String#encode. A few (Mazovia cp620, Kamenický cp895, Macintosh
|
|
8
|
+
# Central European) have no Ruby converter and are decoded through
|
|
9
|
+
# vendored byte-to-Unicode tables.
|
|
9
10
|
module Encoder
|
|
10
11
|
# Byte-to-Unicode tables generated from
|
|
11
|
-
# https://github.com/SheetJS/js-codepage/blob/master/codepages/620.TBL
|
|
12
|
-
# https://github.com/SheetJS/js-codepage/blob/master/codepages/895.TBL
|
|
12
|
+
# https://github.com/SheetJS/js-codepage/blob/master/codepages/620.TBL,
|
|
13
|
+
# https://github.com/SheetJS/js-codepage/blob/master/codepages/895.TBL and
|
|
14
|
+
# (Mac Central European) Python's mac_latin2 codec, which matches iconv's
|
|
15
|
+
# MACCENTRALEUROPE
|
|
13
16
|
|
|
14
17
|
# Mazovia (Polish) MS-DOS
|
|
15
18
|
CP620_TO_UNICODE = [
|
|
@@ -83,9 +86,46 @@ module DBF
|
|
|
83
86
|
0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, 0x25A0, 0x00A0
|
|
84
87
|
].freeze
|
|
85
88
|
|
|
89
|
+
# Macintosh Central European (Ruby has the encoding but no converter)
|
|
90
|
+
MACCENTEURO_TO_UNICODE = [
|
|
91
|
+
0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
|
|
92
|
+
0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F,
|
|
93
|
+
0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017,
|
|
94
|
+
0x0018, 0x0019, 0x001A, 0x001B, 0x001C, 0x001D, 0x001E, 0x001F,
|
|
95
|
+
0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027,
|
|
96
|
+
0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F,
|
|
97
|
+
0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037,
|
|
98
|
+
0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F,
|
|
99
|
+
0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
|
|
100
|
+
0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F,
|
|
101
|
+
0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057,
|
|
102
|
+
0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F,
|
|
103
|
+
0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067,
|
|
104
|
+
0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F,
|
|
105
|
+
0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077,
|
|
106
|
+
0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0x007F,
|
|
107
|
+
0x00C4, 0x0100, 0x0101, 0x00C9, 0x0104, 0x00D6, 0x00DC, 0x00E1,
|
|
108
|
+
0x0105, 0x010C, 0x00E4, 0x010D, 0x0106, 0x0107, 0x00E9, 0x0179,
|
|
109
|
+
0x017A, 0x010E, 0x00ED, 0x010F, 0x0112, 0x0113, 0x0116, 0x00F3,
|
|
110
|
+
0x0117, 0x00F4, 0x00F6, 0x00F5, 0x00FA, 0x011A, 0x011B, 0x00FC,
|
|
111
|
+
0x2020, 0x00B0, 0x0118, 0x00A3, 0x00A7, 0x2022, 0x00B6, 0x00DF,
|
|
112
|
+
0x00AE, 0x00A9, 0x2122, 0x0119, 0x00A8, 0x2260, 0x0123, 0x012E,
|
|
113
|
+
0x012F, 0x012A, 0x2264, 0x2265, 0x012B, 0x0136, 0x2202, 0x2211,
|
|
114
|
+
0x0142, 0x013B, 0x013C, 0x013D, 0x013E, 0x0139, 0x013A, 0x0145,
|
|
115
|
+
0x0146, 0x0143, 0x00AC, 0x221A, 0x0144, 0x0147, 0x2206, 0x00AB,
|
|
116
|
+
0x00BB, 0x2026, 0x00A0, 0x0148, 0x0150, 0x00D5, 0x0151, 0x014C,
|
|
117
|
+
0x2013, 0x2014, 0x201C, 0x201D, 0x2018, 0x2019, 0x00F7, 0x25CA,
|
|
118
|
+
0x014D, 0x0154, 0x0155, 0x0158, 0x2039, 0x203A, 0x0159, 0x0156,
|
|
119
|
+
0x0157, 0x0160, 0x201A, 0x201E, 0x0161, 0x015A, 0x015B, 0x00C1,
|
|
120
|
+
0x0164, 0x0165, 0x00CD, 0x017D, 0x017E, 0x016A, 0x00D3, 0x00D4,
|
|
121
|
+
0x016B, 0x016E, 0x00DA, 0x016F, 0x0170, 0x0171, 0x0172, 0x0173,
|
|
122
|
+
0x00DD, 0x00FD, 0x0137, 0x017B, 0x0141, 0x017C, 0x0122, 0x02C7
|
|
123
|
+
].freeze
|
|
124
|
+
|
|
86
125
|
CUSTOM_TABLES = {
|
|
87
126
|
'cp620' => CP620_TO_UNICODE,
|
|
88
|
-
'cp895' => CP895_TO_UNICODE
|
|
127
|
+
'cp895' => CP895_TO_UNICODE,
|
|
128
|
+
'maccenteuro' => MACCENTEURO_TO_UNICODE
|
|
89
129
|
}.freeze
|
|
90
130
|
|
|
91
131
|
# Is the encoding a code page without a Ruby Encoding, decoded through
|
|
@@ -97,6 +137,21 @@ module DBF
|
|
|
97
137
|
encoding.is_a?(::String) && CUSTOM_TABLES.key?(encoding.downcase)
|
|
98
138
|
end
|
|
99
139
|
|
|
140
|
+
# Raise unless strings in the encoding can be transcoded to Unicode, so
|
|
141
|
+
# an unknown name or a dummy encoding such as UTF-7 fails up front with
|
|
142
|
+
# a DBF::Error rather than with ArgumentError or
|
|
143
|
+
# Encoding::ConverterNotFoundError on the first read.
|
|
144
|
+
#
|
|
145
|
+
# @param encoding [String, Encoding]
|
|
146
|
+
# @raise [DBF::UnsupportedEncodingError]
|
|
147
|
+
def self.check(encoding)
|
|
148
|
+
return if custom?(encoding)
|
|
149
|
+
|
|
150
|
+
(+'').force_encoding(encoding).encode(Encoding::UTF_8)
|
|
151
|
+
rescue ArgumentError, TypeError, Encoding::ConverterNotFoundError
|
|
152
|
+
raise DBF::UnsupportedEncodingError, "unsupported encoding: #{encoding}"
|
|
153
|
+
end
|
|
154
|
+
|
|
100
155
|
# Transcode a string from source_encoding to target_encoding
|
|
101
156
|
#
|
|
102
157
|
# @param string [String]
|
data/lib/dbf/errors.rb
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module DBF
|
|
4
|
+
# Base class for all errors raised by the DBF library, so callers can
|
|
5
|
+
# rescue DBF::Error to catch anything the library raises.
|
|
6
|
+
class Error < StandardError
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
# A file cannot be opened. The message names only the file, not its
|
|
10
|
+
# directory, so an app that shows error messages to users does not reveal
|
|
11
|
+
# server paths; #path has the full path for logging.
|
|
12
|
+
class FileError < Error
|
|
13
|
+
# @return [String, nil] the full path of the file
|
|
14
|
+
attr_reader :path
|
|
15
|
+
|
|
16
|
+
def initialize(message = nil, path: nil)
|
|
17
|
+
super(message)
|
|
18
|
+
@path = path
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
class FileNotFoundError < FileError
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# A file exists but cannot be opened, e.g. it is a directory or is not
|
|
26
|
+
# readable.
|
|
27
|
+
class FileAccessError < FileError
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# The encoding passed to Table.new is unknown, or Ruby cannot transcode
|
|
31
|
+
# it to Unicode.
|
|
32
|
+
class UnsupportedEncodingError < Error
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
class NoColumnsDefined < Error
|
|
36
|
+
end
|
|
37
|
+
end
|
data/lib/dbf/file_handler.rb
CHANGED
|
@@ -9,28 +9,76 @@ module DBF
|
|
|
9
9
|
when StringIO
|
|
10
10
|
data
|
|
11
11
|
when String
|
|
12
|
-
|
|
12
|
+
open_file(data)
|
|
13
13
|
else
|
|
14
|
-
raise ArgumentError, 'data must be a file path or
|
|
14
|
+
raise ArgumentError, 'data must be a file path or an IO-like object responding to #read and #seek' unless data.respond_to?(:read) && data.respond_to?(:seek)
|
|
15
|
+
|
|
16
|
+
# DBF is a binary format; a File opened in text mode would corrupt
|
|
17
|
+
# reads on Windows.
|
|
18
|
+
data.binmode if data.respond_to?(:binmode)
|
|
19
|
+
data
|
|
15
20
|
end
|
|
16
|
-
rescue Errno::ENOENT
|
|
17
|
-
raise DBF::FileNotFoundError, "file not found: #{data}"
|
|
18
21
|
end
|
|
19
22
|
|
|
20
|
-
|
|
23
|
+
# Opens a file for binary reading, raising DBF errors instead of the
|
|
24
|
+
# system errors File.open raises, so callers need only rescue DBF::Error.
|
|
25
|
+
# A path containing a null byte cannot name a file, so it is not found.
|
|
26
|
+
# A directory opens without error but fails on the first read, so it is
|
|
27
|
+
# rejected here.
|
|
28
|
+
def open_file(path)
|
|
29
|
+
raise Errno::EISDIR if File.directory?(path)
|
|
30
|
+
|
|
31
|
+
File.open(path, 'rb')
|
|
32
|
+
rescue Errno::ENOENT, Errno::ENOTDIR, ArgumentError
|
|
33
|
+
raise not_found(path)
|
|
34
|
+
rescue SystemCallError => e
|
|
35
|
+
reason = SystemCallError.new(nil, e.errno).message
|
|
36
|
+
raise DBF::FileAccessError.new("cannot open file: #{display_name(path)} (#{reason})", path:)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def not_found(path)
|
|
40
|
+
DBF::FileNotFoundError.new("file not found: #{display_name(path)}", path:)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# The file name without its directory, for error messages. A null byte,
|
|
44
|
+
# which File.basename rejects, is shown as "?".
|
|
45
|
+
def display_name(path)
|
|
46
|
+
File.basename(path.to_s.tr("\x00", '?'))
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def open_memo(data, memo, memo_class, version, max_size: nil)
|
|
21
50
|
if memo
|
|
22
|
-
meth = memo.is_a?(
|
|
23
|
-
memo_class.send(meth, memo, version)
|
|
24
|
-
elsif
|
|
25
|
-
|
|
26
|
-
|
|
51
|
+
meth = memo.is_a?(String) ? :open : :new
|
|
52
|
+
memo_class.send(meth, memo, version, max_size:)
|
|
53
|
+
elsif (path = data_path(data))
|
|
54
|
+
found = find_memo(path)
|
|
55
|
+
found && memo_class.open(found, version, max_size:)
|
|
27
56
|
end
|
|
28
57
|
end
|
|
29
58
|
|
|
30
|
-
def
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
59
|
+
def data_path(data)
|
|
60
|
+
return data if data.is_a?(String)
|
|
61
|
+
|
|
62
|
+
data.path if data.respond_to?(:path)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
MEMO_EXTENSIONS = %w[fpt dbt].freeze
|
|
66
|
+
|
|
67
|
+
# Finds the memo file sitting next to the data file: same basename, with
|
|
68
|
+
# an .fpt or .dbt extension in any case. The directory is scanned rather
|
|
69
|
+
# than globbed so glob metacharacters in an uploaded filename cannot be
|
|
70
|
+
# expanded, and the basename must match exactly so data.dbf never picks
|
|
71
|
+
# up an unrelated memo such as data2.fpt.
|
|
72
|
+
def find_memo(path)
|
|
73
|
+
dirname = File.dirname(path)
|
|
74
|
+
basename = File.basename(path, '.*')
|
|
75
|
+
entries = Dir.children(dirname).sort.each_with_object({}) do |child, index|
|
|
76
|
+
index[child.downcase(:ascii)] ||= child
|
|
77
|
+
end
|
|
78
|
+
entry = MEMO_EXTENSIONS.filter_map { |ext| entries["#{basename}.#{ext}".downcase(:ascii)] }.first
|
|
79
|
+
entry && File.join(dirname, entry)
|
|
80
|
+
rescue SystemCallError
|
|
81
|
+
nil
|
|
34
82
|
end
|
|
35
83
|
end
|
|
36
84
|
end
|
data/lib/dbf/header.rb
CHANGED
|
@@ -2,14 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
module DBF
|
|
4
4
|
class Header
|
|
5
|
-
HEADER_SIZE = 32
|
|
6
|
-
|
|
7
5
|
attr_reader :version, :record_count, :header_length, :record_length, :encoding_key, :encoding
|
|
8
6
|
|
|
9
7
|
def initialize(data)
|
|
10
8
|
# Pad a nil or truncated header read so unpacking a short file yields
|
|
11
9
|
# empty values instead of raising.
|
|
12
|
-
data = data.to_s.b.ljust(
|
|
10
|
+
data = data.to_s.b.ljust(VersionConfig::DBASE3_HEADER_SIZE, "\x00")
|
|
13
11
|
@version = data.unpack1('H2')
|
|
14
12
|
@encoding_key = nil
|
|
15
13
|
@encoding = nil
|
data/lib/dbf/memo/base.rb
CHANGED
|
@@ -6,19 +6,25 @@ module DBF
|
|
|
6
6
|
BLOCK_HEADER_SIZE = 8
|
|
7
7
|
BLOCK_SIZE = 512
|
|
8
8
|
|
|
9
|
-
def self.open(filename, version)
|
|
10
|
-
new(
|
|
9
|
+
def self.open(filename, version, max_size: nil)
|
|
10
|
+
new(FileHandler.open_file(filename), version, max_size:)
|
|
11
11
|
end
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
# @param max_size [Integer, nil] bytes to read from each memo; longer
|
|
14
|
+
# memos are cut off. nil reads every memo in full.
|
|
15
|
+
def initialize(data, version, max_size: nil)
|
|
14
16
|
@data = data
|
|
15
17
|
@version = version
|
|
18
|
+
@max_size = max_size
|
|
16
19
|
end
|
|
17
20
|
|
|
18
21
|
def get(start_block)
|
|
19
22
|
return nil unless start_block > 0
|
|
20
23
|
|
|
21
24
|
build_memo start_block
|
|
25
|
+
rescue RangeError
|
|
26
|
+
# A corrupt memo pointer too large to seek to has no memo
|
|
27
|
+
nil
|
|
22
28
|
end
|
|
23
29
|
|
|
24
30
|
def close
|
|
@@ -31,7 +37,12 @@ module DBF
|
|
|
31
37
|
|
|
32
38
|
private
|
|
33
39
|
|
|
34
|
-
attr_reader :data
|
|
40
|
+
attr_reader :data, :max_size
|
|
41
|
+
|
|
42
|
+
# Bounds a memo length by max_size, when one is set
|
|
43
|
+
def limit(length) # :nodoc:
|
|
44
|
+
max_size && length > max_size ? max_size : length
|
|
45
|
+
end
|
|
35
46
|
|
|
36
47
|
def offset(start_block) # :nodoc:
|
|
37
48
|
start_block * block_size
|
data/lib/dbf/memo/dbase3.rb
CHANGED
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
module DBF
|
|
4
4
|
module Memo
|
|
5
5
|
class Dbase3 < Base
|
|
6
|
+
# Memo text up to, and whether it reached, the 0x1A terminator
|
|
7
|
+
TERMINATOR_PATTERN = /\A([^\x1A]*)(\x1A)?/n
|
|
8
|
+
|
|
6
9
|
def build_memo(start_block) # :nodoc:
|
|
7
10
|
data.seek offset(start_block)
|
|
8
11
|
memo_string = +''
|
|
@@ -12,11 +15,22 @@ module DBF
|
|
|
12
15
|
# than crashing on nil.gsub.
|
|
13
16
|
break if block.nil?
|
|
14
17
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
+
# The memo ends at its 0x1A terminator; bytes after it in the same
|
|
19
|
+
# block are leftovers, not memo text.
|
|
20
|
+
text, terminated = TERMINATOR_PATTERN.match(block).captures
|
|
21
|
+
text = text.delete("\000")
|
|
22
|
+
memo_string << text
|
|
23
|
+
break if terminated || text.size < BLOCK_SIZE || full?(memo_string)
|
|
18
24
|
end
|
|
19
|
-
memo_string
|
|
25
|
+
memo_string.byteslice(0, limit(memo_string.bytesize))
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
# A memo with no terminator runs to the end of the file, so stop
|
|
31
|
+
# reading blocks once max_size bytes have been read.
|
|
32
|
+
def full?(memo_string) # :nodoc:
|
|
33
|
+
max_size && memo_string.bytesize >= max_size
|
|
20
34
|
end
|
|
21
35
|
end
|
|
22
36
|
end
|
data/lib/dbf/memo/dbase4.rb
CHANGED
|
@@ -11,7 +11,7 @@ module DBF
|
|
|
11
11
|
header = data.read(BLOCK_HEADER_SIZE)
|
|
12
12
|
return nil unless header && header.bytesize == BLOCK_HEADER_SIZE
|
|
13
13
|
|
|
14
|
-
length = header.unpack1('x4L')
|
|
14
|
+
length = limit(header.unpack1('x4L'))
|
|
15
15
|
|
|
16
16
|
# Bound the read by the bytes remaining so a crafted 32-bit length
|
|
17
17
|
# field cannot force a ~4 GiB allocation from a small memo file.
|
data/lib/dbf/memo/foxpro.rb
CHANGED
|
@@ -5,18 +5,17 @@ module DBF
|
|
|
5
5
|
class Foxpro < Base
|
|
6
6
|
FPT_HEADER_SIZE = 512
|
|
7
7
|
|
|
8
|
-
def initialize(data, version)
|
|
9
|
-
@data = data
|
|
10
|
-
super
|
|
11
|
-
end
|
|
12
|
-
|
|
13
8
|
def build_memo(start_block) # :nodoc:
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
return nil unless
|
|
9
|
+
data.seek offset(start_block)
|
|
10
|
+
block = data.read(block_size)
|
|
11
|
+
return nil unless block
|
|
12
|
+
|
|
13
|
+
# memo_size is nil when the block header is truncated (< 8 bytes)
|
|
14
|
+
memo_type, memo_size, memo_string = block.unpack('NNa*')
|
|
15
|
+
return nil unless memo_type == 1 && memo_size.to_i.positive?
|
|
17
16
|
|
|
18
|
-
read_memo_content(memo_string, memo_size)
|
|
19
|
-
rescue
|
|
17
|
+
read_memo_content(memo_string, limit(memo_size))
|
|
18
|
+
rescue IOError, SystemCallError, RangeError
|
|
20
19
|
nil
|
|
21
20
|
end
|
|
22
21
|
|
|
@@ -28,16 +27,18 @@ module DBF
|
|
|
28
27
|
# Bound the read by the bytes remaining so a crafted 32-bit memo_size
|
|
29
28
|
# cannot force a ~4 GiB allocation from a small memo file.
|
|
30
29
|
length = content_size(memo_size)
|
|
31
|
-
remaining =
|
|
30
|
+
remaining = data.size - data.pos
|
|
32
31
|
length = remaining if length > remaining
|
|
33
|
-
memo_string <<
|
|
32
|
+
memo_string << data.read(length) if length.positive?
|
|
34
33
|
memo_string
|
|
35
34
|
end
|
|
36
35
|
|
|
37
36
|
def block_size # :nodoc:
|
|
38
37
|
@block_size ||= begin
|
|
39
|
-
|
|
40
|
-
|
|
38
|
+
data.rewind
|
|
39
|
+
header = data.read(FPT_HEADER_SIZE)
|
|
40
|
+
# A header shorter than 8 bytes cannot contain the block size field
|
|
41
|
+
header && header.bytesize >= 8 ? header.unpack1('x6n') : 0
|
|
41
42
|
end
|
|
42
43
|
end
|
|
43
44
|
end
|
data/lib/dbf/record.rb
CHANGED
|
@@ -3,6 +3,15 @@
|
|
|
3
3
|
module DBF
|
|
4
4
|
# An instance of DBF::Record represents a row in the DBF file
|
|
5
5
|
class Record
|
|
6
|
+
# Column names come from the file, so they are never exposed as these
|
|
7
|
+
# methods: Ruby (and Rails) call them implicitly to convert or render an
|
|
8
|
+
# object, which would let a crafted file control how a record behaves.
|
|
9
|
+
# Such columns remain readable through #[] and #attributes.
|
|
10
|
+
RESERVED_METHODS = %w[
|
|
11
|
+
to_str to_ary to_hash to_proc to_int to_io to_path to_regexp to_open
|
|
12
|
+
to_model to_partial_path to_param to_key
|
|
13
|
+
].freeze
|
|
14
|
+
|
|
6
15
|
# Initialize a new DBF::Record
|
|
7
16
|
#
|
|
8
17
|
# @param data [String, StringIO] data
|
|
@@ -20,9 +29,7 @@ module DBF
|
|
|
20
29
|
# @param [DBF::Record] other
|
|
21
30
|
# @return [Boolean]
|
|
22
31
|
def ==(other)
|
|
23
|
-
attributes == other.attributes
|
|
24
|
-
rescue NoMethodError
|
|
25
|
-
false
|
|
32
|
+
other.respond_to?(:attributes) && attributes == other.attributes
|
|
26
33
|
end
|
|
27
34
|
|
|
28
35
|
# Reads attributes by column name
|
|
@@ -114,7 +121,7 @@ module DBF
|
|
|
114
121
|
|
|
115
122
|
def method_missing(method, *args) # :nodoc:
|
|
116
123
|
key = method.to_s
|
|
117
|
-
if (index =
|
|
124
|
+
if (index = column_method_index(key))
|
|
118
125
|
if @context.column_offsets && !@to_a
|
|
119
126
|
column_value(index)
|
|
120
127
|
else
|
|
@@ -126,7 +133,13 @@ module DBF
|
|
|
126
133
|
end
|
|
127
134
|
|
|
128
135
|
def respond_to_missing?(method, *) # :nodoc:
|
|
129
|
-
|
|
136
|
+
!column_method_index(method.to_s).nil? || super
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def column_method_index(key) # :nodoc:
|
|
140
|
+
return nil if RESERVED_METHODS.include?(key)
|
|
141
|
+
|
|
142
|
+
underscored_column_names.index(key)
|
|
130
143
|
end
|
|
131
144
|
|
|
132
145
|
def underscored_column_names # :nodoc:
|
data/lib/dbf/record_context.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module DBF
|
|
4
|
-
|
|
4
|
+
# Immutable per-table state shared by every record during decoding
|
|
5
|
+
RecordContext = Data.define(:columns, :version, :memo, :column_offsets)
|
|
5
6
|
end
|
data/lib/dbf/record_iterator.rb
CHANGED
|
@@ -2,46 +2,75 @@
|
|
|
2
2
|
|
|
3
3
|
module DBF
|
|
4
4
|
class RecordIterator
|
|
5
|
-
|
|
5
|
+
# Records are read in chunks of whole records totalling roughly this many
|
|
6
|
+
# bytes, so enumerating a multi-gigabyte file (for example a shapefile
|
|
7
|
+
# sidecar) needs only chunk-sized memory instead of the entire record
|
|
8
|
+
# section at once.
|
|
9
|
+
CHUNK_SIZE = 4 * 1024 * 1024
|
|
10
|
+
|
|
11
|
+
def initialize(data, context, header_length, record_length, record_count, chunk_size: CHUNK_SIZE)
|
|
6
12
|
@data = data
|
|
7
13
|
@context = context
|
|
8
14
|
@header_length = header_length
|
|
9
15
|
@record_length = record_length
|
|
10
16
|
@record_count = record_count
|
|
17
|
+
@chunk_size = chunk_size
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def each(&)
|
|
21
|
+
return enum_for(:each) unless block_given?
|
|
22
|
+
|
|
23
|
+
# A record_length of 0 from a crafted header cannot drive an unbounded
|
|
24
|
+
# loop: capacity is 0 and enumeration ends immediately.
|
|
25
|
+
remaining = record_capacity
|
|
26
|
+
@data.seek(@header_length)
|
|
27
|
+
|
|
28
|
+
while remaining.positive?
|
|
29
|
+
wanted = [per_chunk, remaining].min
|
|
30
|
+
buffer = @data.read(wanted * @record_length)
|
|
31
|
+
break unless buffer
|
|
32
|
+
|
|
33
|
+
whole_records = buffer.bytesize / @record_length
|
|
34
|
+
break if whole_records.zero?
|
|
35
|
+
|
|
36
|
+
yield_chunk(buffer, whole_records, &)
|
|
37
|
+
remaining -= whole_records
|
|
38
|
+
|
|
39
|
+
# A short read means the file ended earlier than the header promised
|
|
40
|
+
break if whole_records < wanted
|
|
41
|
+
end
|
|
11
42
|
end
|
|
12
43
|
|
|
13
|
-
|
|
14
|
-
buf = read_buffer
|
|
15
|
-
return unless buf
|
|
44
|
+
private
|
|
16
45
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
46
|
+
# Whole records per read; at least one so a record larger than the chunk
|
|
47
|
+
# size still makes progress. Only called when record_length is positive.
|
|
48
|
+
def per_chunk
|
|
49
|
+
@per_chunk ||= [@chunk_size / @record_length, 1].max
|
|
50
|
+
end
|
|
21
51
|
|
|
52
|
+
def yield_chunk(buffer, count)
|
|
22
53
|
pos = 0
|
|
23
54
|
count.times do
|
|
24
|
-
if
|
|
55
|
+
if buffer.getbyte(pos) == 0x2A
|
|
25
56
|
yield nil
|
|
26
57
|
else
|
|
27
|
-
yield Record.new(
|
|
58
|
+
yield Record.new(buffer, @context, pos + 1)
|
|
28
59
|
end
|
|
29
60
|
pos += @record_length
|
|
30
61
|
end
|
|
31
62
|
end
|
|
32
63
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
64
|
+
# Bound enumeration by the bytes actually available so a crafted header
|
|
65
|
+
# (huge record_length * record_count) cannot force reads past the real
|
|
66
|
+
# file size, while record_count still caps a file with trailing garbage.
|
|
67
|
+
def record_capacity
|
|
68
|
+
return 0 unless @record_length.positive?
|
|
37
69
|
|
|
38
|
-
# Bound the allocation by the bytes actually available so a crafted
|
|
39
|
-
# header (huge record_length * record_count) cannot force a giant read
|
|
40
|
-
# from a tiny file.
|
|
41
|
-
requested = @record_length * @record_count
|
|
42
70
|
available = @data.size - @header_length
|
|
43
|
-
|
|
44
|
-
|
|
71
|
+
return 0 if available.negative?
|
|
72
|
+
|
|
73
|
+
[@record_count, available / @record_length].min
|
|
45
74
|
end
|
|
46
75
|
end
|
|
47
76
|
end
|
data/lib/dbf/schema.rb
CHANGED
|
@@ -43,10 +43,9 @@ module DBF
|
|
|
43
43
|
# @param table_only [Boolean]
|
|
44
44
|
# @return [String]
|
|
45
45
|
def schema(format = :activerecord, table_only: false)
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
raise ArgumentError, ":#{format} is not a valid schema. Valid schemas are: #{FORMATS.join(', ')}."
|
|
46
|
+
raise ArgumentError, ":#{format} is not a valid schema. Valid schemas are: #{FORMATS.join(', ')}." unless FORMATS.include?(format.to_s.to_sym)
|
|
47
|
+
|
|
48
|
+
send(schema_name(format), table_only: table_only)
|
|
50
49
|
end
|
|
51
50
|
|
|
52
51
|
def schema_name(format) # :nodoc:
|