dbf 5.4.0 → 5.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,13 +20,16 @@ module DBF
20
20
  # contacts = db.contacts.record(0)
21
21
  #
22
22
  # @param path [String]
23
- def initialize(path)
23
+ # @param max_memo_size [optional Integer] Bytes to read from each memo
24
+ # of the container and its tables; see DBF::Table.new
25
+ def initialize(path, max_memo_size: nil)
24
26
  @path = path
25
27
  @dirname = File.dirname(@path)
26
- @db = DBF::Table.new(@path)
28
+ @max_memo_size = max_memo_size
29
+ @db = DBF::Table.new(@path, max_memo_size:)
27
30
  @tables = extract_dbc_data
28
31
  rescue Errno::ENOENT
29
- raise DBF::FileNotFoundError, "file not found: #{path}"
32
+ raise FileHandler.not_found(path)
30
33
  end
31
34
 
32
35
  def table_names
@@ -38,7 +41,7 @@ module DBF
38
41
  # @param name [String]
39
42
  # @return [DBF::Table]
40
43
  def table(name)
41
- Table.new(table_path(name), long_names: @tables[name])
44
+ Table.new(table_path(name), long_names: @tables[name], max_memo_size: @max_memo_size)
42
45
  end
43
46
 
44
47
  # Searches the database directory for the table's dbf file
@@ -71,15 +74,21 @@ module DBF
71
74
 
72
75
  def method_missing(method, *args) # :nodoc:
73
76
  name = method.to_s
74
- table_names.index(name) ? table(name) : super
77
+ table_method?(name) ? table(name) : super
75
78
  end
76
79
 
77
80
  def respond_to_missing?(method, *)
78
- table_names.index(method.to_s) || super
81
+ table_method?(method.to_s) || super
79
82
  end
80
83
 
81
84
  private
82
85
 
86
+ # Table names come from the container file, so implicit-conversion
87
+ # method names are never dispatched to a table (see Record).
88
+ def table_method?(name) # :nodoc:
89
+ !Record::RESERVED_METHODS.include?(name) && table_names.include?(name)
90
+ end
91
+
83
92
  # This method extracts the data from the database container. This is
84
93
  # just an ordinary table with a treelike structure. Field definitions
85
94
  # are in the same order as in the linked tables but only the long name
@@ -108,19 +117,18 @@ module DBF
108
117
  class Table < DBF::Table
109
118
  attr_reader :long_names
110
119
 
111
- def initialize(path, long_names:)
120
+ def initialize(path, long_names:, max_memo_size: nil)
112
121
  @long_names = long_names
113
- super(path)
122
+ super(path, max_memo_size:)
114
123
  end
115
124
 
125
+ # Replaces each column's short name with its long name from the
126
+ # container. Long names were decoded when the .dbc was read, so they
127
+ # are not decoded again with this table's code page.
116
128
  def build_columns # :nodoc:
117
- columns = super
118
-
119
- # modify the column definitions to use the long names as the
120
- # columnname property is readonly, recreate the column definitions
121
- columns.map do |column|
122
- long_name = long_names[columns.index(column)]
123
- Column.new(self, long_name, column.type, column.length, column.decimal)
129
+ super.each_with_index.map do |column, index|
130
+ long_name = long_names[index]
131
+ long_name ? column.with_name(long_name) : column
124
132
  end
125
133
  end
126
134
  end
data/lib/dbf/encoder.rb CHANGED
@@ -4,12 +4,15 @@ module DBF
4
4
  # Transcodes strings from a source encoding to a target encoding.
5
5
  #
6
6
  # Most DBF code pages map to encodings built into Ruby and are converted
7
- # with String#encode. A few (Mazovia cp620, Kamenický cp895) have no Ruby
8
- # equivalent and are decoded through vendored byte-to-Unicode tables.
7
+ # with String#encode. A few (Mazovia cp620, Kamenický cp895, Macintosh
8
+ # Central European) have no Ruby converter and are decoded through
9
+ # vendored byte-to-Unicode tables.
9
10
  module Encoder
10
11
  # Byte-to-Unicode tables generated from
11
- # https://github.com/SheetJS/js-codepage/blob/master/codepages/620.TBL and
12
- # https://github.com/SheetJS/js-codepage/blob/master/codepages/895.TBL
12
+ # https://github.com/SheetJS/js-codepage/blob/master/codepages/620.TBL,
13
+ # https://github.com/SheetJS/js-codepage/blob/master/codepages/895.TBL and
14
+ # (Mac Central European) Python's mac_latin2 codec, which matches iconv's
15
+ # MACCENTRALEUROPE
13
16
 
14
17
  # Mazovia (Polish) MS-DOS
15
18
  CP620_TO_UNICODE = [
@@ -83,9 +86,46 @@ module DBF
83
86
  0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, 0x25A0, 0x00A0
84
87
  ].freeze
85
88
 
89
+ # Macintosh Central European (Ruby has the encoding but no converter)
90
+ MACCENTEURO_TO_UNICODE = [
91
+ 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
92
+ 0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F,
93
+ 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017,
94
+ 0x0018, 0x0019, 0x001A, 0x001B, 0x001C, 0x001D, 0x001E, 0x001F,
95
+ 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027,
96
+ 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F,
97
+ 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037,
98
+ 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F,
99
+ 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
100
+ 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F,
101
+ 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057,
102
+ 0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F,
103
+ 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067,
104
+ 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F,
105
+ 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077,
106
+ 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0x007F,
107
+ 0x00C4, 0x0100, 0x0101, 0x00C9, 0x0104, 0x00D6, 0x00DC, 0x00E1,
108
+ 0x0105, 0x010C, 0x00E4, 0x010D, 0x0106, 0x0107, 0x00E9, 0x0179,
109
+ 0x017A, 0x010E, 0x00ED, 0x010F, 0x0112, 0x0113, 0x0116, 0x00F3,
110
+ 0x0117, 0x00F4, 0x00F6, 0x00F5, 0x00FA, 0x011A, 0x011B, 0x00FC,
111
+ 0x2020, 0x00B0, 0x0118, 0x00A3, 0x00A7, 0x2022, 0x00B6, 0x00DF,
112
+ 0x00AE, 0x00A9, 0x2122, 0x0119, 0x00A8, 0x2260, 0x0123, 0x012E,
113
+ 0x012F, 0x012A, 0x2264, 0x2265, 0x012B, 0x0136, 0x2202, 0x2211,
114
+ 0x0142, 0x013B, 0x013C, 0x013D, 0x013E, 0x0139, 0x013A, 0x0145,
115
+ 0x0146, 0x0143, 0x00AC, 0x221A, 0x0144, 0x0147, 0x2206, 0x00AB,
116
+ 0x00BB, 0x2026, 0x00A0, 0x0148, 0x0150, 0x00D5, 0x0151, 0x014C,
117
+ 0x2013, 0x2014, 0x201C, 0x201D, 0x2018, 0x2019, 0x00F7, 0x25CA,
118
+ 0x014D, 0x0154, 0x0155, 0x0158, 0x2039, 0x203A, 0x0159, 0x0156,
119
+ 0x0157, 0x0160, 0x201A, 0x201E, 0x0161, 0x015A, 0x015B, 0x00C1,
120
+ 0x0164, 0x0165, 0x00CD, 0x017D, 0x017E, 0x016A, 0x00D3, 0x00D4,
121
+ 0x016B, 0x016E, 0x00DA, 0x016F, 0x0170, 0x0171, 0x0172, 0x0173,
122
+ 0x00DD, 0x00FD, 0x0137, 0x017B, 0x0141, 0x017C, 0x0122, 0x02C7
123
+ ].freeze
124
+
86
125
  CUSTOM_TABLES = {
87
126
  'cp620' => CP620_TO_UNICODE,
88
- 'cp895' => CP895_TO_UNICODE
127
+ 'cp895' => CP895_TO_UNICODE,
128
+ 'maccenteuro' => MACCENTEURO_TO_UNICODE
89
129
  }.freeze
90
130
 
91
131
  # Is the encoding a code page without a Ruby Encoding, decoded through
@@ -97,6 +137,21 @@ module DBF
97
137
  encoding.is_a?(::String) && CUSTOM_TABLES.key?(encoding.downcase)
98
138
  end
99
139
 
140
+ # Raise unless strings in the encoding can be transcoded to Unicode, so
141
+ # an unknown name or a dummy encoding such as UTF-7 fails up front with
142
+ # a DBF::Error rather than with ArgumentError or
143
+ # Encoding::ConverterNotFoundError on the first read.
144
+ #
145
+ # @param encoding [String, Encoding]
146
+ # @raise [DBF::UnsupportedEncodingError]
147
+ def self.check(encoding)
148
+ return if custom?(encoding)
149
+
150
+ (+'').force_encoding(encoding).encode(Encoding::UTF_8)
151
+ rescue ArgumentError, TypeError, Encoding::ConverterNotFoundError
152
+ raise DBF::UnsupportedEncodingError, "unsupported encoding: #{encoding}"
153
+ end
154
+
100
155
  # Transcode a string from source_encoding to target_encoding
101
156
  #
102
157
  # @param string [String]
data/lib/dbf/errors.rb ADDED
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DBF
4
+ # Base class for all errors raised by the DBF library, so callers can
5
+ # rescue DBF::Error to catch anything the library raises.
6
+ class Error < StandardError
7
+ end
8
+
9
+ # A file cannot be opened. The message names only the file, not its
10
+ # directory, so an app that shows error messages to users does not reveal
11
+ # server paths; #path has the full path for logging.
12
+ class FileError < Error
13
+ # @return [String, nil] the full path of the file
14
+ attr_reader :path
15
+
16
+ def initialize(message = nil, path: nil)
17
+ super(message)
18
+ @path = path
19
+ end
20
+ end
21
+
22
+ class FileNotFoundError < FileError
23
+ end
24
+
25
+ # A file exists but cannot be opened, e.g. it is a directory or is not
26
+ # readable.
27
+ class FileAccessError < FileError
28
+ end
29
+
30
+ # The encoding passed to Table.new is unknown, or Ruby cannot transcode
31
+ # it to Unicode.
32
+ class UnsupportedEncodingError < Error
33
+ end
34
+
35
+ class NoColumnsDefined < Error
36
+ end
37
+ end
@@ -9,28 +9,76 @@ module DBF
9
9
  when StringIO
10
10
  data
11
11
  when String
12
- File.open(data, 'rb')
12
+ open_file(data)
13
13
  else
14
- raise ArgumentError, 'data must be a file path or StringIO object'
14
+ raise ArgumentError, 'data must be a file path or an IO-like object responding to #read and #seek' unless data.respond_to?(:read) && data.respond_to?(:seek)
15
+
16
+ # DBF is a binary format; a File opened in text mode would corrupt
17
+ # reads on Windows.
18
+ data.binmode if data.respond_to?(:binmode)
19
+ data
15
20
  end
16
- rescue Errno::ENOENT
17
- raise DBF::FileNotFoundError, "file not found: #{data}"
18
21
  end
19
22
 
20
- def open_memo(data, memo, memo_class, version)
23
+ # Opens a file for binary reading, raising DBF errors instead of the
24
+ # system errors File.open raises, so callers need only rescue DBF::Error.
25
+ # A path containing a null byte cannot name a file, so it is not found.
26
+ # A directory opens without error but fails on the first read, so it is
27
+ # rejected here.
28
+ def open_file(path)
29
+ raise Errno::EISDIR if File.directory?(path)
30
+
31
+ File.open(path, 'rb')
32
+ rescue Errno::ENOENT, Errno::ENOTDIR, ArgumentError
33
+ raise not_found(path)
34
+ rescue SystemCallError => e
35
+ reason = SystemCallError.new(nil, e.errno).message
36
+ raise DBF::FileAccessError.new("cannot open file: #{display_name(path)} (#{reason})", path:)
37
+ end
38
+
39
+ def not_found(path)
40
+ DBF::FileNotFoundError.new("file not found: #{display_name(path)}", path:)
41
+ end
42
+
43
+ # The file name without its directory, for error messages. A null byte,
44
+ # which File.basename rejects, is shown as "?".
45
+ def display_name(path)
46
+ File.basename(path.to_s.tr("\x00", '?'))
47
+ end
48
+
49
+ def open_memo(data, memo, memo_class, version, max_size: nil)
21
50
  if memo
22
- meth = memo.is_a?(StringIO) ? :new : :open
23
- memo_class.send(meth, memo, version)
24
- elsif !data.is_a?(StringIO)
25
- path = Dir.glob(memo_search_path(data)).first
26
- path && memo_class.open(path, version)
51
+ meth = memo.is_a?(String) ? :open : :new
52
+ memo_class.send(meth, memo, version, max_size:)
53
+ elsif (path = data_path(data))
54
+ found = find_memo(path)
55
+ found && memo_class.open(found, version, max_size:)
27
56
  end
28
57
  end
29
58
 
30
- def memo_search_path(io)
31
- dirname = File.dirname(io)
32
- basename = File.basename(io, '.*')
33
- "#{dirname}/#{basename}*.{fpt,FPT,dbt,DBT}"
59
+ def data_path(data)
60
+ return data if data.is_a?(String)
61
+
62
+ data.path if data.respond_to?(:path)
63
+ end
64
+
65
+ MEMO_EXTENSIONS = %w[fpt dbt].freeze
66
+
67
+ # Finds the memo file sitting next to the data file: same basename, with
68
+ # an .fpt or .dbt extension in any case. The directory is scanned rather
69
+ # than globbed so glob metacharacters in an uploaded filename cannot be
70
+ # expanded, and the basename must match exactly so data.dbf never picks
71
+ # up an unrelated memo such as data2.fpt.
72
+ def find_memo(path)
73
+ dirname = File.dirname(path)
74
+ basename = File.basename(path, '.*')
75
+ entries = Dir.children(dirname).sort.each_with_object({}) do |child, index|
76
+ index[child.downcase(:ascii)] ||= child
77
+ end
78
+ entry = MEMO_EXTENSIONS.filter_map { |ext| entries["#{basename}.#{ext}".downcase(:ascii)] }.first
79
+ entry && File.join(dirname, entry)
80
+ rescue SystemCallError
81
+ nil
34
82
  end
35
83
  end
36
84
  end
data/lib/dbf/header.rb CHANGED
@@ -2,14 +2,12 @@
2
2
 
3
3
  module DBF
4
4
  class Header
5
- HEADER_SIZE = 32
6
-
7
5
  attr_reader :version, :record_count, :header_length, :record_length, :encoding_key, :encoding
8
6
 
9
7
  def initialize(data)
10
8
  # Pad a nil or truncated header read so unpacking a short file yields
11
9
  # empty values instead of raising.
12
- data = data.to_s.b.ljust(HEADER_SIZE, "\x00")
10
+ data = data.to_s.b.ljust(VersionConfig::DBASE3_HEADER_SIZE, "\x00")
13
11
  @version = data.unpack1('H2')
14
12
  @encoding_key = nil
15
13
  @encoding = nil
data/lib/dbf/memo/base.rb CHANGED
@@ -6,19 +6,25 @@ module DBF
6
6
  BLOCK_HEADER_SIZE = 8
7
7
  BLOCK_SIZE = 512
8
8
 
9
- def self.open(filename, version)
10
- new(File.open(filename, 'rb'), version)
9
+ def self.open(filename, version, max_size: nil)
10
+ new(FileHandler.open_file(filename), version, max_size:)
11
11
  end
12
12
 
13
- def initialize(data, version)
13
+ # @param max_size [Integer, nil] bytes to read from each memo; longer
14
+ # memos are cut off. nil reads every memo in full.
15
+ def initialize(data, version, max_size: nil)
14
16
  @data = data
15
17
  @version = version
18
+ @max_size = max_size
16
19
  end
17
20
 
18
21
  def get(start_block)
19
22
  return nil unless start_block > 0
20
23
 
21
24
  build_memo start_block
25
+ rescue RangeError
26
+ # A corrupt memo pointer too large to seek to has no memo
27
+ nil
22
28
  end
23
29
 
24
30
  def close
@@ -31,7 +37,12 @@ module DBF
31
37
 
32
38
  private
33
39
 
34
- attr_reader :data
40
+ attr_reader :data, :max_size
41
+
42
+ # Bounds a memo length by max_size, when one is set
43
+ def limit(length) # :nodoc:
44
+ max_size && length > max_size ? max_size : length
45
+ end
35
46
 
36
47
  def offset(start_block) # :nodoc:
37
48
  start_block * block_size
@@ -3,6 +3,9 @@
3
3
  module DBF
4
4
  module Memo
5
5
  class Dbase3 < Base
6
+ # Memo text up to, and whether it reached, the 0x1A terminator
7
+ TERMINATOR_PATTERN = /\A([^\x1A]*)(\x1A)?/n
8
+
6
9
  def build_memo(start_block) # :nodoc:
7
10
  data.seek offset(start_block)
8
11
  memo_string = +''
@@ -12,11 +15,22 @@ module DBF
12
15
  # than crashing on nil.gsub.
13
16
  break if block.nil?
14
17
 
15
- block = block.gsub(/(\000|\032)/, '')
16
- memo_string << block
17
- break if block.size < BLOCK_SIZE
18
+ # The memo ends at its 0x1A terminator; bytes after it in the same
19
+ # block are leftovers, not memo text.
20
+ text, terminated = TERMINATOR_PATTERN.match(block).captures
21
+ text = text.delete("\000")
22
+ memo_string << text
23
+ break if terminated || text.size < BLOCK_SIZE || full?(memo_string)
18
24
  end
19
- memo_string
25
+ memo_string.byteslice(0, limit(memo_string.bytesize))
26
+ end
27
+
28
+ private
29
+
30
+ # A memo with no terminator runs to the end of the file, so stop
31
+ # reading blocks once max_size bytes have been read.
32
+ def full?(memo_string) # :nodoc:
33
+ max_size && memo_string.bytesize >= max_size
20
34
  end
21
35
  end
22
36
  end
@@ -11,7 +11,7 @@ module DBF
11
11
  header = data.read(BLOCK_HEADER_SIZE)
12
12
  return nil unless header && header.bytesize == BLOCK_HEADER_SIZE
13
13
 
14
- length = header.unpack1('x4L')
14
+ length = limit(header.unpack1('x4L'))
15
15
 
16
16
  # Bound the read by the bytes remaining so a crafted 32-bit length
17
17
  # field cannot force a ~4 GiB allocation from a small memo file.
@@ -5,18 +5,17 @@ module DBF
5
5
  class Foxpro < Base
6
6
  FPT_HEADER_SIZE = 512
7
7
 
8
- def initialize(data, version)
9
- @data = data
10
- super
11
- end
12
-
13
8
  def build_memo(start_block) # :nodoc:
14
- @data.seek offset(start_block)
15
- memo_type, memo_size, memo_string = @data.read(block_size).unpack('NNa*')
16
- return nil unless memo_type == 1 && memo_size > 0
9
+ data.seek offset(start_block)
10
+ block = data.read(block_size)
11
+ return nil unless block
12
+
13
+ # memo_size is nil when the block header is truncated (< 8 bytes)
14
+ memo_type, memo_size, memo_string = block.unpack('NNa*')
15
+ return nil unless memo_type == 1 && memo_size.to_i.positive?
17
16
 
18
- read_memo_content(memo_string, memo_size)
19
- rescue StandardError
17
+ read_memo_content(memo_string, limit(memo_size))
18
+ rescue IOError, SystemCallError, RangeError
20
19
  nil
21
20
  end
22
21
 
@@ -28,16 +27,18 @@ module DBF
28
27
  # Bound the read by the bytes remaining so a crafted 32-bit memo_size
29
28
  # cannot force a ~4 GiB allocation from a small memo file.
30
29
  length = content_size(memo_size)
31
- remaining = @data.size - @data.pos
30
+ remaining = data.size - data.pos
32
31
  length = remaining if length > remaining
33
- memo_string << @data.read(length) if length.positive?
32
+ memo_string << data.read(length) if length.positive?
34
33
  memo_string
35
34
  end
36
35
 
37
36
  def block_size # :nodoc:
38
37
  @block_size ||= begin
39
- @data.rewind
40
- @data.read(FPT_HEADER_SIZE).unpack1('x6n') || 0
38
+ data.rewind
39
+ header = data.read(FPT_HEADER_SIZE)
40
+ # A header shorter than 8 bytes cannot contain the block size field
41
+ header && header.bytesize >= 8 ? header.unpack1('x6n') : 0
41
42
  end
42
43
  end
43
44
  end
data/lib/dbf/record.rb CHANGED
@@ -3,6 +3,15 @@
3
3
  module DBF
4
4
  # An instance of DBF::Record represents a row in the DBF file
5
5
  class Record
6
+ # Column names come from the file, so they are never exposed as these
7
+ # methods: Ruby (and Rails) call them implicitly to convert or render an
8
+ # object, which would let a crafted file control how a record behaves.
9
+ # Such columns remain readable through #[] and #attributes.
10
+ RESERVED_METHODS = %w[
11
+ to_str to_ary to_hash to_proc to_int to_io to_path to_regexp to_open
12
+ to_model to_partial_path to_param to_key
13
+ ].freeze
14
+
6
15
  # Initialize a new DBF::Record
7
16
  #
8
17
  # @param data [String, StringIO] data
@@ -20,9 +29,7 @@ module DBF
20
29
  # @param [DBF::Record] other
21
30
  # @return [Boolean]
22
31
  def ==(other)
23
- attributes == other.attributes
24
- rescue NoMethodError
25
- false
32
+ other.respond_to?(:attributes) && attributes == other.attributes
26
33
  end
27
34
 
28
35
  # Reads attributes by column name
@@ -114,7 +121,7 @@ module DBF
114
121
 
115
122
  def method_missing(method, *args) # :nodoc:
116
123
  key = method.to_s
117
- if (index = underscored_column_names.index(key))
124
+ if (index = column_method_index(key))
118
125
  if @context.column_offsets && !@to_a
119
126
  column_value(index)
120
127
  else
@@ -126,7 +133,13 @@ module DBF
126
133
  end
127
134
 
128
135
  def respond_to_missing?(method, *) # :nodoc:
129
- underscored_column_names.include?(method.to_s) || super
136
+ !column_method_index(method.to_s).nil? || super
137
+ end
138
+
139
+ def column_method_index(key) # :nodoc:
140
+ return nil if RESERVED_METHODS.include?(key)
141
+
142
+ underscored_column_names.index(key)
130
143
  end
131
144
 
132
145
  def underscored_column_names # :nodoc:
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module DBF
4
- RecordContext = Struct.new(:columns, :version, :memo, :column_offsets, keyword_init: true)
4
+ # Immutable per-table state shared by every record during decoding
5
+ RecordContext = Data.define(:columns, :version, :memo, :column_offsets)
5
6
  end
@@ -2,46 +2,75 @@
2
2
 
3
3
  module DBF
4
4
  class RecordIterator
5
- def initialize(data, context, header_length, record_length, record_count)
5
+ # Records are read in chunks of whole records totalling roughly this many
6
+ # bytes, so enumerating a multi-gigabyte file (for example a shapefile
7
+ # sidecar) needs only chunk-sized memory instead of the entire record
8
+ # section at once.
9
+ CHUNK_SIZE = 4 * 1024 * 1024
10
+
11
+ def initialize(data, context, header_length, record_length, record_count, chunk_size: CHUNK_SIZE)
6
12
  @data = data
7
13
  @context = context
8
14
  @header_length = header_length
9
15
  @record_length = record_length
10
16
  @record_count = record_count
17
+ @chunk_size = chunk_size
18
+ end
19
+
20
+ def each(&)
21
+ return enum_for(:each) unless block_given?
22
+
23
+ # A record_length of 0 from a crafted header cannot drive an unbounded
24
+ # loop: capacity is 0 and enumeration ends immediately.
25
+ remaining = record_capacity
26
+ @data.seek(@header_length)
27
+
28
+ while remaining.positive?
29
+ wanted = [per_chunk, remaining].min
30
+ buffer = @data.read(wanted * @record_length)
31
+ break unless buffer
32
+
33
+ whole_records = buffer.bytesize / @record_length
34
+ break if whole_records.zero?
35
+
36
+ yield_chunk(buffer, whole_records, &)
37
+ remaining -= whole_records
38
+
39
+ # A short read means the file ended earlier than the header promised
40
+ break if whole_records < wanted
41
+ end
11
42
  end
12
43
 
13
- def each
14
- buf = read_buffer
15
- return unless buf
44
+ private
16
45
 
17
- # Bound the iteration by the bytes actually read so a crafted
18
- # record_count (or record_length == 0) cannot drive an unbounded loop.
19
- max_records = @record_length > 0 ? buf.bytesize / @record_length : 0
20
- count = @record_count < max_records ? @record_count : max_records
46
+ # Whole records per read; at least one so a record larger than the chunk
47
+ # size still makes progress. Only called when record_length is positive.
48
+ def per_chunk
49
+ @per_chunk ||= [@chunk_size / @record_length, 1].max
50
+ end
21
51
 
52
+ def yield_chunk(buffer, count)
22
53
  pos = 0
23
54
  count.times do
24
- if buf.getbyte(pos) == 0x2A
55
+ if buffer.getbyte(pos) == 0x2A
25
56
  yield nil
26
57
  else
27
- yield Record.new(buf, @context, pos + 1)
58
+ yield Record.new(buffer, @context, pos + 1)
28
59
  end
29
60
  pos += @record_length
30
61
  end
31
62
  end
32
63
 
33
- private
34
-
35
- def read_buffer
36
- @data.seek(@header_length)
64
+ # Bound enumeration by the bytes actually available so a crafted header
65
+ # (huge record_length * record_count) cannot force reads past the real
66
+ # file size, while record_count still caps a file with trailing garbage.
67
+ def record_capacity
68
+ return 0 unless @record_length.positive?
37
69
 
38
- # Bound the allocation by the bytes actually available so a crafted
39
- # header (huge record_length * record_count) cannot force a giant read
40
- # from a tiny file.
41
- requested = @record_length * @record_count
42
70
  available = @data.size - @header_length
43
- available = 0 if available.negative?
44
- @data.read(requested < available ? requested : available)
71
+ return 0 if available.negative?
72
+
73
+ [@record_count, available / @record_length].min
45
74
  end
46
75
  end
47
76
  end
data/lib/dbf/schema.rb CHANGED
@@ -43,10 +43,9 @@ module DBF
43
43
  # @param table_only [Boolean]
44
44
  # @return [String]
45
45
  def schema(format = :activerecord, table_only: false)
46
- schema_method_name = schema_name(format)
47
- send(schema_method_name, table_only: table_only)
48
- rescue NameError
49
- raise ArgumentError, ":#{format} is not a valid schema. Valid schemas are: #{FORMATS.join(', ')}."
46
+ raise ArgumentError, ":#{format} is not a valid schema. Valid schemas are: #{FORMATS.join(', ')}." unless FORMATS.include?(format.to_s.to_sym)
47
+
48
+ send(schema_name(format), table_only: table_only)
50
49
  end
51
50
 
52
51
  def schema_name(format) # :nodoc: