fastercsv 1.4.0 → 1.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +37 -0
- data/INSTALL +4 -6
- data/LICENSE +2 -0
- data/Rakefile +5 -6
- data/examples/shortcut_interface.rb +1 -1
- data/lib/faster_csv.rb +1814 -1786
- data/lib/fastercsv.rb +1 -1
- data/test/line_endings.gz +0 -0
- data/test/tc_csv_parsing.rb +29 -2
- data/test/tc_csv_writing.rb +1 -1
- data/test/tc_data_converters.rb +1 -1
- data/test/tc_encodings.rb +1 -1
- data/test/tc_features.rb +1 -1
- data/test/tc_headers.rb +1 -1
- data/test/tc_interface.rb +18 -2
- data/test/tc_row.rb +1 -1
- data/test/tc_serialization.rb +1 -1
- data/test/tc_speed.rb +1 -1
- data/test/tc_table.rb +9 -1
- data/test/ts_all.rb +1 -1
- metadata +34 -32
- data/setup.rb +0 -1360
data/lib/faster_csv.rb
CHANGED
@@ -7,1732 +7,1766 @@
|
|
7
7
|
#
|
8
8
|
# See FasterCSV for documentation.
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
10
|
+
if RUBY_VERSION >= "1.9"
|
11
|
+
class FasterCSV
|
12
|
+
def self.const_missing(*_)
|
13
|
+
raise NotImplementedError, "Please switch to Ruby 1.9's standard CSV " +
|
14
|
+
"library. It's FasterCSV plus support for " +
|
15
|
+
"Ruby 1.9's m17n encoding engine."
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.method_missing(*_)
|
19
|
+
const_missing
|
20
|
+
end
|
21
|
+
|
22
|
+
def method_missing(*_)
|
23
|
+
self.class.const_missing
|
24
|
+
end
|
25
|
+
end
|
26
|
+
else
|
27
|
+
require "forwardable"
|
28
|
+
require "English"
|
29
|
+
require "enumerator"
|
30
|
+
require "date"
|
31
|
+
require "stringio"
|
15
32
|
|
16
|
-
#
|
17
|
-
# This class provides a complete interface to CSV files and data. It offers
|
18
|
-
# tools to enable you to read and write to and from Strings or IO objects, as
|
19
|
-
# needed.
|
20
|
-
#
|
21
|
-
# == Reading
|
22
|
-
#
|
23
|
-
# === From a File
|
24
|
-
#
|
25
|
-
# ==== A Line at a Time
|
26
|
-
#
|
27
|
-
# FasterCSV.foreach("path/to/file.csv") do |row|
|
28
|
-
# # use row here...
|
29
|
-
# end
|
30
|
-
#
|
31
|
-
# ==== All at Once
|
32
|
-
#
|
33
|
-
# arr_of_arrs = FasterCSV.read("path/to/file.csv")
|
34
|
-
#
|
35
|
-
# === From a String
|
36
|
-
#
|
37
|
-
# ==== A Line at a Time
|
38
|
-
#
|
39
|
-
# FasterCSV.parse("CSV,data,String") do |row|
|
40
|
-
# # use row here...
|
41
|
-
# end
|
42
|
-
#
|
43
|
-
# ==== All at Once
|
44
|
-
#
|
45
|
-
# arr_of_arrs = FasterCSV.parse("CSV,data,String")
|
46
|
-
#
|
47
|
-
# == Writing
|
48
|
-
#
|
49
|
-
# === To a File
|
50
|
-
#
|
51
|
-
# FasterCSV.open("path/to/file.csv", "w") do |csv|
|
52
|
-
# csv << ["row", "of", "CSV", "data"]
|
53
|
-
# csv << ["another", "row"]
|
54
|
-
# # ...
|
55
|
-
# end
|
56
|
-
#
|
57
|
-
# === To a String
|
58
|
-
#
|
59
|
-
# csv_string = FasterCSV.generate do |csv|
|
60
|
-
# csv << ["row", "of", "CSV", "data"]
|
61
|
-
# csv << ["another", "row"]
|
62
|
-
# # ...
|
63
|
-
# end
|
64
|
-
#
|
65
|
-
# == Convert a Single Line
|
66
|
-
#
|
67
|
-
# csv_string = ["CSV", "data"].to_csv # to CSV
|
68
|
-
# csv_array = "CSV,String".parse_csv # from CSV
|
69
|
-
#
|
70
|
-
# == Shortcut Interface
|
71
|
-
#
|
72
|
-
# FCSV { |csv_out| csv_out << %w{my data here} } # to $stdout
|
73
|
-
# FCSV(csv = "") { |csv_str| csv_str << %w{my data here} } # to a String
|
74
|
-
# FCSV($stderr) { |csv_err| csv_err << %w{my data here} } # to $stderr
|
75
|
-
#
|
76
|
-
class FasterCSV
|
77
|
-
# The version of the installed library.
|
78
|
-
VERSION = "1.4.0".freeze
|
79
|
-
|
80
33
|
#
|
81
|
-
#
|
82
|
-
#
|
83
|
-
#
|
34
|
+
# This class provides a complete interface to CSV files and data. It offers
|
35
|
+
# tools to enable you to read and write to and from Strings or IO objects, as
|
36
|
+
# needed.
|
84
37
|
#
|
85
|
-
#
|
86
|
-
# header row processing is activated.
|
38
|
+
# == Reading
|
87
39
|
#
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
40
|
+
# === From a File
|
41
|
+
#
|
42
|
+
# ==== A Line at a Time
|
43
|
+
#
|
44
|
+
# FasterCSV.foreach("path/to/file.csv") do |row|
|
45
|
+
# # use row here...
|
46
|
+
# end
|
47
|
+
#
|
48
|
+
# ==== All at Once
|
49
|
+
#
|
50
|
+
# arr_of_arrs = FasterCSV.read("path/to/file.csv")
|
51
|
+
#
|
52
|
+
# === From a String
|
53
|
+
#
|
54
|
+
# ==== A Line at a Time
|
55
|
+
#
|
56
|
+
# FasterCSV.parse("CSV,data,String") do |row|
|
57
|
+
# # use row here...
|
58
|
+
# end
|
59
|
+
#
|
60
|
+
# ==== All at Once
|
61
|
+
#
|
62
|
+
# arr_of_arrs = FasterCSV.parse("CSV,data,String")
|
63
|
+
#
|
64
|
+
# == Writing
|
65
|
+
#
|
66
|
+
# === To a File
|
67
|
+
#
|
68
|
+
# FasterCSV.open("path/to/file.csv", "w") do |csv|
|
69
|
+
# csv << ["row", "of", "CSV", "data"]
|
70
|
+
# csv << ["another", "row"]
|
71
|
+
# # ...
|
72
|
+
# end
|
73
|
+
#
|
74
|
+
# === To a String
|
75
|
+
#
|
76
|
+
# csv_string = FasterCSV.generate do |csv|
|
77
|
+
# csv << ["row", "of", "CSV", "data"]
|
78
|
+
# csv << ["another", "row"]
|
79
|
+
# # ...
|
80
|
+
# end
|
81
|
+
#
|
82
|
+
# == Convert a Single Line
|
83
|
+
#
|
84
|
+
# csv_string = ["CSV", "data"].to_csv # to CSV
|
85
|
+
# csv_array = "CSV,String".parse_csv # from CSV
|
86
|
+
#
|
87
|
+
# == Shortcut Interface
|
88
|
+
#
|
89
|
+
# FCSV { |csv_out| csv_out << %w{my data here} } # to $stdout
|
90
|
+
# FCSV(csv = "") { |csv_str| csv_str << %w{my data here} } # to a String
|
91
|
+
# FCSV($stderr) { |csv_err| csv_err << %w{my data here} } # to $stderr
|
92
|
+
# FCSV($stdin) { |csv_in| csv_in.each { |row| p row } } # from $stdin
|
93
|
+
#
|
94
|
+
# == Advanced Usage
|
95
|
+
#
|
96
|
+
# === Wrap an IO Object
|
97
|
+
#
|
98
|
+
# csv = FCSV.new(io, options)
|
99
|
+
# # ... read (with gets() or each()) from and write (with <<) to csv here ...
|
100
|
+
#
|
101
|
+
class FasterCSV
|
102
|
+
# The version of the installed library.
|
103
|
+
VERSION = "1.5.5".freeze
|
104
|
+
|
97
105
|
#
|
98
|
-
# A FasterCSV::Row
|
99
|
-
#
|
106
|
+
# A FasterCSV::Row is part Array and part Hash. It retains an order for the
|
107
|
+
# fields and allows duplicates just as an Array would, but also allows you to
|
108
|
+
# access fields by name just as you could if they were in a Hash.
|
100
109
|
#
|
101
|
-
#
|
102
|
-
#
|
103
|
-
# * size()
|
110
|
+
# All rows returned by FasterCSV will be constructed from this class, if
|
111
|
+
# header row processing is activated.
|
104
112
|
#
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
#
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
+
class Row
|
114
|
+
#
|
115
|
+
# Construct a new FasterCSV::Row from +headers+ and +fields+, which are
|
116
|
+
# expected to be Arrays. If one Array is shorter than the other, it will be
|
117
|
+
# padded with +nil+ objects.
|
118
|
+
#
|
119
|
+
# The optional +header_row+ parameter can be set to +true+ to indicate, via
|
120
|
+
# FasterCSV::Row.header_row?() and FasterCSV::Row.field_row?(), that this is
|
121
|
+
# a header row. Otherwise, the row is assumes to be a field row.
|
122
|
+
#
|
123
|
+
# A FasterCSV::Row object supports the following Array methods through
|
124
|
+
# delegation:
|
125
|
+
#
|
126
|
+
# * empty?()
|
127
|
+
# * length()
|
128
|
+
# * size()
|
129
|
+
#
|
130
|
+
def initialize(headers, fields, header_row = false)
|
131
|
+
@header_row = header_row
|
132
|
+
|
133
|
+
# handle extra headers or fields
|
134
|
+
@row = if headers.size > fields.size
|
135
|
+
headers.zip(fields)
|
136
|
+
else
|
137
|
+
fields.zip(headers).map { |pair| pair.reverse }
|
138
|
+
end
|
113
139
|
end
|
114
|
-
end
|
115
|
-
|
116
|
-
# Internal data format used to compare equality.
|
117
|
-
attr_reader :row
|
118
|
-
protected :row
|
119
140
|
|
120
|
-
|
141
|
+
# Internal data format used to compare equality.
|
142
|
+
attr_reader :row
|
143
|
+
protected :row
|
121
144
|
|
122
|
-
|
123
|
-
def_delegators :@row, :empty?, :length, :size
|
124
|
-
|
125
|
-
# Returns +true+ if this is a header row.
|
126
|
-
def header_row?
|
127
|
-
@header_row
|
128
|
-
end
|
129
|
-
|
130
|
-
# Returns +true+ if this is a field row.
|
131
|
-
def field_row?
|
132
|
-
not header_row?
|
133
|
-
end
|
134
|
-
|
135
|
-
# Returns the headers of this row.
|
136
|
-
def headers
|
137
|
-
@row.map { |pair| pair.first }
|
138
|
-
end
|
139
|
-
|
140
|
-
#
|
141
|
-
# :call-seq:
|
142
|
-
# field( header )
|
143
|
-
# field( header, offset )
|
144
|
-
# field( index )
|
145
|
-
#
|
146
|
-
# This method will fetch the field value by +header+ or +index+. If a field
|
147
|
-
# is not found, +nil+ is returned.
|
148
|
-
#
|
149
|
-
# When provided, +offset+ ensures that a header match occurrs on or later
|
150
|
-
# than the +offset+ index. You can use this to find duplicate headers,
|
151
|
-
# without resorting to hard-coding exact indices.
|
152
|
-
#
|
153
|
-
def field(header_or_index, minimum_index = 0)
|
154
|
-
# locate the pair
|
155
|
-
finder = header_or_index.is_a?(Integer) ? :[] : :assoc
|
156
|
-
pair = @row[minimum_index..-1].send(finder, header_or_index)
|
145
|
+
### Array Delegation ###
|
157
146
|
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
147
|
+
extend Forwardable
|
148
|
+
def_delegators :@row, :empty?, :length, :size
|
149
|
+
|
150
|
+
# Returns +true+ if this is a header row.
|
151
|
+
def header_row?
|
152
|
+
@header_row
|
153
|
+
end
|
154
|
+
|
155
|
+
# Returns +true+ if this is a field row.
|
156
|
+
def field_row?
|
157
|
+
not header_row?
|
158
|
+
end
|
159
|
+
|
160
|
+
# Returns the headers of this row.
|
161
|
+
def headers
|
162
|
+
@row.map { |pair| pair.first }
|
163
|
+
end
|
164
|
+
|
165
|
+
#
|
166
|
+
# :call-seq:
|
167
|
+
# field( header )
|
168
|
+
# field( header, offset )
|
169
|
+
# field( index )
|
170
|
+
#
|
171
|
+
# This method will fetch the field value by +header+ or +index+. If a field
|
172
|
+
# is not found, +nil+ is returned.
|
173
|
+
#
|
174
|
+
# When provided, +offset+ ensures that a header match occurrs on or later
|
175
|
+
# than the +offset+ index. You can use this to find duplicate headers,
|
176
|
+
# without resorting to hard-coding exact indices.
|
177
|
+
#
|
178
|
+
def field(header_or_index, minimum_index = 0)
|
179
|
+
# locate the pair
|
180
|
+
finder = header_or_index.is_a?(Integer) ? :[] : :assoc
|
181
|
+
pair = @row[minimum_index..-1].send(finder, header_or_index)
|
182
|
+
|
183
|
+
# return the field if we have a pair
|
184
|
+
pair.nil? ? nil : pair.last
|
185
|
+
end
|
186
|
+
alias_method :[], :field
|
187
|
+
|
188
|
+
#
|
189
|
+
# :call-seq:
|
190
|
+
# []=( header, value )
|
191
|
+
# []=( header, offset, value )
|
192
|
+
# []=( index, value )
|
193
|
+
#
|
194
|
+
# Looks up the field by the semantics described in FasterCSV::Row.field()
|
195
|
+
# and assigns the +value+.
|
196
|
+
#
|
197
|
+
# Assigning past the end of the row with an index will set all pairs between
|
198
|
+
# to <tt>[nil, nil]</tt>. Assigning to an unused header appends the new
|
199
|
+
# pair.
|
200
|
+
#
|
201
|
+
def []=(*args)
|
202
|
+
value = args.pop
|
203
|
+
|
204
|
+
if args.first.is_a? Integer
|
205
|
+
if @row[args.first].nil? # extending past the end with index
|
206
|
+
@row[args.first] = [nil, value]
|
207
|
+
@row.map! { |pair| pair.nil? ? [nil, nil] : pair }
|
208
|
+
else # normal index assignment
|
209
|
+
@row[args.first][1] = value
|
210
|
+
end
|
211
|
+
else
|
212
|
+
index = index(*args)
|
213
|
+
if index.nil? # appending a field
|
214
|
+
self << [args.first, value]
|
215
|
+
else # normal header assignment
|
216
|
+
@row[index][1] = value
|
217
|
+
end
|
185
218
|
end
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
219
|
+
end
|
220
|
+
|
221
|
+
#
|
222
|
+
# :call-seq:
|
223
|
+
# <<( field )
|
224
|
+
# <<( header_and_field_array )
|
225
|
+
# <<( header_and_field_hash )
|
226
|
+
#
|
227
|
+
# If a two-element Array is provided, it is assumed to be a header and field
|
228
|
+
# and the pair is appended. A Hash works the same way with the key being
|
229
|
+
# the header and the value being the field. Anything else is assumed to be
|
230
|
+
# a lone field which is appended with a +nil+ header.
|
231
|
+
#
|
232
|
+
# This method returns the row for chaining.
|
233
|
+
#
|
234
|
+
def <<(arg)
|
235
|
+
if arg.is_a?(Array) and arg.size == 2 # appending a header and name
|
236
|
+
@row << arg
|
237
|
+
elsif arg.is_a?(Hash) # append header and name pairs
|
238
|
+
arg.each { |pair| @row << pair }
|
239
|
+
else # append field value
|
240
|
+
@row << [nil, arg]
|
192
241
|
end
|
242
|
+
|
243
|
+
self # for chaining
|
193
244
|
end
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
245
|
+
|
246
|
+
#
|
247
|
+
# A shortcut for appending multiple fields. Equivalent to:
|
248
|
+
#
|
249
|
+
# args.each { |arg| faster_csv_row << arg }
|
250
|
+
#
|
251
|
+
# This method returns the row for chaining.
|
252
|
+
#
|
253
|
+
def push(*args)
|
254
|
+
args.each { |arg| self << arg }
|
255
|
+
|
256
|
+
self # for chaining
|
257
|
+
end
|
258
|
+
|
259
|
+
#
|
260
|
+
# :call-seq:
|
261
|
+
# delete( header )
|
262
|
+
# delete( header, offset )
|
263
|
+
# delete( index )
|
264
|
+
#
|
265
|
+
# Used to remove a pair from the row by +header+ or +index+. The pair is
|
266
|
+
# located as described in FasterCSV::Row.field(). The deleted pair is
|
267
|
+
# returned, or +nil+ if a pair could not be found.
|
268
|
+
#
|
269
|
+
def delete(header_or_index, minimum_index = 0)
|
270
|
+
if header_or_index.is_a? Integer # by index
|
271
|
+
@row.delete_at(header_or_index)
|
272
|
+
elsif i = index(header_or_index, minimum_index) # by header
|
273
|
+
@row.delete_at(i)
|
274
|
+
else
|
275
|
+
[ ]
|
276
|
+
end
|
277
|
+
end
|
278
|
+
|
279
|
+
#
|
280
|
+
# The provided +block+ is passed a header and field for each pair in the row
|
281
|
+
# and expected to return +true+ or +false+, depending on whether the pair
|
282
|
+
# should be deleted.
|
283
|
+
#
|
284
|
+
# This method returns the row for chaining.
|
285
|
+
#
|
286
|
+
def delete_if(&block)
|
287
|
+
@row.delete_if(&block)
|
288
|
+
|
289
|
+
self # for chaining
|
290
|
+
end
|
291
|
+
|
292
|
+
#
|
293
|
+
# This method accepts any number of arguments which can be headers, indices,
|
294
|
+
# Ranges of either, or two-element Arrays containing a header and offset.
|
295
|
+
# Each argument will be replaced with a field lookup as described in
|
296
|
+
# FasterCSV::Row.field().
|
297
|
+
#
|
298
|
+
# If called with no arguments, all fields are returned.
|
299
|
+
#
|
300
|
+
def fields(*headers_and_or_indices)
|
301
|
+
if headers_and_or_indices.empty? # return all fields--no arguments
|
302
|
+
@row.map { |pair| pair.last }
|
303
|
+
else # or work like values_at()
|
304
|
+
headers_and_or_indices.inject(Array.new) do |all, h_or_i|
|
305
|
+
all + if h_or_i.is_a? Range
|
306
|
+
index_begin = h_or_i.begin.is_a?(Integer) ? h_or_i.begin :
|
307
|
+
index(h_or_i.begin)
|
308
|
+
index_end = h_or_i.end.is_a?(Integer) ? h_or_i.end :
|
309
|
+
index(h_or_i.end)
|
310
|
+
new_range = h_or_i.exclude_end? ? (index_begin...index_end) :
|
311
|
+
(index_begin..index_end)
|
312
|
+
fields.values_at(new_range)
|
313
|
+
else
|
314
|
+
[field(*Array(h_or_i))]
|
315
|
+
end
|
316
|
+
end
|
317
|
+
end
|
318
|
+
end
|
319
|
+
alias_method :values_at, :fields
|
320
|
+
|
321
|
+
#
|
322
|
+
# :call-seq:
|
323
|
+
# index( header )
|
324
|
+
# index( header, offset )
|
325
|
+
#
|
326
|
+
# This method will return the index of a field with the provided +header+.
|
327
|
+
# The +offset+ can be used to locate duplicate header names, as described in
|
328
|
+
# FasterCSV::Row.field().
|
329
|
+
#
|
330
|
+
def index(header, minimum_index = 0)
|
331
|
+
# find the pair
|
332
|
+
index = headers[minimum_index..-1].index(header)
|
333
|
+
# return the index at the right offset, if we found one
|
334
|
+
index.nil? ? nil : index + minimum_index
|
335
|
+
end
|
336
|
+
|
337
|
+
# Returns +true+ if +name+ is a header for this row, and +false+ otherwise.
|
338
|
+
def header?(name)
|
339
|
+
headers.include? name
|
340
|
+
end
|
341
|
+
alias_method :include?, :header?
|
342
|
+
|
343
|
+
#
|
344
|
+
# Returns +true+ if +data+ matches a field in this row, and +false+
|
345
|
+
# otherwise.
|
346
|
+
#
|
347
|
+
def field?(data)
|
348
|
+
fields.include? data
|
349
|
+
end
|
350
|
+
|
351
|
+
include Enumerable
|
352
|
+
|
353
|
+
#
|
354
|
+
# Yields each pair of the row as header and field tuples (much like
|
355
|
+
# iterating over a Hash).
|
356
|
+
#
|
357
|
+
# Support for Enumerable.
|
358
|
+
#
|
359
|
+
# This method returns the row for chaining.
|
360
|
+
#
|
361
|
+
def each(&block)
|
362
|
+
@row.each(&block)
|
363
|
+
|
364
|
+
self # for chaining
|
365
|
+
end
|
366
|
+
|
367
|
+
#
|
368
|
+
# Returns +true+ if this row contains the same headers and fields in the
|
369
|
+
# same order as +other+.
|
370
|
+
#
|
371
|
+
def ==(other)
|
372
|
+
@row == other.row
|
373
|
+
end
|
374
|
+
|
375
|
+
#
|
376
|
+
# Collapses the row into a simple Hash. Be warning that this discards field
|
377
|
+
# order and clobbers duplicate fields.
|
378
|
+
#
|
379
|
+
def to_hash
|
380
|
+
# flatten just one level of the internal Array
|
381
|
+
Hash[*@row.inject(Array.new) { |ary, pair| ary.push(*pair) }]
|
382
|
+
end
|
383
|
+
|
384
|
+
#
|
385
|
+
# Returns the row as a CSV String. Headers are not used. Equivalent to:
|
386
|
+
#
|
387
|
+
# faster_csv_row.fields.to_csv( options )
|
388
|
+
#
|
389
|
+
def to_csv(options = Hash.new)
|
390
|
+
fields.to_csv(options)
|
391
|
+
end
|
392
|
+
alias_method :to_s, :to_csv
|
393
|
+
|
394
|
+
# A summary of fields, by header.
|
395
|
+
def inspect
|
396
|
+
str = "#<#{self.class}"
|
397
|
+
each do |header, field|
|
398
|
+
str << " #{header.is_a?(Symbol) ? header.to_s : header.inspect}:" <<
|
399
|
+
field.inspect
|
400
|
+
end
|
401
|
+
str << ">"
|
249
402
|
end
|
250
403
|
end
|
251
|
-
|
404
|
+
|
252
405
|
#
|
253
|
-
#
|
254
|
-
#
|
255
|
-
#
|
406
|
+
# A FasterCSV::Table is a two-dimensional data structure for representing CSV
|
407
|
+
# documents. Tables allow you to work with the data by row or column,
|
408
|
+
# manipulate the data, and even convert the results back to CSV, if needed.
|
256
409
|
#
|
257
|
-
#
|
410
|
+
# All tables returned by FasterCSV will be constructed from this class, if
|
411
|
+
# header row processing is activated.
|
258
412
|
#
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
413
|
+
class Table
|
414
|
+
#
|
415
|
+
# Construct a new FasterCSV::Table from +array_of_rows+, which are expected
|
416
|
+
# to be FasterCSV::Row objects. All rows are assumed to have the same
|
417
|
+
# headers.
|
418
|
+
#
|
419
|
+
# A FasterCSV::Table object supports the following Array methods through
|
420
|
+
# delegation:
|
421
|
+
#
|
422
|
+
# * empty?()
|
423
|
+
# * length()
|
424
|
+
# * size()
|
425
|
+
#
|
426
|
+
def initialize(array_of_rows)
|
427
|
+
@table = array_of_rows
|
428
|
+
@mode = :col_or_row
|
429
|
+
end
|
430
|
+
|
431
|
+
# The current access mode for indexing and iteration.
|
432
|
+
attr_reader :mode
|
433
|
+
|
434
|
+
# Internal data format used to compare equality.
|
435
|
+
attr_reader :table
|
436
|
+
protected :table
|
437
|
+
|
438
|
+
### Array Delegation ###
|
439
|
+
|
440
|
+
extend Forwardable
|
441
|
+
def_delegators :@table, :empty?, :length, :size
|
442
|
+
|
443
|
+
#
|
444
|
+
# Returns a duplicate table object, in column mode. This is handy for
|
445
|
+
# chaining in a single call without changing the table mode, but be aware
|
446
|
+
# that this method can consume a fair amount of memory for bigger data sets.
|
447
|
+
#
|
448
|
+
# This method returns the duplicate table for chaining. Don't chain
|
449
|
+
# destructive methods (like []=()) this way though, since you are working
|
450
|
+
# with a duplicate.
|
451
|
+
#
|
452
|
+
def by_col
|
453
|
+
self.class.new(@table.dup).by_col!
|
454
|
+
end
|
455
|
+
|
456
|
+
#
|
457
|
+
# Switches the mode of this table to column mode. All calls to indexing and
|
458
|
+
# iteration methods will work with columns until the mode is changed again.
|
459
|
+
#
|
460
|
+
# This method returns the table and is safe to chain.
|
461
|
+
#
|
462
|
+
def by_col!
|
463
|
+
@mode = :col
|
464
|
+
|
465
|
+
self
|
466
|
+
end
|
467
|
+
|
468
|
+
#
|
469
|
+
# Returns a duplicate table object, in mixed mode. This is handy for
|
470
|
+
# chaining in a single call without changing the table mode, but be aware
|
471
|
+
# that this method can consume a fair amount of memory for bigger data sets.
|
472
|
+
#
|
473
|
+
# This method returns the duplicate table for chaining. Don't chain
|
474
|
+
# destructive methods (like []=()) this way though, since you are working
|
475
|
+
# with a duplicate.
|
476
|
+
#
|
477
|
+
def by_col_or_row
|
478
|
+
self.class.new(@table.dup).by_col_or_row!
|
479
|
+
end
|
480
|
+
|
481
|
+
#
|
482
|
+
# Switches the mode of this table to mixed mode. All calls to indexing and
|
483
|
+
# iteration methods will use the default intelligent indexing system until
|
484
|
+
# the mode is changed again. In mixed mode an index is assumed to be a row
|
485
|
+
# reference while anything else is assumed to be column access by headers.
|
486
|
+
#
|
487
|
+
# This method returns the table and is safe to chain.
|
488
|
+
#
|
489
|
+
def by_col_or_row!
|
490
|
+
@mode = :col_or_row
|
491
|
+
|
492
|
+
self
|
493
|
+
end
|
494
|
+
|
495
|
+
#
|
496
|
+
# Returns a duplicate table object, in row mode. This is handy for chaining
|
497
|
+
# in a single call without changing the table mode, but be aware that this
|
498
|
+
# method can consume a fair amount of memory for bigger data sets.
|
499
|
+
#
|
500
|
+
# This method returns the duplicate table for chaining. Don't chain
|
501
|
+
# destructive methods (like []=()) this way though, since you are working
|
502
|
+
# with a duplicate.
|
503
|
+
#
|
504
|
+
def by_row
|
505
|
+
self.class.new(@table.dup).by_row!
|
506
|
+
end
|
507
|
+
|
508
|
+
#
|
509
|
+
# Switches the mode of this table to row mode. All calls to indexing and
|
510
|
+
# iteration methods will work with rows until the mode is changed again.
|
511
|
+
#
|
512
|
+
# This method returns the table and is safe to chain.
|
513
|
+
#
|
514
|
+
def by_row!
|
515
|
+
@mode = :row
|
516
|
+
|
517
|
+
self
|
518
|
+
end
|
519
|
+
|
520
|
+
#
|
521
|
+
# Returns the headers for the first row of this table (assumed to match all
|
522
|
+
# other rows). An empty Array is returned for empty tables.
|
523
|
+
#
|
524
|
+
def headers
|
525
|
+
if @table.empty?
|
526
|
+
Array.new
|
527
|
+
else
|
528
|
+
@table.first.headers
|
529
|
+
end
|
530
|
+
end
|
531
|
+
|
532
|
+
#
|
533
|
+
# In the default mixed mode, this method returns rows for index access and
|
534
|
+
# columns for header access. You can force the index association by first
|
535
|
+
# calling by_col!() or by_row!().
|
536
|
+
#
|
537
|
+
# Columns are returned as an Array of values. Altering that Array has no
|
538
|
+
# effect on the table.
|
539
|
+
#
|
540
|
+
def [](index_or_header)
|
541
|
+
if @mode == :row or # by index
|
542
|
+
(@mode == :col_or_row and index_or_header.is_a? Integer)
|
543
|
+
@table[index_or_header]
|
544
|
+
else # by header
|
545
|
+
@table.map { |row| row[index_or_header] }
|
546
|
+
end
|
547
|
+
end
|
548
|
+
|
549
|
+
#
|
550
|
+
# In the default mixed mode, this method assigns rows for index access and
|
551
|
+
# columns for header access. You can force the index association by first
|
552
|
+
# calling by_col!() or by_row!().
|
553
|
+
#
|
554
|
+
# Rows may be set to an Array of values (which will inherit the table's
|
555
|
+
# headers()) or a FasterCSV::Row.
|
556
|
+
#
|
557
|
+
# Columns may be set to a single value, which is copied to each row of the
|
558
|
+
# column, or an Array of values. Arrays of values are assigned to rows top
|
559
|
+
# to bottom in row major order. Excess values are ignored and if the Array
|
560
|
+
# does not have a value for each row the extra rows will receive a +nil+.
|
561
|
+
#
|
562
|
+
# Assigning to an existing column or row clobbers the data. Assigning to
|
563
|
+
# new columns creates them at the right end of the table.
|
564
|
+
#
|
565
|
+
def []=(index_or_header, value)
|
566
|
+
if @mode == :row or # by index
|
567
|
+
(@mode == :col_or_row and index_or_header.is_a? Integer)
|
568
|
+
if value.is_a? Array
|
569
|
+
@table[index_or_header] = Row.new(headers, value)
|
286
570
|
else
|
287
|
-
[
|
571
|
+
@table[index_or_header] = value
|
572
|
+
end
|
573
|
+
else # set column
|
574
|
+
if value.is_a? Array # multiple values
|
575
|
+
@table.each_with_index do |row, i|
|
576
|
+
if row.header_row?
|
577
|
+
row[index_or_header] = index_or_header
|
578
|
+
else
|
579
|
+
row[index_or_header] = value[i]
|
580
|
+
end
|
581
|
+
end
|
582
|
+
else # repeated value
|
583
|
+
@table.each do |row|
|
584
|
+
if row.header_row?
|
585
|
+
row[index_or_header] = index_or_header
|
586
|
+
else
|
587
|
+
row[index_or_header] = value
|
588
|
+
end
|
589
|
+
end
|
288
590
|
end
|
289
591
|
end
|
290
592
|
end
|
291
|
-
end
|
292
|
-
alias_method :values_at, :fields
|
293
|
-
|
294
|
-
#
|
295
|
-
# :call-seq:
|
296
|
-
# index( header )
|
297
|
-
# index( header, offset )
|
298
|
-
#
|
299
|
-
# This method will return the index of a field with the provided +header+.
|
300
|
-
# The +offset+ can be used to locate duplicate header names, as described in
|
301
|
-
# FasterCSV::Row.field().
|
302
|
-
#
|
303
|
-
def index(header, minimum_index = 0)
|
304
|
-
# find the pair
|
305
|
-
index = headers[minimum_index..-1].index(header)
|
306
|
-
# return the index at the right offset, if we found one
|
307
|
-
index.nil? ? nil : index + minimum_index
|
308
|
-
end
|
309
|
-
|
310
|
-
# Returns +true+ if +name+ is a header for this row, and +false+ otherwise.
|
311
|
-
def header?(name)
|
312
|
-
headers.include? name
|
313
|
-
end
|
314
|
-
alias_method :include?, :header?
|
315
|
-
|
316
|
-
#
|
317
|
-
# Returns +true+ if +data+ matches a field in this row, and +false+
|
318
|
-
# otherwise.
|
319
|
-
#
|
320
|
-
def field?(data)
|
321
|
-
fields.include? data
|
322
|
-
end
|
323
593
|
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
#
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
594
|
+
#
|
595
|
+
# The mixed mode default is to treat a list of indices as row access,
|
596
|
+
# returning the rows indicated. Anything else is considered columnar
|
597
|
+
# access. For columnar access, the return set has an Array for each row
|
598
|
+
# with the values indicated by the headers in each Array. You can force
|
599
|
+
# column or row mode using by_col!() or by_row!().
|
600
|
+
#
|
601
|
+
# You cannot mix column and row access.
|
602
|
+
#
|
603
|
+
def values_at(*indices_or_headers)
|
604
|
+
if @mode == :row or # by indices
|
605
|
+
( @mode == :col_or_row and indices_or_headers.all? do |index|
|
606
|
+
index.is_a?(Integer) or
|
607
|
+
( index.is_a?(Range) and
|
608
|
+
index.first.is_a?(Integer) and
|
609
|
+
index.last.is_a?(Integer) )
|
610
|
+
end )
|
611
|
+
@table.values_at(*indices_or_headers)
|
612
|
+
else # by headers
|
613
|
+
@table.map { |row| row.values_at(*indices_or_headers) }
|
614
|
+
end
|
615
|
+
end
|
616
|
+
|
617
|
+
#
|
618
|
+
# Adds a new row to the bottom end of this table. You can provide an Array,
|
619
|
+
# which will be converted to a FasterCSV::Row (inheriting the table's
|
620
|
+
# headers()), or a FasterCSV::Row.
|
621
|
+
#
|
622
|
+
# This method returns the table for chaining.
|
623
|
+
#
|
624
|
+
def <<(row_or_array)
|
625
|
+
if row_or_array.is_a? Array # append Array
|
626
|
+
@table << Row.new(headers, row_or_array)
|
627
|
+
else # append Row
|
628
|
+
@table << row_or_array
|
629
|
+
end
|
630
|
+
|
631
|
+
self # for chaining
|
632
|
+
end
|
633
|
+
|
634
|
+
#
|
635
|
+
# A shortcut for appending multiple rows. Equivalent to:
|
636
|
+
#
|
637
|
+
# rows.each { |row| self << row }
|
638
|
+
#
|
639
|
+
# This method returns the table for chaining.
|
640
|
+
#
|
641
|
+
def push(*rows)
|
642
|
+
rows.each { |row| self << row }
|
643
|
+
|
644
|
+
self # for chaining
|
645
|
+
end
|
646
|
+
|
647
|
+
#
|
648
|
+
# Removes and returns the indicated column or row. In the default mixed
|
649
|
+
# mode indices refer to rows and everything else is assumed to be a column
|
650
|
+
# header. Use by_col!() or by_row!() to force the lookup.
|
651
|
+
#
|
652
|
+
def delete(index_or_header)
|
653
|
+
if @mode == :row or # by index
|
654
|
+
(@mode == :col_or_row and index_or_header.is_a? Integer)
|
655
|
+
@table.delete_at(index_or_header)
|
656
|
+
else # by header
|
657
|
+
@table.map { |row| row.delete(index_or_header).last }
|
658
|
+
end
|
659
|
+
end
|
660
|
+
|
661
|
+
#
|
662
|
+
# Removes any column or row for which the block returns +true+. In the
|
663
|
+
# default mixed mode or row mode, iteration is the standard row major
|
664
|
+
# walking of rows. In column mode, interation will +yield+ two element
|
665
|
+
# tuples containing the column name and an Array of values for that column.
|
666
|
+
#
|
667
|
+
# This method returns the table for chaining.
|
668
|
+
#
|
669
|
+
def delete_if(&block)
|
670
|
+
if @mode == :row or @mode == :col_or_row # by index
|
671
|
+
@table.delete_if(&block)
|
672
|
+
else # by header
|
673
|
+
to_delete = Array.new
|
674
|
+
headers.each_with_index do |header, i|
|
675
|
+
to_delete << header if block[[header, self[header]]]
|
676
|
+
end
|
677
|
+
to_delete.map { |header| delete(header) }
|
678
|
+
end
|
679
|
+
|
680
|
+
self # for chaining
|
681
|
+
end
|
682
|
+
|
683
|
+
include Enumerable
|
684
|
+
|
685
|
+
#
|
686
|
+
# In the default mixed mode or row mode, iteration is the standard row major
|
687
|
+
# walking of rows. In column mode, interation will +yield+ two element
|
688
|
+
# tuples containing the column name and an Array of values for that column.
|
689
|
+
#
|
690
|
+
# This method returns the table for chaining.
|
691
|
+
#
|
692
|
+
def each(&block)
|
693
|
+
if @mode == :col
|
694
|
+
headers.each { |header| block[[header, self[header]]] }
|
695
|
+
else
|
696
|
+
@table.each(&block)
|
697
|
+
end
|
698
|
+
|
699
|
+
self # for chaining
|
700
|
+
end
|
701
|
+
|
702
|
+
# Returns +true+ if all rows of this table ==() +other+'s rows.
|
703
|
+
def ==(other)
|
704
|
+
@table == other.table
|
705
|
+
end
|
706
|
+
|
707
|
+
#
|
708
|
+
# Returns the table as an Array of Arrays. Headers will be the first row,
|
709
|
+
# then all of the field rows will follow.
|
710
|
+
#
|
711
|
+
def to_a
|
712
|
+
@table.inject([headers]) do |array, row|
|
713
|
+
if row.header_row?
|
714
|
+
array
|
715
|
+
else
|
716
|
+
array + [row.fields]
|
717
|
+
end
|
718
|
+
end
|
719
|
+
end
|
720
|
+
|
721
|
+
#
|
722
|
+
# Returns the table as a complete CSV String. Headers will be listed first,
|
723
|
+
# then all of the field rows.
|
724
|
+
#
|
725
|
+
# This method assumes you want the Table.headers(), unless you explicitly
|
726
|
+
# pass <tt>:write_headers => false</tt>.
|
727
|
+
#
|
728
|
+
def to_csv(options = Hash.new)
|
729
|
+
wh = options.fetch(:write_headers, true)
|
730
|
+
@table.inject(wh ? [headers.to_csv(options)] : [ ]) do |rows, row|
|
731
|
+
if row.header_row?
|
732
|
+
rows
|
733
|
+
else
|
734
|
+
rows + [row.fields.to_csv(options)]
|
735
|
+
end
|
736
|
+
end.join
|
737
|
+
end
|
738
|
+
alias_method :to_s, :to_csv
|
739
|
+
|
740
|
+
def inspect
|
741
|
+
"#<#{self.class} mode:#{@mode} row_count:#{to_a.size}>"
|
373
742
|
end
|
374
|
-
str << ">"
|
375
|
-
end
|
376
|
-
end
|
377
|
-
|
378
|
-
#
|
379
|
-
# A FasterCSV::Table is a two-dimensional data structure for representing CSV
|
380
|
-
# documents. Tables allow you to work with the data by row or column,
|
381
|
-
# manipulate the data, and even convert the results back to CSV, if needed.
|
382
|
-
#
|
383
|
-
# All tables returned by FasterCSV will be constructed from this class, if
|
384
|
-
# header row processing is activated.
|
385
|
-
#
|
386
|
-
class Table
|
387
|
-
#
|
388
|
-
# Construct a new FasterCSV::Table from +array_of_rows+, which are expected
|
389
|
-
# to be FasterCSV::Row objects. All rows are assumed to have the same
|
390
|
-
# headers.
|
391
|
-
#
|
392
|
-
# A FasterCSV::Table object supports the following Array methods through
|
393
|
-
# delegation:
|
394
|
-
#
|
395
|
-
# * empty?()
|
396
|
-
# * length()
|
397
|
-
# * size()
|
398
|
-
#
|
399
|
-
def initialize(array_of_rows)
|
400
|
-
@table = array_of_rows
|
401
|
-
@mode = :col_or_row
|
402
743
|
end
|
403
|
-
|
404
|
-
# The current access mode for indexing and iteration.
|
405
|
-
attr_reader :mode
|
406
|
-
|
407
|
-
# Internal data format used to compare equality.
|
408
|
-
attr_reader :table
|
409
|
-
protected :table
|
410
744
|
|
411
|
-
|
745
|
+
# The error thrown when the parser encounters illegal CSV formatting.
|
746
|
+
class MalformedCSVError < RuntimeError; end
|
412
747
|
|
413
|
-
extend Forwardable
|
414
|
-
def_delegators :@table, :empty?, :length, :size
|
415
|
-
|
416
|
-
#
|
417
|
-
# Returns a duplicate table object, in column mode. This is handy for
|
418
|
-
# chaining in a single call without changing the table mode, but be aware
|
419
|
-
# that this method can consume a fair amount of memory for bigger data sets.
|
420
748
|
#
|
421
|
-
#
|
422
|
-
#
|
423
|
-
#
|
749
|
+
# A FieldInfo Struct contains details about a field's position in the data
|
750
|
+
# source it was read from. FasterCSV will pass this Struct to some blocks
|
751
|
+
# that make decisions based on field structure. See
|
752
|
+
# FasterCSV.convert_fields() for an example.
|
424
753
|
#
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
#
|
430
|
-
# Switches the mode of this table to column mode. All calls to indexing and
|
431
|
-
# iteration methods will work with columns until the mode is changed again.
|
432
|
-
#
|
433
|
-
# This method returns the table and is safe to chain.
|
434
|
-
#
|
435
|
-
def by_col!
|
436
|
-
@mode = :col
|
437
|
-
|
438
|
-
self
|
439
|
-
end
|
440
|
-
|
441
|
-
#
|
442
|
-
# Returns a duplicate table object, in mixed mode. This is handy for
|
443
|
-
# chaining in a single call without changing the table mode, but be aware
|
444
|
-
# that this method can consume a fair amount of memory for bigger data sets.
|
445
|
-
#
|
446
|
-
# This method returns the duplicate table for chaining. Don't chain
|
447
|
-
# destructive methods (like []=()) this way though, since you are working
|
448
|
-
# with a duplicate.
|
449
|
-
#
|
450
|
-
def by_col_or_row
|
451
|
-
self.class.new(@table.dup).by_col_or_row!
|
452
|
-
end
|
453
|
-
|
454
|
-
#
|
455
|
-
# Switches the mode of this table to mixed mode. All calls to indexing and
|
456
|
-
# iteration methods will use the default intelligent indexing system until
|
457
|
-
# the mode is changed again. In mixed mode an index is assumed to be a row
|
458
|
-
# reference while anything else is assumed to be column access by headers.
|
459
|
-
#
|
460
|
-
# This method returns the table and is safe to chain.
|
461
|
-
#
|
462
|
-
def by_col_or_row!
|
463
|
-
@mode = :col_or_row
|
464
|
-
|
465
|
-
self
|
466
|
-
end
|
467
|
-
|
754
|
+
# <b><tt>index</tt></b>:: The zero-based index of the field in its row.
|
755
|
+
# <b><tt>line</tt></b>:: The line of the data source this row is from.
|
756
|
+
# <b><tt>header</tt></b>:: The header for the column, when available.
|
468
757
|
#
|
469
|
-
|
470
|
-
|
471
|
-
#
|
758
|
+
FieldInfo = Struct.new(:index, :line, :header)
|
759
|
+
|
760
|
+
# A Regexp used to find and convert some common Date formats.
|
761
|
+
DateMatcher = / \A(?: (\w+,?\s+)?\w+\s+\d{1,2},?\s+\d{2,4} |
|
762
|
+
\d{4}-\d{2}-\d{2} )\z /x
|
763
|
+
# A Regexp used to find and convert some common DateTime formats.
|
764
|
+
DateTimeMatcher =
|
765
|
+
/ \A(?: (\w+,?\s+)?\w+\s+\d{1,2}\s+\d{1,2}:\d{1,2}:\d{1,2},?\s+\d{2,4} |
|
766
|
+
\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2} )\z /x
|
767
|
+
#
|
768
|
+
# This Hash holds the built-in converters of FasterCSV that can be accessed by
|
769
|
+
# name. You can select Converters with FasterCSV.convert() or through the
|
770
|
+
# +options+ Hash passed to FasterCSV::new().
|
771
|
+
#
|
772
|
+
# <b><tt>:integer</tt></b>:: Converts any field Integer() accepts.
|
773
|
+
# <b><tt>:float</tt></b>:: Converts any field Float() accepts.
|
774
|
+
# <b><tt>:numeric</tt></b>:: A combination of <tt>:integer</tt>
|
775
|
+
# and <tt>:float</tt>.
|
776
|
+
# <b><tt>:date</tt></b>:: Converts any field Date::parse() accepts.
|
777
|
+
# <b><tt>:date_time</tt></b>:: Converts any field DateTime::parse() accepts.
|
778
|
+
# <b><tt>:all</tt></b>:: All built-in converters. A combination of
|
779
|
+
# <tt>:date_time</tt> and <tt>:numeric</tt>.
|
780
|
+
#
|
781
|
+
# This Hash is intetionally left unfrozen and users should feel free to add
|
782
|
+
# values to it that can be accessed by all FasterCSV objects.
|
783
|
+
#
|
784
|
+
# To add a combo field, the value should be an Array of names. Combo fields
|
785
|
+
# can be nested with other combo fields.
|
786
|
+
#
|
787
|
+
Converters = { :integer => lambda { |f| Integer(f) rescue f },
|
788
|
+
:float => lambda { |f| Float(f) rescue f },
|
789
|
+
:numeric => [:integer, :float],
|
790
|
+
:date => lambda { |f|
|
791
|
+
f =~ DateMatcher ? (Date.parse(f) rescue f) : f
|
792
|
+
},
|
793
|
+
:date_time => lambda { |f|
|
794
|
+
f =~ DateTimeMatcher ? (DateTime.parse(f) rescue f) : f
|
795
|
+
},
|
796
|
+
:all => [:date_time, :numeric] }
|
797
|
+
|
472
798
|
#
|
473
|
-
# This
|
474
|
-
#
|
475
|
-
#
|
799
|
+
# This Hash holds the built-in header converters of FasterCSV that can be
|
800
|
+
# accessed by name. You can select HeaderConverters with
|
801
|
+
# FasterCSV.header_convert() or through the +options+ Hash passed to
|
802
|
+
# FasterCSV::new().
|
476
803
|
#
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
|
804
|
+
# <b><tt>:downcase</tt></b>:: Calls downcase() on the header String.
|
805
|
+
# <b><tt>:symbol</tt></b>:: The header String is downcased, spaces are
|
806
|
+
# replaced with underscores, non-word characters
|
807
|
+
# are dropped, and finally to_sym() is called.
|
481
808
|
#
|
482
|
-
#
|
483
|
-
#
|
809
|
+
# This Hash is intetionally left unfrozen and users should feel free to add
|
810
|
+
# values to it that can be accessed by all FasterCSV objects.
|
484
811
|
#
|
485
|
-
#
|
812
|
+
# To add a combo field, the value should be an Array of names. Combo fields
|
813
|
+
# can be nested with other combo fields.
|
486
814
|
#
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
|
492
|
-
|
815
|
+
HeaderConverters = {
|
816
|
+
:downcase => lambda { |h| h.downcase },
|
817
|
+
:symbol => lambda { |h|
|
818
|
+
h.downcase.tr(" ", "_").delete("^a-z0-9_").to_sym
|
819
|
+
}
|
820
|
+
}
|
821
|
+
|
493
822
|
#
|
494
|
-
#
|
495
|
-
#
|
823
|
+
# The options used when no overrides are given by calling code. They are:
|
824
|
+
#
|
825
|
+
# <b><tt>:col_sep</tt></b>:: <tt>","</tt>
|
826
|
+
# <b><tt>:row_sep</tt></b>:: <tt>:auto</tt>
|
827
|
+
# <b><tt>:quote_char</tt></b>:: <tt>'"'</tt>
|
828
|
+
# <b><tt>:converters</tt></b>:: +nil+
|
829
|
+
# <b><tt>:unconverted_fields</tt></b>:: +nil+
|
830
|
+
# <b><tt>:headers</tt></b>:: +false+
|
831
|
+
# <b><tt>:return_headers</tt></b>:: +false+
|
832
|
+
# <b><tt>:header_converters</tt></b>:: +nil+
|
833
|
+
# <b><tt>:skip_blanks</tt></b>:: +false+
|
834
|
+
# <b><tt>:force_quotes</tt></b>:: +false+
|
835
|
+
#
|
836
|
+
DEFAULT_OPTIONS = { :col_sep => ",",
|
837
|
+
:row_sep => :auto,
|
838
|
+
:quote_char => '"',
|
839
|
+
:converters => nil,
|
840
|
+
:unconverted_fields => nil,
|
841
|
+
:headers => false,
|
842
|
+
:return_headers => false,
|
843
|
+
:header_converters => nil,
|
844
|
+
:skip_blanks => false,
|
845
|
+
:force_quotes => false }.freeze
|
846
|
+
|
496
847
|
#
|
497
|
-
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
848
|
+
# This method will build a drop-in replacement for many of the standard CSV
|
849
|
+
# methods. It allows you to write code like:
|
850
|
+
#
|
851
|
+
# begin
|
852
|
+
# require "faster_csv"
|
853
|
+
# FasterCSV.build_csv_interface
|
854
|
+
# rescue LoadError
|
855
|
+
# require "csv"
|
856
|
+
# end
|
857
|
+
# # ... use CSV here ...
|
858
|
+
#
|
859
|
+
# This is not a complete interface with completely identical behavior.
|
860
|
+
# However, it is intended to be close enough that you won't notice the
|
861
|
+
# difference in most cases. CSV methods supported are:
|
862
|
+
#
|
863
|
+
# * foreach()
|
864
|
+
# * generate_line()
|
865
|
+
# * open()
|
866
|
+
# * parse()
|
867
|
+
# * parse_line()
|
868
|
+
# * readlines()
|
869
|
+
#
|
870
|
+
# Be warned that this interface is slower than vanilla FasterCSV due to the
|
871
|
+
# extra layer of method calls. Depending on usage, this can slow it down to
|
872
|
+
# near CSV speeds.
|
873
|
+
#
|
874
|
+
def self.build_csv_interface
|
875
|
+
Object.const_set(:CSV, Class.new).class_eval do
|
876
|
+
def self.foreach(path, rs = :auto, &block) # :nodoc:
|
877
|
+
FasterCSV.foreach(path, :row_sep => rs, &block)
|
878
|
+
end
|
879
|
+
|
880
|
+
def self.generate_line(row, fs = ",", rs = "") # :nodoc:
|
881
|
+
FasterCSV.generate_line(row, :col_sep => fs, :row_sep => rs)
|
882
|
+
end
|
883
|
+
|
884
|
+
def self.open(path, mode, fs = ",", rs = :auto, &block) # :nodoc:
|
885
|
+
if block and mode.include? "r"
|
886
|
+
FasterCSV.open(path, mode, :col_sep => fs, :row_sep => rs) do |csv|
|
887
|
+
csv.each(&block)
|
888
|
+
end
|
889
|
+
else
|
890
|
+
FasterCSV.open(path, mode, :col_sep => fs, :row_sep => rs, &block)
|
891
|
+
end
|
892
|
+
end
|
893
|
+
|
894
|
+
def self.parse(str_or_readable, fs = ",", rs = :auto, &block) # :nodoc:
|
895
|
+
FasterCSV.parse(str_or_readable, :col_sep => fs, :row_sep => rs, &block)
|
896
|
+
end
|
897
|
+
|
898
|
+
def self.parse_line(src, fs = ",", rs = :auto) # :nodoc:
|
899
|
+
FasterCSV.parse_line(src, :col_sep => fs, :row_sep => rs)
|
900
|
+
end
|
901
|
+
|
902
|
+
def self.readlines(path, rs = :auto) # :nodoc:
|
903
|
+
FasterCSV.readlines(path, :row_sep => rs)
|
904
|
+
end
|
502
905
|
end
|
503
906
|
end
|
504
|
-
|
505
|
-
#
|
506
|
-
# In the default mixed mode, this method returns rows for index access and
|
507
|
-
# columns for header access. You can force the index association by first
|
508
|
-
# calling by_col!() or by_row!().
|
509
|
-
#
|
510
|
-
# Columns are returned as an Array of values. Altering that Array has no
|
511
|
-
# effect on the table.
|
907
|
+
|
512
908
|
#
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
909
|
+
# This method allows you to serialize an Array of Ruby objects to a String or
|
910
|
+
# File of CSV data. This is not as powerful as Marshal or YAML, but perhaps
|
911
|
+
# useful for spreadsheet and database interaction.
|
912
|
+
#
|
913
|
+
# Out of the box, this method is intended to work with simple data objects or
|
914
|
+
# Structs. It will serialize a list of instance variables and/or
|
915
|
+
# Struct.members().
|
916
|
+
#
|
917
|
+
# If you need need more complicated serialization, you can control the process
|
918
|
+
# by adding methods to the class to be serialized.
|
919
|
+
#
|
920
|
+
# A class method csv_meta() is responsible for returning the first row of the
|
921
|
+
# document (as an Array). This row is considered to be a Hash of the form
|
922
|
+
# key_1,value_1,key_2,value_2,... FasterCSV::load() expects to find a class
|
923
|
+
# key with a value of the stringified class name and FasterCSV::dump() will
|
924
|
+
# create this, if you do not define this method. This method is only called
|
925
|
+
# on the first object of the Array.
|
926
|
+
#
|
927
|
+
# The next method you can provide is an instance method called csv_headers().
|
928
|
+
# This method is expected to return the second line of the document (again as
|
929
|
+
# an Array), which is to be used to give each column a header. By default,
|
930
|
+
# FasterCSV::load() will set an instance variable if the field header starts
|
931
|
+
# with an @ character or call send() passing the header as the method name and
|
932
|
+
# the field value as an argument. This method is only called on the first
|
933
|
+
# object of the Array.
|
934
|
+
#
|
935
|
+
# Finally, you can provide an instance method called csv_dump(), which will
|
936
|
+
# be passed the headers. This should return an Array of fields that can be
|
937
|
+
# serialized for this object. This method is called once for every object in
|
938
|
+
# the Array.
|
939
|
+
#
|
940
|
+
# The +io+ parameter can be used to serialize to a File, and +options+ can be
|
941
|
+
# anything FasterCSV::new() accepts.
|
942
|
+
#
|
943
|
+
def self.dump(ary_of_objs, io = "", options = Hash.new)
|
944
|
+
obj_template = ary_of_objs.first
|
945
|
+
|
946
|
+
csv = FasterCSV.new(io, options)
|
947
|
+
|
948
|
+
# write meta information
|
949
|
+
begin
|
950
|
+
csv << obj_template.class.csv_meta
|
951
|
+
rescue NoMethodError
|
952
|
+
csv << [:class, obj_template.class]
|
519
953
|
end
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
# headers()) or a FasterCSV::Row.
|
529
|
-
#
|
530
|
-
# Columns may be set to a single value, which is copied to each row of the
|
531
|
-
# column, or an Array of values. Arrays of values are assigned to rows top
|
532
|
-
# to bottom in row major order. Excess values are ignored and if the Array
|
533
|
-
# does not have a value for each row the extra rows will receive a +nil+.
|
534
|
-
#
|
535
|
-
# Assigning to an existing column or row clobbers the data. Assigning to
|
536
|
-
# new columns creates them at the right end of the table.
|
537
|
-
#
|
538
|
-
def []=(index_or_header, value)
|
539
|
-
if @mode == :row or # by index
|
540
|
-
(@mode == :col_or_row and index_or_header.is_a? Integer)
|
541
|
-
if value.is_a? Array
|
542
|
-
@table[index_or_header] = Row.new(headers, value)
|
543
|
-
else
|
544
|
-
@table[index_or_header] = value
|
954
|
+
|
955
|
+
# write headers
|
956
|
+
begin
|
957
|
+
headers = obj_template.csv_headers
|
958
|
+
rescue NoMethodError
|
959
|
+
headers = obj_template.instance_variables.sort
|
960
|
+
if obj_template.class.ancestors.find { |cls| cls.to_s =~ /\AStruct\b/ }
|
961
|
+
headers += obj_template.members.map { |mem| "#{mem}=" }.sort
|
545
962
|
end
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
if row.header_row?
|
558
|
-
row[index_or_header] = index_or_header
|
963
|
+
end
|
964
|
+
csv << headers
|
965
|
+
|
966
|
+
# serialize each object
|
967
|
+
ary_of_objs.each do |obj|
|
968
|
+
begin
|
969
|
+
csv << obj.csv_dump(headers)
|
970
|
+
rescue NoMethodError
|
971
|
+
csv << headers.map do |var|
|
972
|
+
if var[0] == ?@
|
973
|
+
obj.instance_variable_get(var)
|
559
974
|
else
|
560
|
-
|
975
|
+
obj[var[0..-2]]
|
561
976
|
end
|
562
977
|
end
|
563
978
|
end
|
564
979
|
end
|
980
|
+
|
981
|
+
if io.is_a? String
|
982
|
+
csv.string
|
983
|
+
else
|
984
|
+
csv.close
|
985
|
+
end
|
565
986
|
end
|
566
|
-
|
987
|
+
|
567
988
|
#
|
568
|
-
#
|
569
|
-
#
|
570
|
-
#
|
571
|
-
#
|
572
|
-
#
|
573
|
-
#
|
574
|
-
#
|
575
|
-
#
|
576
|
-
|
577
|
-
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
|
582
|
-
|
583
|
-
|
584
|
-
|
585
|
-
|
586
|
-
|
989
|
+
# :call-seq:
|
990
|
+
# filter( options = Hash.new ) { |row| ... }
|
991
|
+
# filter( input, options = Hash.new ) { |row| ... }
|
992
|
+
# filter( input, output, options = Hash.new ) { |row| ... }
|
993
|
+
#
|
994
|
+
# This method is a convenience for building Unix-like filters for CSV data.
|
995
|
+
# Each row is yielded to the provided block which can alter it as needed.
|
996
|
+
# After the block returns, the row is appended to +output+ altered or not.
|
997
|
+
#
|
998
|
+
# The +input+ and +output+ arguments can be anything FasterCSV::new() accepts
|
999
|
+
# (generally String or IO objects). If not given, they default to
|
1000
|
+
# <tt>ARGF</tt> and <tt>$stdout</tt>.
|
1001
|
+
#
|
1002
|
+
# The +options+ parameter is also filtered down to FasterCSV::new() after some
|
1003
|
+
# clever key parsing. Any key beginning with <tt>:in_</tt> or
|
1004
|
+
# <tt>:input_</tt> will have that leading identifier stripped and will only
|
1005
|
+
# be used in the +options+ Hash for the +input+ object. Keys starting with
|
1006
|
+
# <tt>:out_</tt> or <tt>:output_</tt> affect only +output+. All other keys
|
1007
|
+
# are assigned to both objects.
|
1008
|
+
#
|
1009
|
+
# The <tt>:output_row_sep</tt> +option+ defaults to
|
1010
|
+
# <tt>$INPUT_RECORD_SEPARATOR</tt> (<tt>$/</tt>).
|
1011
|
+
#
|
1012
|
+
def self.filter(*args)
|
1013
|
+
# parse options for input, output, or both
|
1014
|
+
in_options, out_options = Hash.new, {:row_sep => $INPUT_RECORD_SEPARATOR}
|
1015
|
+
if args.last.is_a? Hash
|
1016
|
+
args.pop.each do |key, value|
|
1017
|
+
case key.to_s
|
1018
|
+
when /\Ain(?:put)?_(.+)\Z/
|
1019
|
+
in_options[$1.to_sym] = value
|
1020
|
+
when /\Aout(?:put)?_(.+)\Z/
|
1021
|
+
out_options[$1.to_sym] = value
|
1022
|
+
else
|
1023
|
+
in_options[key] = value
|
1024
|
+
out_options[key] = value
|
1025
|
+
end
|
1026
|
+
end
|
1027
|
+
end
|
1028
|
+
# build input and output wrappers
|
1029
|
+
input = FasterCSV.new(args.shift || ARGF, in_options)
|
1030
|
+
output = FasterCSV.new(args.shift || $stdout, out_options)
|
1031
|
+
|
1032
|
+
# read, yield, write
|
1033
|
+
input.each do |row|
|
1034
|
+
yield row
|
1035
|
+
output << row
|
587
1036
|
end
|
588
1037
|
end
|
589
1038
|
|
590
1039
|
#
|
591
|
-
#
|
592
|
-
#
|
593
|
-
#
|
1040
|
+
# This method is intended as the primary interface for reading CSV files. You
|
1041
|
+
# pass a +path+ and any +options+ you wish to set for the read. Each row of
|
1042
|
+
# file will be passed to the provided +block+ in turn.
|
594
1043
|
#
|
595
|
-
#
|
1044
|
+
# The +options+ parameter can be anything FasterCSV::new() understands.
|
596
1045
|
#
|
597
|
-
def
|
598
|
-
|
599
|
-
|
600
|
-
else # append Row
|
601
|
-
@table << row_or_array
|
1046
|
+
def self.foreach(path, options = Hash.new, &block)
|
1047
|
+
open(path, "rb", options) do |csv|
|
1048
|
+
csv.each(&block)
|
602
1049
|
end
|
603
|
-
|
604
|
-
self # for chaining
|
605
1050
|
end
|
606
|
-
|
607
|
-
#
|
608
|
-
# A shortcut for appending multiple rows. Equivalent to:
|
1051
|
+
|
609
1052
|
#
|
610
|
-
#
|
1053
|
+
# :call-seq:
|
1054
|
+
# generate( str, options = Hash.new ) { |faster_csv| ... }
|
1055
|
+
# generate( options = Hash.new ) { |faster_csv| ... }
|
611
1056
|
#
|
612
|
-
# This method
|
1057
|
+
# This method wraps a String you provide, or an empty default String, in a
|
1058
|
+
# FasterCSV object which is passed to the provided block. You can use the
|
1059
|
+
# block to append CSV rows to the String and when the block exits, the
|
1060
|
+
# final String will be returned.
|
613
1061
|
#
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
self # for chaining
|
618
|
-
end
|
619
|
-
|
1062
|
+
# Note that a passed String *is* modfied by this method. Call dup() before
|
1063
|
+
# passing if you need a new String.
|
620
1064
|
#
|
621
|
-
#
|
622
|
-
# mode indices refer to rows and everything else is assumed to be a column
|
623
|
-
# header. Use by_col!() or by_row!() to force the lookup.
|
1065
|
+
# The +options+ parameter can be anthing FasterCSV::new() understands.
|
624
1066
|
#
|
625
|
-
def
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
1067
|
+
def self.generate(*args)
|
1068
|
+
# add a default empty String, if none was given
|
1069
|
+
if args.first.is_a? String
|
1070
|
+
io = StringIO.new(args.shift)
|
1071
|
+
io.seek(0, IO::SEEK_END)
|
1072
|
+
args.unshift(io)
|
1073
|
+
else
|
1074
|
+
args.unshift("")
|
631
1075
|
end
|
1076
|
+
faster_csv = new(*args) # wrap
|
1077
|
+
yield faster_csv # yield for appending
|
1078
|
+
faster_csv.string # return final String
|
632
1079
|
end
|
633
|
-
|
1080
|
+
|
634
1081
|
#
|
635
|
-
#
|
636
|
-
#
|
637
|
-
# walking of rows. In column mode, interation will +yield+ two element
|
638
|
-
# tuples containing the column name and an Array of values for that column.
|
1082
|
+
# This method is a shortcut for converting a single row (Array) into a CSV
|
1083
|
+
# String.
|
639
1084
|
#
|
640
|
-
#
|
1085
|
+
# The +options+ parameter can be anthing FasterCSV::new() understands.
|
641
1086
|
#
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
to_delete << header if block[[header, self[header]]]
|
649
|
-
end
|
650
|
-
to_delete.map { |header| delete(header) }
|
651
|
-
end
|
652
|
-
|
653
|
-
self # for chaining
|
1087
|
+
# The <tt>:row_sep</tt> +option+ defaults to <tt>$INPUT_RECORD_SEPARATOR</tt>
|
1088
|
+
# (<tt>$/</tt>) when calling this method.
|
1089
|
+
#
|
1090
|
+
def self.generate_line(row, options = Hash.new)
|
1091
|
+
options = {:row_sep => $INPUT_RECORD_SEPARATOR}.merge(options)
|
1092
|
+
(new("", options) << row).string
|
654
1093
|
end
|
655
|
-
|
656
|
-
include Enumerable
|
657
|
-
|
1094
|
+
|
658
1095
|
#
|
659
|
-
#
|
660
|
-
#
|
661
|
-
#
|
1096
|
+
# This method will return a FasterCSV instance, just like FasterCSV::new(),
|
1097
|
+
# but the instance will be cached and returned for all future calls to this
|
1098
|
+
# method for the same +data+ object (tested by Object#object_id()) with the
|
1099
|
+
# same +options+.
|
662
1100
|
#
|
663
|
-
#
|
1101
|
+
# If a block is given, the instance is passed to the block and the return
|
1102
|
+
# value becomes the return value of the block.
|
664
1103
|
#
|
665
|
-
def
|
666
|
-
|
667
|
-
|
1104
|
+
def self.instance(data = $stdout, options = Hash.new)
|
1105
|
+
# create a _signature_ for this method call, data object and options
|
1106
|
+
sig = [data.object_id] +
|
1107
|
+
options.values_at(*DEFAULT_OPTIONS.keys.sort_by { |sym| sym.to_s })
|
1108
|
+
|
1109
|
+
# fetch or create the instance for this signature
|
1110
|
+
@@instances ||= Hash.new
|
1111
|
+
instance = (@@instances[sig] ||= new(data, options))
|
1112
|
+
|
1113
|
+
if block_given?
|
1114
|
+
yield instance # run block, if given, returning result
|
668
1115
|
else
|
669
|
-
|
1116
|
+
instance # or return the instance
|
670
1117
|
end
|
671
|
-
|
672
|
-
self # for chaining
|
673
1118
|
end
|
674
|
-
|
675
|
-
# Returns +true+ if all rows of this table ==() +other+'s rows.
|
676
|
-
def ==(other)
|
677
|
-
@table == other.table
|
678
|
-
end
|
679
|
-
|
1119
|
+
|
680
1120
|
#
|
681
|
-
#
|
682
|
-
#
|
1121
|
+
# This method is the reading counterpart to FasterCSV::dump(). See that
|
1122
|
+
# method for a detailed description of the process.
|
683
1123
|
#
|
684
|
-
|
685
|
-
|
686
|
-
|
687
|
-
array
|
688
|
-
else
|
689
|
-
array + [row.fields]
|
690
|
-
end
|
691
|
-
end
|
692
|
-
end
|
693
|
-
|
1124
|
+
# You can customize loading by adding a class method called csv_load() which
|
1125
|
+
# will be passed a Hash of meta information, an Array of headers, and an Array
|
1126
|
+
# of fields for the object the method is expected to return.
|
694
1127
|
#
|
695
|
-
#
|
696
|
-
#
|
1128
|
+
# Remember that all fields will be Strings after this load. If you need
|
1129
|
+
# something else, use +options+ to setup converters or provide a custom
|
1130
|
+
# csv_load() implementation.
|
697
1131
|
#
|
698
|
-
def
|
699
|
-
|
700
|
-
if row.header_row?
|
701
|
-
rows
|
702
|
-
else
|
703
|
-
rows + [row.fields.to_csv(options)]
|
704
|
-
end
|
705
|
-
end.join
|
706
|
-
end
|
707
|
-
alias_method :to_s, :to_csv
|
708
|
-
|
709
|
-
def inspect
|
710
|
-
"#<#{self.class} mode:#{@mode} row_count:#{to_a.size}>"
|
711
|
-
end
|
712
|
-
end
|
1132
|
+
def self.load(io_or_str, options = Hash.new)
|
1133
|
+
csv = FasterCSV.new(io_or_str, options)
|
713
1134
|
|
714
|
-
|
715
|
-
|
716
|
-
|
717
|
-
|
718
|
-
|
719
|
-
# source it was read from. FasterCSV will pass this Struct to some blocks
|
720
|
-
# that make decisions based on field structure. See
|
721
|
-
# FasterCSV.convert_fields() for an example.
|
722
|
-
#
|
723
|
-
# <b><tt>index</tt></b>:: The zero-based index of the field in its row.
|
724
|
-
# <b><tt>line</tt></b>:: The line of the data source this row is from.
|
725
|
-
# <b><tt>header</tt></b>:: The header for the column, when available.
|
726
|
-
#
|
727
|
-
FieldInfo = Struct.new(:index, :line, :header)
|
728
|
-
|
729
|
-
# A Regexp used to find and convert some common Date formats.
|
730
|
-
DateMatcher = / \A(?: (\w+,?\s+)?\w+\s+\d{1,2},?\s+\d{2,4} |
|
731
|
-
\d{4}-\d{2}-\d{2} )\z /x
|
732
|
-
# A Regexp used to find and convert some common DateTime formats.
|
733
|
-
DateTimeMatcher =
|
734
|
-
/ \A(?: (\w+,?\s+)?\w+\s+\d{1,2}\s+\d{1,2}:\d{1,2}:\d{1,2},?\s+\d{2,4} |
|
735
|
-
\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2} )\z /x
|
736
|
-
#
|
737
|
-
# This Hash holds the built-in converters of FasterCSV that can be accessed by
|
738
|
-
# name. You can select Converters with FasterCSV.convert() or through the
|
739
|
-
# +options+ Hash passed to FasterCSV::new().
|
740
|
-
#
|
741
|
-
# <b><tt>:integer</tt></b>:: Converts any field Integer() accepts.
|
742
|
-
# <b><tt>:float</tt></b>:: Converts any field Float() accepts.
|
743
|
-
# <b><tt>:numeric</tt></b>:: A combination of <tt>:integer</tt>
|
744
|
-
# and <tt>:float</tt>.
|
745
|
-
# <b><tt>:date</tt></b>:: Converts any field Date::parse() accepts.
|
746
|
-
# <b><tt>:date_time</tt></b>:: Converts any field DateTime::parse() accepts.
|
747
|
-
# <b><tt>:all</tt></b>:: All built-in converters. A combination of
|
748
|
-
# <tt>:date_time</tt> and <tt>:numeric</tt>.
|
749
|
-
#
|
750
|
-
# This Hash is intetionally left unfrozen and users should feel free to add
|
751
|
-
# values to it that can be accessed by all FasterCSV objects.
|
752
|
-
#
|
753
|
-
# To add a combo field, the value should be an Array of names. Combo fields
|
754
|
-
# can be nested with other combo fields.
|
755
|
-
#
|
756
|
-
Converters = { :integer => lambda { |f| Integer(f) rescue f },
|
757
|
-
:float => lambda { |f| Float(f) rescue f },
|
758
|
-
:numeric => [:integer, :float],
|
759
|
-
:date => lambda { |f|
|
760
|
-
f =~ DateMatcher ? (Date.parse(f) rescue f) : f
|
761
|
-
},
|
762
|
-
:date_time => lambda { |f|
|
763
|
-
f =~ DateTimeMatcher ? (DateTime.parse(f) rescue f) : f
|
764
|
-
},
|
765
|
-
:all => [:date_time, :numeric] }
|
1135
|
+
# load meta information
|
1136
|
+
meta = Hash[*csv.shift]
|
1137
|
+
cls = meta["class"].split("::").inject(Object) do |c, const|
|
1138
|
+
c.const_get(const)
|
1139
|
+
end
|
766
1140
|
|
767
|
-
|
768
|
-
|
769
|
-
|
770
|
-
|
771
|
-
|
772
|
-
|
773
|
-
|
774
|
-
|
775
|
-
|
776
|
-
|
777
|
-
|
778
|
-
|
779
|
-
|
780
|
-
|
781
|
-
|
782
|
-
# can be nested with other combo fields.
|
783
|
-
#
|
784
|
-
HeaderConverters = {
|
785
|
-
:downcase => lambda { |h| h.downcase },
|
786
|
-
:symbol => lambda { |h|
|
787
|
-
h.downcase.tr(" ", "_").delete("^a-z0-9_").to_sym
|
788
|
-
}
|
789
|
-
}
|
790
|
-
|
791
|
-
#
|
792
|
-
# The options used when no overrides are given by calling code. They are:
|
793
|
-
#
|
794
|
-
# <b><tt>:col_sep</tt></b>:: <tt>","</tt>
|
795
|
-
# <b><tt>:row_sep</tt></b>:: <tt>:auto</tt>
|
796
|
-
# <b><tt>:quote_char</tt></b>:: <tt>'"'</tt>
|
797
|
-
# <b><tt>:converters</tt></b>:: +nil+
|
798
|
-
# <b><tt>:unconverted_fields</tt></b>:: +nil+
|
799
|
-
# <b><tt>:headers</tt></b>:: +false+
|
800
|
-
# <b><tt>:return_headers</tt></b>:: +false+
|
801
|
-
# <b><tt>:header_converters</tt></b>:: +nil+
|
802
|
-
# <b><tt>:skip_blanks</tt></b>:: +false+
|
803
|
-
# <b><tt>:force_quotes</tt></b>:: +false+
|
804
|
-
#
|
805
|
-
DEFAULT_OPTIONS = { :col_sep => ",",
|
806
|
-
:row_sep => :auto,
|
807
|
-
:quote_char => '"',
|
808
|
-
:converters => nil,
|
809
|
-
:unconverted_fields => nil,
|
810
|
-
:headers => false,
|
811
|
-
:return_headers => false,
|
812
|
-
:header_converters => nil,
|
813
|
-
:skip_blanks => false,
|
814
|
-
:force_quotes => false }.freeze
|
815
|
-
|
816
|
-
#
|
817
|
-
# This method will build a drop-in replacement for many of the standard CSV
|
818
|
-
# methods. It allows you to write code like:
|
819
|
-
#
|
820
|
-
# begin
|
821
|
-
# require "faster_csv"
|
822
|
-
# FasterCSV.build_csv_interface
|
823
|
-
# rescue LoadError
|
824
|
-
# require "csv"
|
825
|
-
# end
|
826
|
-
# # ... use CSV here ...
|
827
|
-
#
|
828
|
-
# This is not a complete interface with completely identical behavior.
|
829
|
-
# However, it is intended to be close enough that you won't notice the
|
830
|
-
# difference in most cases. CSV methods supported are:
|
831
|
-
#
|
832
|
-
# * foreach()
|
833
|
-
# * generate_line()
|
834
|
-
# * open()
|
835
|
-
# * parse()
|
836
|
-
# * parse_line()
|
837
|
-
# * readlines()
|
838
|
-
#
|
839
|
-
# Be warned that this interface is slower than vanilla FasterCSV due to the
|
840
|
-
# extra layer of method calls. Depending on usage, this can slow it down to
|
841
|
-
# near CSV speeds.
|
842
|
-
#
|
843
|
-
def self.build_csv_interface
|
844
|
-
Object.const_set(:CSV, Class.new).class_eval do
|
845
|
-
def self.foreach(path, rs = :auto, &block) # :nodoc:
|
846
|
-
FasterCSV.foreach(path, :row_sep => rs, &block)
|
847
|
-
end
|
848
|
-
|
849
|
-
def self.generate_line(row, fs = ",", rs = "") # :nodoc:
|
850
|
-
FasterCSV.generate_line(row, :col_sep => fs, :row_sep => rs)
|
851
|
-
end
|
852
|
-
|
853
|
-
def self.open(path, mode, fs = ",", rs = :auto, &block) # :nodoc:
|
854
|
-
if block and mode.include? "r"
|
855
|
-
FasterCSV.open(path, mode, :col_sep => fs, :row_sep => rs) do |csv|
|
856
|
-
csv.each(&block)
|
1141
|
+
# load headers
|
1142
|
+
headers = csv.shift
|
1143
|
+
|
1144
|
+
# unserialize each object stored in the file
|
1145
|
+
results = csv.inject(Array.new) do |all, row|
|
1146
|
+
begin
|
1147
|
+
obj = cls.csv_load(meta, headers, row)
|
1148
|
+
rescue NoMethodError
|
1149
|
+
obj = cls.allocate
|
1150
|
+
headers.zip(row) do |name, value|
|
1151
|
+
if name[0] == ?@
|
1152
|
+
obj.instance_variable_set(name, value)
|
1153
|
+
else
|
1154
|
+
obj.send(name, value)
|
1155
|
+
end
|
857
1156
|
end
|
858
|
-
else
|
859
|
-
FasterCSV.open(path, mode, :col_sep => fs, :row_sep => rs, &block)
|
860
1157
|
end
|
1158
|
+
all << obj
|
861
1159
|
end
|
862
|
-
|
863
|
-
def self.parse(str_or_readable, fs = ",", rs = :auto, &block) # :nodoc:
|
864
|
-
FasterCSV.parse(str_or_readable, :col_sep => fs, :row_sep => rs, &block)
|
865
|
-
end
|
866
|
-
|
867
|
-
def self.parse_line(src, fs = ",", rs = :auto) # :nodoc:
|
868
|
-
FasterCSV.parse_line(src, :col_sep => fs, :row_sep => rs)
|
869
|
-
end
|
870
|
-
|
871
|
-
def self.readlines(path, rs = :auto) # :nodoc:
|
872
|
-
FasterCSV.readlines(path, :row_sep => rs)
|
873
|
-
end
|
874
|
-
end
|
875
|
-
end
|
876
|
-
|
877
|
-
#
|
878
|
-
# This method allows you to serialize an Array of Ruby objects to a String or
|
879
|
-
# File of CSV data. This is not as powerful as Marshal or YAML, but perhaps
|
880
|
-
# useful for spreadsheet and database interaction.
|
881
|
-
#
|
882
|
-
# Out of the box, this method is intended to work with simple data objects or
|
883
|
-
# Structs. It will serialize a list of instance variables and/or
|
884
|
-
# Struct.members().
|
885
|
-
#
|
886
|
-
# If you need need more complicated serialization, you can control the process
|
887
|
-
# by adding methods to the class to be serialized.
|
888
|
-
#
|
889
|
-
# A class method csv_meta() is responsible for returning the first row of the
|
890
|
-
# document (as an Array). This row is considered to be a Hash of the form
|
891
|
-
# key_1,value_1,key_2,value_2,... FasterCSV::load() expects to find a class
|
892
|
-
# key with a value of the stringified class name and FasterCSV::dump() will
|
893
|
-
# create this, if you do not define this method. This method is only called
|
894
|
-
# on the first object of the Array.
|
895
|
-
#
|
896
|
-
# The next method you can provide is an instance method called csv_headers().
|
897
|
-
# This method is expected to return the second line of the document (again as
|
898
|
-
# an Array), which is to be used to give each column a header. By default,
|
899
|
-
# FasterCSV::load() will set an instance variable if the field header starts
|
900
|
-
# with an @ character or call send() passing the header as the method name and
|
901
|
-
# the field value as an argument. This method is only called on the first
|
902
|
-
# object of the Array.
|
903
|
-
#
|
904
|
-
# Finally, you can provide an instance method called csv_dump(), which will
|
905
|
-
# be passed the headers. This should return an Array of fields that can be
|
906
|
-
# serialized for this object. This method is called once for every object in
|
907
|
-
# the Array.
|
908
|
-
#
|
909
|
-
# The +io+ parameter can be used to serialize to a File, and +options+ can be
|
910
|
-
# anything FasterCSV::new() accepts.
|
911
|
-
#
|
912
|
-
def self.dump(ary_of_objs, io = "", options = Hash.new)
|
913
|
-
obj_template = ary_of_objs.first
|
914
|
-
|
915
|
-
csv = FasterCSV.new(io, options)
|
916
|
-
|
917
|
-
# write meta information
|
918
|
-
begin
|
919
|
-
csv << obj_template.class.csv_meta
|
920
|
-
rescue NoMethodError
|
921
|
-
csv << [:class, obj_template.class]
|
922
|
-
end
|
923
1160
|
|
924
|
-
|
925
|
-
|
926
|
-
|
927
|
-
rescue NoMethodError
|
928
|
-
headers = obj_template.instance_variables.sort
|
929
|
-
if obj_template.class.ancestors.find { |cls| cls.to_s =~ /\AStruct\b/ }
|
930
|
-
headers += obj_template.members.map { |mem| "#{mem}=" }.sort
|
931
|
-
end
|
1161
|
+
csv.close unless io_or_str.is_a? String
|
1162
|
+
|
1163
|
+
results
|
932
1164
|
end
|
933
|
-
|
934
|
-
|
935
|
-
#
|
936
|
-
|
937
|
-
|
938
|
-
|
939
|
-
|
940
|
-
|
941
|
-
|
942
|
-
|
943
|
-
|
944
|
-
|
945
|
-
|
1165
|
+
|
1166
|
+
#
|
1167
|
+
# :call-seq:
|
1168
|
+
# open( filename, mode="rb", options = Hash.new ) { |faster_csv| ... }
|
1169
|
+
# open( filename, mode="rb", options = Hash.new )
|
1170
|
+
#
|
1171
|
+
# This method opens an IO object, and wraps that with FasterCSV. This is
|
1172
|
+
# intended as the primary interface for writing a CSV file.
|
1173
|
+
#
|
1174
|
+
# You may pass any +args+ Ruby's open() understands followed by an optional
|
1175
|
+
# Hash containing any +options+ FasterCSV::new() understands.
|
1176
|
+
#
|
1177
|
+
# This method works like Ruby's open() call, in that it will pass a FasterCSV
|
1178
|
+
# object to a provided block and close it when the block termminates, or it
|
1179
|
+
# will return the FasterCSV object when no block is provided. (*Note*: This
|
1180
|
+
# is different from the standard CSV library which passes rows to the block.
|
1181
|
+
# Use FasterCSV::foreach() for that behavior.)
|
1182
|
+
#
|
1183
|
+
# An opened FasterCSV object will delegate to many IO methods, for
|
1184
|
+
# convenience. You may call:
|
1185
|
+
#
|
1186
|
+
# * binmode()
|
1187
|
+
# * close()
|
1188
|
+
# * close_read()
|
1189
|
+
# * close_write()
|
1190
|
+
# * closed?()
|
1191
|
+
# * eof()
|
1192
|
+
# * eof?()
|
1193
|
+
# * fcntl()
|
1194
|
+
# * fileno()
|
1195
|
+
# * flush()
|
1196
|
+
# * fsync()
|
1197
|
+
# * ioctl()
|
1198
|
+
# * isatty()
|
1199
|
+
# * pid()
|
1200
|
+
# * pos()
|
1201
|
+
# * reopen()
|
1202
|
+
# * seek()
|
1203
|
+
# * stat()
|
1204
|
+
# * sync()
|
1205
|
+
# * sync=()
|
1206
|
+
# * tell()
|
1207
|
+
# * to_i()
|
1208
|
+
# * to_io()
|
1209
|
+
# * tty?()
|
1210
|
+
#
|
1211
|
+
def self.open(*args)
|
1212
|
+
# find the +options+ Hash
|
1213
|
+
options = if args.last.is_a? Hash then args.pop else Hash.new end
|
1214
|
+
# default to a binary open mode
|
1215
|
+
args << "rb" if args.size == 1
|
1216
|
+
# wrap a File opened with the remaining +args+
|
1217
|
+
csv = new(File.open(*args), options)
|
1218
|
+
|
1219
|
+
# handle blocks like Ruby's open(), not like the CSV library
|
1220
|
+
if block_given?
|
1221
|
+
begin
|
1222
|
+
yield csv
|
1223
|
+
ensure
|
1224
|
+
csv.close
|
946
1225
|
end
|
1226
|
+
else
|
1227
|
+
csv
|
947
1228
|
end
|
948
1229
|
end
|
949
|
-
|
950
|
-
|
951
|
-
|
952
|
-
|
953
|
-
|
954
|
-
|
955
|
-
|
956
|
-
|
957
|
-
|
958
|
-
|
959
|
-
|
960
|
-
|
961
|
-
|
962
|
-
|
963
|
-
|
964
|
-
#
|
965
|
-
|
966
|
-
|
967
|
-
|
968
|
-
|
969
|
-
# <tt>ARGF</tt> and <tt>$stdout</tt>.
|
970
|
-
#
|
971
|
-
# The +options+ parameter is also filtered down to FasterCSV::new() after some
|
972
|
-
# clever key parsing. Any key beginning with <tt>:in_</tt> or
|
973
|
-
# <tt>:input_</tt> will have that leading identifier stripped and will only
|
974
|
-
# be used in the +options+ Hash for the +input+ object. Keys starting with
|
975
|
-
# <tt>:out_</tt> or <tt>:output_</tt> affect only +output+. All other keys
|
976
|
-
# are assigned to both objects.
|
977
|
-
#
|
978
|
-
# The <tt>:output_row_sep</tt> +option+ defaults to
|
979
|
-
# <tt>$INPUT_RECORD_SEPARATOR</tt> (<tt>$/</tt>).
|
980
|
-
#
|
981
|
-
def self.filter(*args)
|
982
|
-
# parse options for input, output, or both
|
983
|
-
in_options, out_options = Hash.new, {:row_sep => $INPUT_RECORD_SEPARATOR}
|
984
|
-
if args.last.is_a? Hash
|
985
|
-
args.pop.each do |key, value|
|
986
|
-
case key.to_s
|
987
|
-
when /\Ain(?:put)?_(.+)\Z/
|
988
|
-
in_options[$1.to_sym] = value
|
989
|
-
when /\Aout(?:put)?_(.+)\Z/
|
990
|
-
out_options[$1.to_sym] = value
|
991
|
-
else
|
992
|
-
in_options[key] = value
|
993
|
-
out_options[key] = value
|
1230
|
+
|
1231
|
+
#
|
1232
|
+
# :call-seq:
|
1233
|
+
# parse( str, options = Hash.new ) { |row| ... }
|
1234
|
+
# parse( str, options = Hash.new )
|
1235
|
+
#
|
1236
|
+
# This method can be used to easily parse CSV out of a String. You may either
|
1237
|
+
# provide a +block+ which will be called with each row of the String in turn,
|
1238
|
+
# or just use the returned Array of Arrays (when no +block+ is given).
|
1239
|
+
#
|
1240
|
+
# You pass your +str+ to read from, and an optional +options+ Hash containing
|
1241
|
+
# anything FasterCSV::new() understands.
|
1242
|
+
#
|
1243
|
+
def self.parse(*args, &block)
|
1244
|
+
csv = new(*args)
|
1245
|
+
if block.nil? # slurp contents, if no block is given
|
1246
|
+
begin
|
1247
|
+
csv.read
|
1248
|
+
ensure
|
1249
|
+
csv.close
|
994
1250
|
end
|
1251
|
+
else # or pass each row to a provided block
|
1252
|
+
csv.each(&block)
|
995
1253
|
end
|
996
1254
|
end
|
997
|
-
|
998
|
-
|
999
|
-
|
1000
|
-
|
1001
|
-
#
|
1002
|
-
|
1003
|
-
|
1004
|
-
|
1005
|
-
|
1006
|
-
|
1007
|
-
|
1008
|
-
#
|
1009
|
-
# This method is intended as the primary interface for reading CSV files. You
|
1010
|
-
# pass a +path+ and any +options+ you wish to set for the read. Each row of
|
1011
|
-
# file will be passed to the provided +block+ in turn.
|
1012
|
-
#
|
1013
|
-
# The +options+ parameter can be anything FasterCSV::new() understands.
|
1014
|
-
#
|
1015
|
-
def self.foreach(path, options = Hash.new, &block)
|
1016
|
-
open(path, "rb", options) do |csv|
|
1017
|
-
csv.each(&block)
|
1255
|
+
|
1256
|
+
#
|
1257
|
+
# This method is a shortcut for converting a single line of a CSV String into
|
1258
|
+
# a into an Array. Note that if +line+ contains multiple rows, anything
|
1259
|
+
# beyond the first row is ignored.
|
1260
|
+
#
|
1261
|
+
# The +options+ parameter can be anything FasterCSV::new() understands.
|
1262
|
+
#
|
1263
|
+
def self.parse_line(line, options = Hash.new)
|
1264
|
+
new(line, options).shift
|
1018
1265
|
end
|
1019
|
-
end
|
1020
1266
|
|
1021
|
-
|
1022
|
-
|
1023
|
-
|
1024
|
-
|
1025
|
-
|
1026
|
-
|
1027
|
-
# FasterCSV object which is passed to the provided block. You can use the
|
1028
|
-
# block to append CSV rows to the String and when the block exits, the
|
1029
|
-
# final String will be returned.
|
1030
|
-
#
|
1031
|
-
# Note that a passed String *is* modfied by this method. Call dup() before
|
1032
|
-
# passing if you need a new String.
|
1033
|
-
#
|
1034
|
-
# The +options+ parameter can be anthing FasterCSV::new() understands.
|
1035
|
-
#
|
1036
|
-
def self.generate(*args)
|
1037
|
-
# add a default empty String, if none was given
|
1038
|
-
if args.first.is_a? String
|
1039
|
-
io = StringIO.new(args.shift)
|
1040
|
-
io.seek(0, IO::SEEK_END)
|
1041
|
-
args.unshift(io)
|
1042
|
-
else
|
1043
|
-
args.unshift("")
|
1267
|
+
#
|
1268
|
+
# Use to slurp a CSV file into an Array of Arrays. Pass the +path+ to the
|
1269
|
+
# file and any +options+ FasterCSV::new() understands.
|
1270
|
+
#
|
1271
|
+
def self.read(path, options = Hash.new)
|
1272
|
+
open(path, "rb", options) { |csv| csv.read }
|
1044
1273
|
end
|
1045
|
-
faster_csv = new(*args) # wrap
|
1046
|
-
yield faster_csv # yield for appending
|
1047
|
-
faster_csv.string # return final String
|
1048
|
-
end
|
1049
1274
|
|
1050
|
-
|
1051
|
-
|
1052
|
-
|
1053
|
-
#
|
1054
|
-
# The +options+ parameter can be anthing FasterCSV::new() understands.
|
1055
|
-
#
|
1056
|
-
# The <tt>:row_sep</tt> +option+ defaults to <tt>$INPUT_RECORD_SEPARATOR</tt>
|
1057
|
-
# (<tt>$/</tt>) when calling this method.
|
1058
|
-
#
|
1059
|
-
def self.generate_line(row, options = Hash.new)
|
1060
|
-
options = {:row_sep => $INPUT_RECORD_SEPARATOR}.merge(options)
|
1061
|
-
(new("", options) << row).string
|
1062
|
-
end
|
1063
|
-
|
1064
|
-
#
|
1065
|
-
# This method will return a FasterCSV instance, just like FasterCSV::new(),
|
1066
|
-
# but the instance will be cached and returned for all future calls to this
|
1067
|
-
# method for the same +data+ object (tested by Object#object_id()) with the
|
1068
|
-
# same +options+.
|
1069
|
-
#
|
1070
|
-
# If a block is given, the instance is passed to the block and the return
|
1071
|
-
# value becomes the return value of the block.
|
1072
|
-
#
|
1073
|
-
def self.instance(data = $stdout, options = Hash.new)
|
1074
|
-
# create a _signature_ for this method call, data object and options
|
1075
|
-
sig = [data.object_id] +
|
1076
|
-
options.values_at(*DEFAULT_OPTIONS.keys.sort_by { |sym| sym.to_s })
|
1077
|
-
|
1078
|
-
# fetch or create the instance for this signature
|
1079
|
-
@@instances ||= Hash.new
|
1080
|
-
instance = (@@instances[sig] ||= new(data, options))
|
1081
|
-
|
1082
|
-
if block_given?
|
1083
|
-
yield instance # run block, if given, returning result
|
1084
|
-
else
|
1085
|
-
instance # or return the instance
|
1275
|
+
# Alias for FasterCSV::read().
|
1276
|
+
def self.readlines(*args)
|
1277
|
+
read(*args)
|
1086
1278
|
end
|
1087
|
-
|
1088
|
-
|
1089
|
-
|
1090
|
-
|
1091
|
-
|
1092
|
-
|
1093
|
-
|
1094
|
-
|
1095
|
-
|
1096
|
-
|
1097
|
-
|
1098
|
-
|
1099
|
-
# csv_load() implementation.
|
1100
|
-
#
|
1101
|
-
def self.load(io_or_str, options = Hash.new)
|
1102
|
-
csv = FasterCSV.new(io_or_str, options)
|
1103
|
-
|
1104
|
-
# load meta information
|
1105
|
-
meta = Hash[*csv.shift]
|
1106
|
-
cls = meta["class"].split("::").inject(Object) do |c, const|
|
1107
|
-
c.const_get(const)
|
1279
|
+
|
1280
|
+
#
|
1281
|
+
# A shortcut for:
|
1282
|
+
#
|
1283
|
+
# FasterCSV.read( path, { :headers => true,
|
1284
|
+
# :converters => :numeric,
|
1285
|
+
# :header_converters => :symbol }.merge(options) )
|
1286
|
+
#
|
1287
|
+
def self.table(path, options = Hash.new)
|
1288
|
+
read( path, { :headers => true,
|
1289
|
+
:converters => :numeric,
|
1290
|
+
:header_converters => :symbol }.merge(options) )
|
1108
1291
|
end
|
1109
|
-
|
1110
|
-
#
|
1111
|
-
|
1112
|
-
|
1113
|
-
#
|
1114
|
-
|
1115
|
-
|
1116
|
-
|
1117
|
-
|
1118
|
-
|
1119
|
-
|
1120
|
-
|
1121
|
-
|
1122
|
-
|
1123
|
-
|
1124
|
-
|
1125
|
-
|
1292
|
+
|
1293
|
+
#
|
1294
|
+
# This constructor will wrap either a String or IO object passed in +data+ for
|
1295
|
+
# reading and/or writing. In addition to the FasterCSV instance methods,
|
1296
|
+
# several IO methods are delegated. (See FasterCSV::open() for a complete
|
1297
|
+
# list.) If you pass a String for +data+, you can later retrieve it (after
|
1298
|
+
# writing to it, for example) with FasterCSV.string().
|
1299
|
+
#
|
1300
|
+
# Note that a wrapped String will be positioned at the beginning (for
|
1301
|
+
# reading). If you want it at the end (for writing), use
|
1302
|
+
# FasterCSV::generate(). If you want any other positioning, pass a preset
|
1303
|
+
# StringIO object instead.
|
1304
|
+
#
|
1305
|
+
# You may set any reading and/or writing preferences in the +options+ Hash.
|
1306
|
+
# Available options are:
|
1307
|
+
#
|
1308
|
+
# <b><tt>:col_sep</tt></b>:: The String placed between each field.
|
1309
|
+
# <b><tt>:row_sep</tt></b>:: The String appended to the end of each
|
1310
|
+
# row. This can be set to the special
|
1311
|
+
# <tt>:auto</tt> setting, which requests
|
1312
|
+
# that FasterCSV automatically discover
|
1313
|
+
# this from the data. Auto-discovery
|
1314
|
+
# reads ahead in the data looking for
|
1315
|
+
# the next <tt>"\r\n"</tt>,
|
1316
|
+
# <tt>"\n"</tt>, or <tt>"\r"</tt>
|
1317
|
+
# sequence. A sequence will be selected
|
1318
|
+
# even if it occurs in a quoted field,
|
1319
|
+
# assuming that you would have the same
|
1320
|
+
# line endings there. If none of those
|
1321
|
+
# sequences is found,
|
1322
|
+
# or the stream is only available for
|
1323
|
+
# output, the default
|
1324
|
+
# <tt>$INPUT_RECORD_SEPARATOR</tt>
|
1325
|
+
# (<tt>$/</tt>) is used. Obviously,
|
1326
|
+
# discovery takes a little time. Set
|
1327
|
+
# manually if speed is important. Also
|
1328
|
+
# note that IO objects should be opened
|
1329
|
+
# in binary mode on Windows if this
|
1330
|
+
# feature will be used as the
|
1331
|
+
# line-ending translation can cause
|
1332
|
+
# problems with resetting the document
|
1333
|
+
# position to where it was before the
|
1334
|
+
# read ahead.
|
1335
|
+
# <b><tt>:quote_char</tt></b>:: The character used to quote fields.
|
1336
|
+
# This has to be a single character
|
1337
|
+
# String. This is useful for
|
1338
|
+
# application that incorrectly use
|
1339
|
+
# <tt>'</tt> as the quote character
|
1340
|
+
# instead of the correct <tt>"</tt>.
|
1341
|
+
# FasterCSV will always consider a
|
1342
|
+
# double sequence this character to be
|
1343
|
+
# an escaped quote.
|
1344
|
+
# <b><tt>:encoding</tt></b>:: The encoding to use when parsing the
|
1345
|
+
# file. Defaults to your <tt>$KCODE</tt>
|
1346
|
+
# setting. Valid values: <tt>`n’</tt> or
|
1347
|
+
# <tt>`N’</tt> for none, <tt>`e’</tt> or
|
1348
|
+
# <tt>`E’</tt> for EUC, <tt>`s’</tt> or
|
1349
|
+
# <tt>`S’</tt> for SJIS, and
|
1350
|
+
# <tt>`u’</tt> or <tt>`U’</tt> for UTF-8
|
1351
|
+
# (see Regexp.new()).
|
1352
|
+
# <b><tt>:field_size_limit</tt></b>:: This is a maximum size FasterCSV will
|
1353
|
+
# read ahead looking for the closing
|
1354
|
+
# quote for a field. (In truth, it
|
1355
|
+
# reads to the first line ending beyond
|
1356
|
+
# this size.) If a quote cannot be
|
1357
|
+
# found within the limit FasterCSV will
|
1358
|
+
# raise a MalformedCSVError, assuming
|
1359
|
+
# the data is faulty. You can use this
|
1360
|
+
# limit to prevent what are effectively
|
1361
|
+
# DoS attacks on the parser. However,
|
1362
|
+
# this limit can cause a legitimate
|
1363
|
+
# parse to fail and thus is set to
|
1364
|
+
# +nil+, or off, by default.
|
1365
|
+
# <b><tt>:converters</tt></b>:: An Array of names from the Converters
|
1366
|
+
# Hash and/or lambdas that handle custom
|
1367
|
+
# conversion. A single converter
|
1368
|
+
# doesn't have to be in an Array.
|
1369
|
+
# <b><tt>:unconverted_fields</tt></b>:: If set to +true+, an
|
1370
|
+
# unconverted_fields() method will be
|
1371
|
+
# added to all returned rows (Array or
|
1372
|
+
# FasterCSV::Row) that will return the
|
1373
|
+
# fields as they were before convertion.
|
1374
|
+
# Note that <tt>:headers</tt> supplied
|
1375
|
+
# by Array or String were not fields of
|
1376
|
+
# the document and thus will have an
|
1377
|
+
# empty Array attached.
|
1378
|
+
# <b><tt>:headers</tt></b>:: If set to <tt>:first_row</tt> or
|
1379
|
+
# +true+, the initial row of the CSV
|
1380
|
+
# file will be treated as a row of
|
1381
|
+
# headers. If set to an Array, the
|
1382
|
+
# contents will be used as the headers.
|
1383
|
+
# If set to a String, the String is run
|
1384
|
+
# through a call of
|
1385
|
+
# FasterCSV::parse_line() with the same
|
1386
|
+
# <tt>:col_sep</tt>, <tt>:row_sep</tt>,
|
1387
|
+
# and <tt>:quote_char</tt> as this
|
1388
|
+
# instance to produce an Array of
|
1389
|
+
# headers. This setting causes
|
1390
|
+
# FasterCSV.shift() to return rows as
|
1391
|
+
# FasterCSV::Row objects instead of
|
1392
|
+
# Arrays and FasterCSV.read() to return
|
1393
|
+
# FasterCSV::Table objects instead of
|
1394
|
+
# an Array of Arrays.
|
1395
|
+
# <b><tt>:return_headers</tt></b>:: When +false+, header rows are silently
|
1396
|
+
# swallowed. If set to +true+, header
|
1397
|
+
# rows are returned in a FasterCSV::Row
|
1398
|
+
# object with identical headers and
|
1399
|
+
# fields (save that the fields do not go
|
1400
|
+
# through the converters).
|
1401
|
+
# <b><tt>:write_headers</tt></b>:: When +true+ and <tt>:headers</tt> is
|
1402
|
+
# set, a header row will be added to the
|
1403
|
+
# output. Note that if the table only
|
1404
|
+
# contains header rows,
|
1405
|
+
# <tt>:return_headers</tt> must also be
|
1406
|
+
# set in order for a header row to be
|
1407
|
+
# output.
|
1408
|
+
# <b><tt>:header_converters</tt></b>:: Identical in functionality to
|
1409
|
+
# <tt>:converters</tt> save that the
|
1410
|
+
# conversions are only made to header
|
1411
|
+
# rows.
|
1412
|
+
# <b><tt>:skip_blanks</tt></b>:: When set to a +true+ value, FasterCSV
|
1413
|
+
# will skip over any rows with no
|
1414
|
+
# content.
|
1415
|
+
# <b><tt>:force_quotes</tt></b>:: When set to a +true+ value, FasterCSV
|
1416
|
+
# will quote all CSV fields it creates.
|
1417
|
+
#
|
1418
|
+
# See FasterCSV::DEFAULT_OPTIONS for the default settings.
|
1419
|
+
#
|
1420
|
+
# Options cannot be overriden in the instance methods for performance reasons,
|
1421
|
+
# so be sure to set what you want here.
|
1422
|
+
#
|
1423
|
+
def initialize(data, options = Hash.new)
|
1424
|
+
# build the options for this read/write
|
1425
|
+
options = DEFAULT_OPTIONS.merge(options)
|
1426
|
+
|
1427
|
+
# create the IO object we will read from
|
1428
|
+
@io = if data.is_a? String then StringIO.new(data) else data end
|
1429
|
+
|
1430
|
+
init_separators(options)
|
1431
|
+
init_parsers(options)
|
1432
|
+
init_converters(options)
|
1433
|
+
init_headers(options)
|
1434
|
+
|
1435
|
+
unless options.empty?
|
1436
|
+
raise ArgumentError, "Unknown options: #{options.keys.join(', ')}."
|
1126
1437
|
end
|
1127
|
-
|
1438
|
+
|
1439
|
+
# track our own lineno since IO gets confused about line-ends is CSV fields
|
1440
|
+
@lineno = 0
|
1128
1441
|
end
|
1129
|
-
|
1130
|
-
|
1131
|
-
|
1132
|
-
|
1133
|
-
|
1134
|
-
|
1135
|
-
|
1136
|
-
|
1137
|
-
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1145
|
-
|
1146
|
-
|
1147
|
-
|
1148
|
-
|
1149
|
-
|
1150
|
-
# Use FasterCSV::foreach() for that behavior.)
|
1151
|
-
#
|
1152
|
-
# An opened FasterCSV object will delegate to many IO methods, for
|
1153
|
-
# convenience. You may call:
|
1154
|
-
#
|
1155
|
-
# * binmode()
|
1156
|
-
# * close()
|
1157
|
-
# * close_read()
|
1158
|
-
# * close_write()
|
1159
|
-
# * closed?()
|
1160
|
-
# * eof()
|
1161
|
-
# * eof?()
|
1162
|
-
# * fcntl()
|
1163
|
-
# * fileno()
|
1164
|
-
# * flush()
|
1165
|
-
# * fsync()
|
1166
|
-
# * ioctl()
|
1167
|
-
# * isatty()
|
1168
|
-
# * pid()
|
1169
|
-
# * pos()
|
1170
|
-
# * reopen()
|
1171
|
-
# * seek()
|
1172
|
-
# * stat()
|
1173
|
-
# * sync()
|
1174
|
-
# * sync=()
|
1175
|
-
# * tell()
|
1176
|
-
# * to_i()
|
1177
|
-
# * to_io()
|
1178
|
-
# * tty?()
|
1179
|
-
#
|
1180
|
-
def self.open(*args)
|
1181
|
-
# find the +options+ Hash
|
1182
|
-
options = if args.last.is_a? Hash then args.pop else Hash.new end
|
1183
|
-
# default to a binary open mode
|
1184
|
-
args << "rb" if args.size == 1
|
1185
|
-
# wrap a File opened with the remaining +args+
|
1186
|
-
csv = new(File.open(*args), options)
|
1187
|
-
|
1188
|
-
# handle blocks like Ruby's open(), not like the CSV library
|
1189
|
-
if block_given?
|
1190
|
-
begin
|
1191
|
-
yield csv
|
1192
|
-
ensure
|
1193
|
-
csv.close
|
1194
|
-
end
|
1195
|
-
else
|
1196
|
-
csv
|
1442
|
+
|
1443
|
+
#
|
1444
|
+
# The line number of the last row read from this file. Fields with nested
|
1445
|
+
# line-end characters will not affect this count.
|
1446
|
+
#
|
1447
|
+
attr_reader :lineno
|
1448
|
+
|
1449
|
+
### IO and StringIO Delegation ###
|
1450
|
+
|
1451
|
+
extend Forwardable
|
1452
|
+
def_delegators :@io, :binmode, :close, :close_read, :close_write, :closed?,
|
1453
|
+
:eof, :eof?, :fcntl, :fileno, :flush, :fsync, :ioctl,
|
1454
|
+
:isatty, :pid, :pos, :reopen, :seek, :stat, :string,
|
1455
|
+
:sync, :sync=, :tell, :to_i, :to_io, :tty?
|
1456
|
+
|
1457
|
+
# Rewinds the underlying IO object and resets FasterCSV's lineno() counter.
|
1458
|
+
def rewind
|
1459
|
+
@headers = nil
|
1460
|
+
@lineno = 0
|
1461
|
+
|
1462
|
+
@io.rewind
|
1197
1463
|
end
|
1198
|
-
|
1199
|
-
|
1200
|
-
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
|
1206
|
-
|
1207
|
-
|
1208
|
-
|
1209
|
-
|
1210
|
-
|
1211
|
-
|
1212
|
-
|
1213
|
-
|
1214
|
-
if block.nil? # slurp contents, if no block is given
|
1215
|
-
begin
|
1216
|
-
csv.read
|
1217
|
-
ensure
|
1218
|
-
csv.close
|
1464
|
+
|
1465
|
+
### End Delegation ###
|
1466
|
+
|
1467
|
+
#
|
1468
|
+
# The primary write method for wrapped Strings and IOs, +row+ (an Array or
|
1469
|
+
# FasterCSV::Row) is converted to CSV and appended to the data source. When a
|
1470
|
+
# FasterCSV::Row is passed, only the row's fields() are appended to the
|
1471
|
+
# output.
|
1472
|
+
#
|
1473
|
+
# The data source must be open for writing.
|
1474
|
+
#
|
1475
|
+
def <<(row)
|
1476
|
+
# make sure headers have been assigned
|
1477
|
+
if header_row? and [Array, String].include? @use_headers.class
|
1478
|
+
parse_headers # won't read data for Array or String
|
1479
|
+
self << @headers if @write_headers
|
1219
1480
|
end
|
1220
|
-
|
1221
|
-
|
1222
|
-
|
1223
|
-
|
1224
|
-
|
1225
|
-
|
1226
|
-
|
1227
|
-
|
1228
|
-
|
1229
|
-
|
1230
|
-
|
1231
|
-
#
|
1232
|
-
|
1233
|
-
|
1234
|
-
end
|
1235
|
-
|
1236
|
-
#
|
1237
|
-
# Use to slurp a CSV file into an Array of Arrays. Pass the +path+ to the
|
1238
|
-
# file and any +options+ FasterCSV::new() understands.
|
1239
|
-
#
|
1240
|
-
def self.read(path, options = Hash.new)
|
1241
|
-
open(path, "rb", options) { |csv| csv.read }
|
1242
|
-
end
|
1243
|
-
|
1244
|
-
# Alias for FasterCSV::read().
|
1245
|
-
def self.readlines(*args)
|
1246
|
-
read(*args)
|
1247
|
-
end
|
1248
|
-
|
1249
|
-
#
|
1250
|
-
# A shortcut for:
|
1251
|
-
#
|
1252
|
-
# FasterCSV.read( path, { :headers => true,
|
1253
|
-
# :converters => :numeric,
|
1254
|
-
# :header_converters => :symbol }.merge(options) )
|
1255
|
-
#
|
1256
|
-
def self.table(path, options = Hash.new)
|
1257
|
-
read( path, { :headers => true,
|
1258
|
-
:converters => :numeric,
|
1259
|
-
:header_converters => :symbol }.merge(options) )
|
1260
|
-
end
|
1261
|
-
|
1262
|
-
#
|
1263
|
-
# This constructor will wrap either a String or IO object passed in +data+ for
|
1264
|
-
# reading and/or writing. In addition to the FasterCSV instance methods,
|
1265
|
-
# several IO methods are delegated. (See FasterCSV::open() for a complete
|
1266
|
-
# list.) If you pass a String for +data+, you can later retrieve it (after
|
1267
|
-
# writing to it, for example) with FasterCSV.string().
|
1268
|
-
#
|
1269
|
-
# Note that a wrapped String will be positioned at at the beginning (for
|
1270
|
-
# reading). If you want it at the end (for writing), use
|
1271
|
-
# FasterCSV::generate(). If you want any other positioning, pass a preset
|
1272
|
-
# StringIO object instead.
|
1273
|
-
#
|
1274
|
-
# You may set any reading and/or writing preferences in the +options+ Hash.
|
1275
|
-
# Available options are:
|
1276
|
-
#
|
1277
|
-
# <b><tt>:col_sep</tt></b>:: The String placed between each field.
|
1278
|
-
# <b><tt>:row_sep</tt></b>:: The String appended to the end of each
|
1279
|
-
# row. This can be set to the special
|
1280
|
-
# <tt>:auto</tt> setting, which requests
|
1281
|
-
# that FasterCSV automatically discover
|
1282
|
-
# this from the data. Auto-discovery
|
1283
|
-
# reads ahead in the data looking for
|
1284
|
-
# the next <tt>"\r\n"</tt>,
|
1285
|
-
# <tt>"\n"</tt>, or <tt>"\r"</tt>
|
1286
|
-
# sequence. A sequence will be selected
|
1287
|
-
# even if it occurs in a quoted field,
|
1288
|
-
# assuming that you would have the same
|
1289
|
-
# line endings there. If none of those
|
1290
|
-
# sequences is found, +data+ is
|
1291
|
-
# <tt>ARGF</tt>, <tt>STDIN</tt>,
|
1292
|
-
# <tt>STDOUT</tt>, or <tt>STDERR</tt>,
|
1293
|
-
# or the stream is only available for
|
1294
|
-
# output, the default
|
1295
|
-
# <tt>$INPUT_RECORD_SEPARATOR</tt>
|
1296
|
-
# (<tt>$/</tt>) is used. Obviously,
|
1297
|
-
# discovery takes a little time. Set
|
1298
|
-
# manually if speed is important. Also
|
1299
|
-
# note that IO objects should be opened
|
1300
|
-
# in binary mode on Windows if this
|
1301
|
-
# feature will be used as the
|
1302
|
-
# line-ending translation can cause
|
1303
|
-
# problems with resetting the document
|
1304
|
-
# position to where it was before the
|
1305
|
-
# read ahead.
|
1306
|
-
# <b><tt>:quote_char</tt></b>:: The character used to quote fields.
|
1307
|
-
# This has to be a single character
|
1308
|
-
# String. This is useful for
|
1309
|
-
# application that incorrectly use
|
1310
|
-
# <tt>'</tt> as the quote character
|
1311
|
-
# instead of the correct <tt>"</tt>.
|
1312
|
-
# FasterCSV will always consider a
|
1313
|
-
# double sequence this character to be
|
1314
|
-
# an escaped quote.
|
1315
|
-
# <b><tt>:encoding</tt></b>:: The encoding to use when parsing the
|
1316
|
-
# file. Defaults to your <tt>$KDOCE</tt>
|
1317
|
-
# setting. Valid values: <tt>`n’</tt> or
|
1318
|
-
# <tt>`N’</tt> for none, <tt>`e’</tt> or
|
1319
|
-
# <tt>`E’</tt> for EUC, <tt>`s’</tt> or
|
1320
|
-
# <tt>`S’</tt> for SJIS, and
|
1321
|
-
# <tt>`u’</tt> or <tt>`U’</tt> for UTF-8
|
1322
|
-
# (see Regexp.new()).
|
1323
|
-
# <b><tt>:field_size_limit</tt></b>:: This is a maximum size FasterCSV will
|
1324
|
-
# read ahead looking for the closing
|
1325
|
-
# quote for a field. (In truth, it
|
1326
|
-
# reads to the first line ending beyond
|
1327
|
-
# this size.) If a quote cannot be
|
1328
|
-
# found within the limit FasterCSV will
|
1329
|
-
# raise a MalformedCSVError, assuming
|
1330
|
-
# the data is faulty. You can use this
|
1331
|
-
# limit to prevent what are effectively
|
1332
|
-
# DoS attacks on the parser. However,
|
1333
|
-
# this limit can cause a legitimate
|
1334
|
-
# parse to fail and thus is set to
|
1335
|
-
# +nil+, or off, by default.
|
1336
|
-
# <b><tt>:converters</tt></b>:: An Array of names from the Converters
|
1337
|
-
# Hash and/or lambdas that handle custom
|
1338
|
-
# conversion. A single converter
|
1339
|
-
# doesn't have to be in an Array.
|
1340
|
-
# <b><tt>:unconverted_fields</tt></b>:: If set to +true+, an
|
1341
|
-
# unconverted_fields() method will be
|
1342
|
-
# added to all returned rows (Array or
|
1343
|
-
# FasterCSV::Row) that will return the
|
1344
|
-
# fields as they were before convertion.
|
1345
|
-
# Note that <tt>:headers</tt> supplied
|
1346
|
-
# by Array or String were not fields of
|
1347
|
-
# the document and thus will have an
|
1348
|
-
# empty Array attached.
|
1349
|
-
# <b><tt>:headers</tt></b>:: If set to <tt>:first_row</tt> or
|
1350
|
-
# +true+, the initial row of the CSV
|
1351
|
-
# file will be treated as a row of
|
1352
|
-
# headers. If set to an Array, the
|
1353
|
-
# contents will be used as the headers.
|
1354
|
-
# If set to a String, the String is run
|
1355
|
-
# through a call of
|
1356
|
-
# FasterCSV::parse_line() with the same
|
1357
|
-
# <tt>:col_sep</tt>, <tt>:row_sep</tt>,
|
1358
|
-
# and <tt>:quote_char</tt> as this
|
1359
|
-
# instance to produce an Array of
|
1360
|
-
# headers. This setting causes
|
1361
|
-
# FasterCSV.shift() to return rows as
|
1362
|
-
# FasterCSV::Row objects instead of
|
1363
|
-
# Arrays and FasterCSV.read() to return
|
1364
|
-
# FasterCSV::Table objects instead of
|
1365
|
-
# an Array of Arrays.
|
1366
|
-
# <b><tt>:return_headers</tt></b>:: When +false+, header rows are silently
|
1367
|
-
# swallowed. If set to +true+, header
|
1368
|
-
# rows are returned in a FasterCSV::Row
|
1369
|
-
# object with identical headers and
|
1370
|
-
# fields (save that the fields do not go
|
1371
|
-
# through the converters).
|
1372
|
-
# <b><tt>:write_headers</tt></b>:: When +true+ and <tt>:headers</tt> is
|
1373
|
-
# set, a header row will be added to the
|
1374
|
-
# output.
|
1375
|
-
# <b><tt>:header_converters</tt></b>:: Identical in functionality to
|
1376
|
-
# <tt>:converters</tt> save that the
|
1377
|
-
# conversions are only made to header
|
1378
|
-
# rows.
|
1379
|
-
# <b><tt>:skip_blanks</tt></b>:: When set to a +true+ value, FasterCSV
|
1380
|
-
# will skip over any rows with no
|
1381
|
-
# content.
|
1382
|
-
# <b><tt>:force_quotes</tt></b>:: When set to a +true+ value, FasterCSV
|
1383
|
-
# will quote all CSV fields it creates.
|
1384
|
-
#
|
1385
|
-
# See FasterCSV::DEFAULT_OPTIONS for the default settings.
|
1386
|
-
#
|
1387
|
-
# Options cannot be overriden in the instance methods for performance reasons,
|
1388
|
-
# so be sure to set what you want here.
|
1389
|
-
#
|
1390
|
-
def initialize(data, options = Hash.new)
|
1391
|
-
# build the options for this read/write
|
1392
|
-
options = DEFAULT_OPTIONS.merge(options)
|
1393
|
-
|
1394
|
-
# create the IO object we will read from
|
1395
|
-
@io = if data.is_a? String then StringIO.new(data) else data end
|
1396
|
-
|
1397
|
-
init_separators(options)
|
1398
|
-
init_parsers(options)
|
1399
|
-
init_converters(options)
|
1400
|
-
init_headers(options)
|
1401
|
-
|
1402
|
-
unless options.empty?
|
1403
|
-
raise ArgumentError, "Unknown options: #{options.keys.join(', ')}."
|
1481
|
+
|
1482
|
+
# Handle FasterCSV::Row objects and Hashes
|
1483
|
+
row = case row
|
1484
|
+
when self.class::Row then row.fields
|
1485
|
+
when Hash then @headers.map { |header| row[header] }
|
1486
|
+
else row
|
1487
|
+
end
|
1488
|
+
|
1489
|
+
@headers = row if header_row?
|
1490
|
+
@lineno += 1
|
1491
|
+
|
1492
|
+
@io << row.map(&@quote).join(@col_sep) + @row_sep # quote and separate
|
1493
|
+
|
1494
|
+
self # for chaining
|
1404
1495
|
end
|
1405
|
-
|
1406
|
-
|
1407
|
-
@lineno = 0
|
1408
|
-
end
|
1409
|
-
|
1410
|
-
#
|
1411
|
-
# The line number of the last row read from this file. Fields with nested
|
1412
|
-
# line-end characters will not affect this count.
|
1413
|
-
#
|
1414
|
-
attr_reader :lineno
|
1415
|
-
|
1416
|
-
### IO and StringIO Delegation ###
|
1417
|
-
|
1418
|
-
extend Forwardable
|
1419
|
-
def_delegators :@io, :binmode, :close, :close_read, :close_write, :closed?,
|
1420
|
-
:eof, :eof?, :fcntl, :fileno, :flush, :fsync, :ioctl,
|
1421
|
-
:isatty, :pid, :pos, :reopen, :seek, :stat, :string,
|
1422
|
-
:sync, :sync=, :tell, :to_i, :to_io, :tty?
|
1423
|
-
|
1424
|
-
# Rewinds the underlying IO object and resets FasterCSV's lineno() counter.
|
1425
|
-
def rewind
|
1426
|
-
@headers = nil
|
1427
|
-
@lineno = 0
|
1428
|
-
|
1429
|
-
@io.rewind
|
1430
|
-
end
|
1496
|
+
alias_method :add_row, :<<
|
1497
|
+
alias_method :puts, :<<
|
1431
1498
|
|
1432
|
-
|
1433
|
-
|
1434
|
-
|
1435
|
-
|
1436
|
-
|
1437
|
-
|
1438
|
-
|
1439
|
-
|
1440
|
-
|
1441
|
-
|
1442
|
-
|
1443
|
-
#
|
1444
|
-
|
1445
|
-
|
1446
|
-
|
1499
|
+
#
|
1500
|
+
# :call-seq:
|
1501
|
+
# convert( name )
|
1502
|
+
# convert { |field| ... }
|
1503
|
+
# convert { |field, field_info| ... }
|
1504
|
+
#
|
1505
|
+
# You can use this method to install a FasterCSV::Converters built-in, or
|
1506
|
+
# provide a block that handles a custom conversion.
|
1507
|
+
#
|
1508
|
+
# If you provide a block that takes one argument, it will be passed the field
|
1509
|
+
# and is expected to return the converted value or the field itself. If your
|
1510
|
+
# block takes two arguments, it will also be passed a FieldInfo Struct,
|
1511
|
+
# containing details about the field. Again, the block should return a
|
1512
|
+
# converted field or the field itself.
|
1513
|
+
#
|
1514
|
+
def convert(name = nil, &converter)
|
1515
|
+
add_converter(:converters, self.class::Converters, name, &converter)
|
1447
1516
|
end
|
1448
|
-
|
1449
|
-
# Handle FasterCSV::Row objects and Hashes
|
1450
|
-
row = case row
|
1451
|
-
when self.class::Row then row.fields
|
1452
|
-
when Hash then @headers.map { |header| row[header] }
|
1453
|
-
else row
|
1454
|
-
end
|
1455
1517
|
|
1456
|
-
|
1457
|
-
|
1518
|
+
#
|
1519
|
+
# :call-seq:
|
1520
|
+
# header_convert( name )
|
1521
|
+
# header_convert { |field| ... }
|
1522
|
+
# header_convert { |field, field_info| ... }
|
1523
|
+
#
|
1524
|
+
# Identical to FasterCSV.convert(), but for header rows.
|
1525
|
+
#
|
1526
|
+
# Note that this method must be called before header rows are read to have any
|
1527
|
+
# effect.
|
1528
|
+
#
|
1529
|
+
def header_convert(name = nil, &converter)
|
1530
|
+
add_converter( :header_converters,
|
1531
|
+
self.class::HeaderConverters,
|
1532
|
+
name,
|
1533
|
+
&converter )
|
1534
|
+
end
|
1458
1535
|
|
1459
|
-
|
1460
|
-
|
1461
|
-
self # for chaining
|
1462
|
-
end
|
1463
|
-
alias_method :add_row, :<<
|
1464
|
-
alias_method :puts, :<<
|
1465
|
-
|
1466
|
-
#
|
1467
|
-
# :call-seq:
|
1468
|
-
# convert( name )
|
1469
|
-
# convert { |field| ... }
|
1470
|
-
# convert { |field, field_info| ... }
|
1471
|
-
#
|
1472
|
-
# You can use this method to install a FasterCSV::Converters built-in, or
|
1473
|
-
# provide a block that handles a custom conversion.
|
1474
|
-
#
|
1475
|
-
# If you provide a block that takes one argument, it will be passed the field
|
1476
|
-
# and is expected to return the converted value or the field itself. If your
|
1477
|
-
# block takes two arguments, it will also be passed a FieldInfo Struct,
|
1478
|
-
# containing details about the field. Again, the block should return a
|
1479
|
-
# converted field or the field itself.
|
1480
|
-
#
|
1481
|
-
def convert(name = nil, &converter)
|
1482
|
-
add_converter(:converters, self.class::Converters, name, &converter)
|
1483
|
-
end
|
1536
|
+
include Enumerable
|
1484
1537
|
|
1485
|
-
|
1486
|
-
|
1487
|
-
|
1488
|
-
|
1489
|
-
|
1490
|
-
|
1491
|
-
|
1492
|
-
|
1493
|
-
|
1494
|
-
|
1495
|
-
|
1496
|
-
def header_convert(name = nil, &converter)
|
1497
|
-
add_converter( :header_converters,
|
1498
|
-
self.class::HeaderConverters,
|
1499
|
-
name,
|
1500
|
-
&converter )
|
1501
|
-
end
|
1502
|
-
|
1503
|
-
include Enumerable
|
1504
|
-
|
1505
|
-
#
|
1506
|
-
# Yields each row of the data source in turn.
|
1507
|
-
#
|
1508
|
-
# Support for Enumerable.
|
1509
|
-
#
|
1510
|
-
# The data source must be open for reading.
|
1511
|
-
#
|
1512
|
-
def each
|
1513
|
-
while row = shift
|
1514
|
-
yield row
|
1515
|
-
end
|
1516
|
-
end
|
1517
|
-
|
1518
|
-
#
|
1519
|
-
# Slurps the remaining rows and returns an Array of Arrays.
|
1520
|
-
#
|
1521
|
-
# The data source must be open for reading.
|
1522
|
-
#
|
1523
|
-
def read
|
1524
|
-
rows = to_a
|
1525
|
-
if @use_headers
|
1526
|
-
Table.new(rows)
|
1527
|
-
else
|
1528
|
-
rows
|
1538
|
+
#
|
1539
|
+
# Yields each row of the data source in turn.
|
1540
|
+
#
|
1541
|
+
# Support for Enumerable.
|
1542
|
+
#
|
1543
|
+
# The data source must be open for reading.
|
1544
|
+
#
|
1545
|
+
def each
|
1546
|
+
while row = shift
|
1547
|
+
yield row
|
1548
|
+
end
|
1529
1549
|
end
|
1530
|
-
|
1531
|
-
|
1532
|
-
|
1533
|
-
|
1534
|
-
|
1535
|
-
|
1536
|
-
|
1537
|
-
|
1538
|
-
|
1539
|
-
|
1540
|
-
# from the data source, parsed and returned as an Array of fields (if header
|
1541
|
-
# rows are not used) or a FasterCSV::Row (when header rows are used).
|
1542
|
-
#
|
1543
|
-
# The data source must be open for reading.
|
1544
|
-
#
|
1545
|
-
def shift
|
1546
|
-
#########################################################################
|
1547
|
-
### This method is purposefully kept a bit long as simple conditional ###
|
1548
|
-
### checks are faster than numerous (expensive) method calls. ###
|
1549
|
-
#########################################################################
|
1550
|
-
|
1551
|
-
# handle headers not based on document content
|
1552
|
-
if header_row? and @return_headers and
|
1553
|
-
[Array, String].include? @use_headers.class
|
1554
|
-
if @unconverted_fields
|
1555
|
-
return add_unconverted_fields(parse_headers, Array.new)
|
1550
|
+
|
1551
|
+
#
|
1552
|
+
# Slurps the remaining rows and returns an Array of Arrays.
|
1553
|
+
#
|
1554
|
+
# The data source must be open for reading.
|
1555
|
+
#
|
1556
|
+
def read
|
1557
|
+
rows = to_a
|
1558
|
+
if @use_headers
|
1559
|
+
Table.new(rows)
|
1556
1560
|
else
|
1557
|
-
|
1561
|
+
rows
|
1558
1562
|
end
|
1559
1563
|
end
|
1560
|
-
|
1561
|
-
|
1562
|
-
|
1563
|
-
|
1564
|
-
|
1565
|
-
|
1566
|
-
|
1567
|
-
#
|
1568
|
-
|
1569
|
-
|
1570
|
-
|
1571
|
-
|
1572
|
-
|
1573
|
-
|
1574
|
-
|
1575
|
-
|
1576
|
-
|
1577
|
-
|
1578
|
-
|
1579
|
-
|
1580
|
-
|
1581
|
-
|
1582
|
-
|
1583
|
-
|
1584
|
-
|
1585
|
-
return add_unconverted_fields(Array.new, Array.new)
|
1586
|
-
elsif @use_headers
|
1587
|
-
return FasterCSV::Row.new(Array.new, Array.new)
|
1564
|
+
alias_method :readlines, :read
|
1565
|
+
|
1566
|
+
# Returns +true+ if the next row read will be a header row.
|
1567
|
+
def header_row?
|
1568
|
+
@use_headers and @headers.nil?
|
1569
|
+
end
|
1570
|
+
|
1571
|
+
#
|
1572
|
+
# The primary read method for wrapped Strings and IOs, a single row is pulled
|
1573
|
+
# from the data source, parsed and returned as an Array of fields (if header
|
1574
|
+
# rows are not used) or a FasterCSV::Row (when header rows are used).
|
1575
|
+
#
|
1576
|
+
# The data source must be open for reading.
|
1577
|
+
#
|
1578
|
+
def shift
|
1579
|
+
#########################################################################
|
1580
|
+
### This method is purposefully kept a bit long as simple conditional ###
|
1581
|
+
### checks are faster than numerous (expensive) method calls. ###
|
1582
|
+
#########################################################################
|
1583
|
+
|
1584
|
+
# handle headers not based on document content
|
1585
|
+
if header_row? and @return_headers and
|
1586
|
+
[Array, String].include? @use_headers.class
|
1587
|
+
if @unconverted_fields
|
1588
|
+
return add_unconverted_fields(parse_headers, Array.new)
|
1588
1589
|
else
|
1589
|
-
return
|
1590
|
+
return parse_headers
|
1590
1591
|
end
|
1591
1592
|
end
|
1592
1593
|
|
1594
|
+
# begin with a blank line, so we can always add to it
|
1595
|
+
line = String.new
|
1596
|
+
|
1593
1597
|
#
|
1594
|
-
#
|
1595
|
-
#
|
1596
|
-
#
|
1597
|
-
csv = if parse.sub!(@parsers[:leading_fields], "")
|
1598
|
-
[nil] * ($&.length / @col_sep.length)
|
1599
|
-
else
|
1600
|
-
Array.new
|
1601
|
-
end
|
1602
|
-
#
|
1603
|
-
# then parse the main fields with a hyper-tuned Regexp from
|
1604
|
-
# Mastering Regular Expressions, Second Edition
|
1598
|
+
# it can take multiple calls to <tt>@io.gets()</tt> to get a full line,
|
1599
|
+
# because of \r and/or \n characters embedded in quoted fields
|
1605
1600
|
#
|
1606
|
-
|
1607
|
-
|
1608
|
-
|
1609
|
-
|
1601
|
+
loop do
|
1602
|
+
# add another read to the line
|
1603
|
+
if read_line = @io.gets(@row_sep)
|
1604
|
+
line += read_line
|
1605
|
+
else
|
1606
|
+
return nil
|
1607
|
+
end
|
1608
|
+
# copy the line so we can chop it up in parsing
|
1609
|
+
parse = line.dup
|
1610
|
+
parse.sub!(@parsers[:line_end], "")
|
1611
|
+
|
1612
|
+
#
|
1613
|
+
# I believe a blank line should be an <tt>Array.new</tt>, not
|
1614
|
+
# CSV's <tt>[nil]</tt>
|
1615
|
+
#
|
1616
|
+
if parse.empty?
|
1617
|
+
@lineno += 1
|
1618
|
+
if @skip_blanks
|
1619
|
+
line = ""
|
1620
|
+
next
|
1621
|
+
elsif @unconverted_fields
|
1622
|
+
return add_unconverted_fields(Array.new, Array.new)
|
1623
|
+
elsif @use_headers
|
1624
|
+
return FasterCSV::Row.new(Array.new, Array.new)
|
1610
1625
|
else
|
1611
|
-
|
1612
|
-
|
1613
|
-
|
1614
|
-
|
1615
|
-
|
1616
|
-
|
1617
|
-
|
1626
|
+
return Array.new
|
1627
|
+
end
|
1628
|
+
end
|
1629
|
+
|
1630
|
+
# parse the fields with a mix of String#split and regular expressions
|
1631
|
+
csv = Array.new
|
1632
|
+
current_field = String.new
|
1633
|
+
field_quotes = 0
|
1634
|
+
parse.split(@col_sep, -1).each do |match|
|
1635
|
+
if current_field.empty? && match.count(@quote_and_newlines).zero?
|
1636
|
+
csv << (match.empty? ? nil : match)
|
1637
|
+
elsif (current_field.empty? ? match[0] : current_field[0]) ==
|
1638
|
+
@quote_char[0]
|
1639
|
+
current_field << match
|
1640
|
+
field_quotes += match.count(@quote_char)
|
1641
|
+
if field_quotes % 2 == 0
|
1642
|
+
in_quotes = current_field[@parsers[:quoted_field], 1]
|
1643
|
+
if !in_quotes || in_quotes[@parsers[:stray_quote]]
|
1644
|
+
raise MalformedCSVError,
|
1645
|
+
"Missing or stray quote in line #{lineno + 1}"
|
1646
|
+
end
|
1647
|
+
current_field = in_quotes
|
1648
|
+
current_field.gsub!(@quote_char * 2, @quote_char) # unescape contents
|
1649
|
+
csv << current_field
|
1650
|
+
current_field = String.new
|
1651
|
+
field_quotes = 0
|
1652
|
+
else # we found a quoted field that spans multiple lines
|
1653
|
+
current_field << @col_sep
|
1618
1654
|
end
|
1655
|
+
elsif match.count("\r\n").zero?
|
1656
|
+
raise MalformedCSVError, "Illegal quoting in line #{lineno + 1}."
|
1657
|
+
else
|
1658
|
+
raise MalformedCSVError, "Unquoted fields do not allow " +
|
1659
|
+
"\\r or \\n (line #{lineno + 1})."
|
1619
1660
|
end
|
1620
|
-
else # we found a quoted field...
|
1621
|
-
$1.gsub(@quote_char * 2, @quote_char) # unescape contents
|
1622
1661
|
end
|
1623
|
-
"" # gsub!'s replacement, clear the field
|
1624
|
-
end
|
1625
1662
|
|
1626
|
-
|
1627
|
-
|
1628
|
-
|
1663
|
+
# if parse is empty?(), we found all the fields on the line...
|
1664
|
+
if field_quotes % 2 == 0
|
1665
|
+
@lineno += 1
|
1629
1666
|
|
1630
|
-
|
1631
|
-
|
1667
|
+
# save fields unconverted fields, if needed...
|
1668
|
+
unconverted = csv.dup if @unconverted_fields
|
1632
1669
|
|
1633
|
-
|
1634
|
-
|
1635
|
-
|
1636
|
-
|
1670
|
+
# convert fields, if needed...
|
1671
|
+
csv = convert_fields(csv) unless @use_headers or @converters.empty?
|
1672
|
+
# parse out header rows and handle FasterCSV::Row conversions...
|
1673
|
+
csv = parse_headers(csv) if @use_headers
|
1674
|
+
|
1675
|
+
# inject unconverted fields and accessor, if requested...
|
1676
|
+
if @unconverted_fields and not csv.respond_to? :unconverted_fields
|
1677
|
+
add_unconverted_fields(csv, unconverted)
|
1678
|
+
end
|
1637
1679
|
|
1638
|
-
|
1639
|
-
|
1640
|
-
add_unconverted_fields(csv, unconverted)
|
1680
|
+
# return the results
|
1681
|
+
break csv
|
1641
1682
|
end
|
1683
|
+
# if we're not empty?() but at eof?(), a quoted field wasn't closed...
|
1684
|
+
if @io.eof?
|
1685
|
+
raise MalformedCSVError, "Unclosed quoted field on line #{lineno + 1}."
|
1686
|
+
elsif @field_size_limit and current_field.size >= @field_size_limit
|
1687
|
+
raise MalformedCSVError, "Field size exceeded on line #{lineno + 1}."
|
1688
|
+
end
|
1689
|
+
# otherwise, we need to loop and pull some more data to complete the row
|
1690
|
+
end
|
1691
|
+
end
|
1692
|
+
alias_method :gets, :shift
|
1693
|
+
alias_method :readline, :shift
|
1642
1694
|
|
1643
|
-
|
1644
|
-
|
1695
|
+
# Returns a simplified description of the key FasterCSV attributes.
|
1696
|
+
def inspect
|
1697
|
+
str = "<##{self.class} io_type:"
|
1698
|
+
# show type of wrapped IO
|
1699
|
+
if @io == $stdout then str << "$stdout"
|
1700
|
+
elsif @io == $stdin then str << "$stdin"
|
1701
|
+
elsif @io == $stderr then str << "$stderr"
|
1702
|
+
else str << @io.class.to_s
|
1645
1703
|
end
|
1646
|
-
#
|
1647
|
-
if @io.
|
1648
|
-
|
1649
|
-
elsif parse =~ @parsers[:bad_field]
|
1650
|
-
raise MalformedCSVError, "Illegal quoting on line #{lineno + 1}."
|
1651
|
-
elsif @field_size_limit and parse.length >= @field_size_limit
|
1652
|
-
raise MalformedCSVError, "Field size exceeded on line #{lineno + 1}."
|
1704
|
+
# show IO.path(), if available
|
1705
|
+
if @io.respond_to?(:path) and (p = @io.path)
|
1706
|
+
str << " io_path:#{p.inspect}"
|
1653
1707
|
end
|
1654
|
-
#
|
1655
|
-
|
1656
|
-
|
1657
|
-
|
1658
|
-
|
1659
|
-
|
1660
|
-
# Returns a simplified description of the key FasterCSV attributes.
|
1661
|
-
def inspect
|
1662
|
-
str = "<##{self.class} io_type:"
|
1663
|
-
# show type of wrapped IO
|
1664
|
-
if @io == $stdout then str << "$stdout"
|
1665
|
-
elsif @io == $stdin then str << "$stdin"
|
1666
|
-
elsif @io == $stderr then str << "$stderr"
|
1667
|
-
else str << @io.class.to_s
|
1668
|
-
end
|
1669
|
-
# show IO.path(), if available
|
1670
|
-
if @io.respond_to?(:path) and (p = @io.path)
|
1671
|
-
str << " io_path:#{p.inspect}"
|
1672
|
-
end
|
1673
|
-
# show other attributes
|
1674
|
-
%w[ lineno col_sep row_sep
|
1675
|
-
quote_char skip_blanks encoding ].each do |attr_name|
|
1676
|
-
if a = instance_variable_get("@#{attr_name}")
|
1677
|
-
str << " #{attr_name}:#{a.inspect}"
|
1708
|
+
# show other attributes
|
1709
|
+
%w[ lineno col_sep row_sep
|
1710
|
+
quote_char skip_blanks encoding ].each do |attr_name|
|
1711
|
+
if a = instance_variable_get("@#{attr_name}")
|
1712
|
+
str << " #{attr_name}:#{a.inspect}"
|
1713
|
+
end
|
1678
1714
|
end
|
1715
|
+
if @use_headers
|
1716
|
+
str << " headers:#{(@headers || true).inspect}"
|
1717
|
+
end
|
1718
|
+
str << ">"
|
1679
1719
|
end
|
1680
|
-
|
1681
|
-
|
1682
|
-
|
1683
|
-
|
1684
|
-
|
1685
|
-
|
1686
|
-
|
1687
|
-
|
1688
|
-
|
1689
|
-
|
1690
|
-
|
1691
|
-
|
1692
|
-
|
1693
|
-
|
1694
|
-
|
1695
|
-
|
1696
|
-
|
1697
|
-
|
1698
|
-
|
1699
|
-
|
1700
|
-
|
1701
|
-
|
1702
|
-
|
1703
|
-
|
1704
|
-
|
1705
|
-
|
1706
|
-
end
|
1707
|
-
|
1708
|
-
# automatically discover row separator when requested
|
1709
|
-
if @row_sep == :auto
|
1710
|
-
if [ARGF, STDIN, STDOUT, STDERR].include?(@io) or
|
1711
|
-
(defined?(Zlib) and @io.class == Zlib::GzipWriter)
|
1712
|
-
@row_sep = $INPUT_RECORD_SEPARATOR
|
1713
|
-
else
|
1720
|
+
|
1721
|
+
private
|
1722
|
+
|
1723
|
+
#
|
1724
|
+
# Stores the indicated separators for later use.
|
1725
|
+
#
|
1726
|
+
# If auto-discovery was requested for <tt>@row_sep</tt>, this method will read
|
1727
|
+
# ahead in the <tt>@io</tt> and try to find one. +ARGF+, +STDIN+, +STDOUT+,
|
1728
|
+
# +STDERR+ and any stream open for output only with a default
|
1729
|
+
# <tt>@row_sep</tt> of <tt>$INPUT_RECORD_SEPARATOR</tt> (<tt>$/</tt>).
|
1730
|
+
#
|
1731
|
+
# This method also establishes the quoting rules used for CSV output.
|
1732
|
+
#
|
1733
|
+
def init_separators(options)
|
1734
|
+
# store the selected separators
|
1735
|
+
@col_sep = options.delete(:col_sep)
|
1736
|
+
@row_sep = options.delete(:row_sep)
|
1737
|
+
@quote_char = options.delete(:quote_char)
|
1738
|
+
@quote_and_newlines = "\r\n#{@quote_char}"
|
1739
|
+
|
1740
|
+
if @quote_char.length != 1
|
1741
|
+
raise ArgumentError, ":quote_char has to be a single character String"
|
1742
|
+
end
|
1743
|
+
|
1744
|
+
# automatically discover row separator when requested
|
1745
|
+
if @row_sep == :auto
|
1714
1746
|
begin
|
1715
|
-
|
1747
|
+
#
|
1748
|
+
# remember where we were (pos() will raise an axception if @io is pipe
|
1749
|
+
# or not opened for reading)
|
1750
|
+
#
|
1751
|
+
saved_pos = @io.pos
|
1716
1752
|
while @row_sep == :auto
|
1717
|
-
#
|
1718
|
-
# if we run out of data, it's probably a single line
|
1719
|
-
# (
|
1720
|
-
#
|
1721
|
-
if @io.eof?
|
1722
|
-
|
1723
|
-
break
|
1724
|
-
end
|
1725
|
-
|
1753
|
+
#
|
1754
|
+
# if we run out of data, it's probably a single line
|
1755
|
+
# (ensure will set default value)
|
1756
|
+
#
|
1757
|
+
break if @io.eof?
|
1758
|
+
|
1726
1759
|
# read ahead a bit
|
1727
1760
|
sample = @io.read(1024)
|
1728
1761
|
sample += @io.read(1) if sample[-1..-1] == "\r" and not @io.eof?
|
1729
|
-
|
1762
|
+
|
1730
1763
|
# try to find a standard separator
|
1731
1764
|
if sample =~ /\r\n?|\n/
|
1732
1765
|
@row_sep = $&
|
1733
1766
|
break
|
1734
1767
|
end
|
1735
1768
|
end
|
1769
|
+
|
1736
1770
|
# tricky seek() clone to work around GzipReader's lack of seek()
|
1737
1771
|
@io.rewind
|
1738
1772
|
# reset back to the remembered position
|
@@ -1741,231 +1775,225 @@ class FasterCSV
|
|
1741
1775
|
saved_pos -= 1024
|
1742
1776
|
end
|
1743
1777
|
@io.read(saved_pos) if saved_pos.nonzero?
|
1744
|
-
rescue IOError
|
1745
|
-
|
1778
|
+
rescue IOError # not opened for reading
|
1779
|
+
# do nothing: ensure will set default
|
1780
|
+
rescue NoMethodError # Zlib::GzipWriter doesn't have eof?
|
1781
|
+
# do nothing: ensure will set default
|
1782
|
+
rescue SystemCallError # pipe
|
1783
|
+
# do nothing: ensure will set default
|
1784
|
+
ensure
|
1785
|
+
#
|
1786
|
+
# set default if we failed to detect
|
1787
|
+
# (stream not opened for reading, a pipe, or a single line of data)
|
1788
|
+
#
|
1789
|
+
@row_sep = $INPUT_RECORD_SEPARATOR if @row_sep == :auto
|
1746
1790
|
end
|
1747
1791
|
end
|
1748
|
-
|
1749
|
-
|
1750
|
-
|
1751
|
-
|
1752
|
-
|
1753
|
-
|
1754
|
-
|
1755
|
-
|
1756
|
-
|
1757
|
-
|
1758
|
-
|
1759
|
-
|
1760
|
-
|
1761
|
-
""
|
1762
|
-
else
|
1763
|
-
field = String(field) # Stringify fields
|
1764
|
-
# represent empty fields as empty quoted fields
|
1765
|
-
if field.empty? or
|
1766
|
-
field.count("\r\n#{@col_sep}#{@quote_char}").nonzero?
|
1767
|
-
do_quote.call(field)
|
1792
|
+
|
1793
|
+
# establish quoting rules
|
1794
|
+
do_quote = lambda do |field|
|
1795
|
+
@quote_char +
|
1796
|
+
String(field).gsub(@quote_char, @quote_char * 2) +
|
1797
|
+
@quote_char
|
1798
|
+
end
|
1799
|
+
@quote = if options.delete(:force_quotes)
|
1800
|
+
do_quote
|
1801
|
+
else
|
1802
|
+
lambda do |field|
|
1803
|
+
if field.nil? # represent +nil+ fields as empty unquoted fields
|
1804
|
+
""
|
1768
1805
|
else
|
1769
|
-
field #
|
1806
|
+
field = String(field) # Stringify fields
|
1807
|
+
# represent empty fields as empty quoted fields
|
1808
|
+
if field.empty? or
|
1809
|
+
field.count("\r\n#{@col_sep}#{@quote_char}").nonzero?
|
1810
|
+
do_quote.call(field)
|
1811
|
+
else
|
1812
|
+
field # unquoted field
|
1813
|
+
end
|
1770
1814
|
end
|
1771
1815
|
end
|
1772
1816
|
end
|
1773
1817
|
end
|
1774
|
-
|
1775
|
-
|
1776
|
-
|
1777
|
-
|
1778
|
-
|
1779
|
-
|
1780
|
-
|
1781
|
-
|
1782
|
-
|
1783
|
-
|
1784
|
-
|
1785
|
-
|
1786
|
-
|
1787
|
-
|
1788
|
-
|
1789
|
-
|
1790
|
-
|
1791
|
-
|
1792
|
-
|
1793
|
-
|
1794
|
-
|
1795
|
-
|
1796
|
-
|
1797
|
-
|
1798
|
-
|
1799
|
-
(?=#{esc_col_sep}|\\z) # ensure we are at field's end
|
1800
|
-
END_PARSER
|
1801
|
-
# a test for unescaped quotes
|
1802
|
-
:bad_field => Regexp.new(<<-END_BAD, Regexp::EXTENDED, @encoding),
|
1803
|
-
\\A#{esc_col_sep}? # starts with an optional comma
|
1804
|
-
(?: #{esc_quote} (?>[^#{esc_quote}]*) # an extra quote
|
1805
|
-
(?> #{esc_quote*2}
|
1806
|
-
[^#{esc_quote}]* )*
|
1807
|
-
#{esc_quote}[^#{esc_quote}]
|
1808
|
-
| # ... or ...
|
1809
|
-
[^#{esc_quote}#{esc_col_sep}]+
|
1810
|
-
#{esc_quote} # unescaped quote
|
1811
|
-
)
|
1812
|
-
END_BAD
|
1813
|
-
# safer than chomp!()
|
1814
|
-
:line_end => Regexp.new("#{esc_row_sep}\\z", nil, @encoding)
|
1815
|
-
}
|
1816
|
-
end
|
1817
|
-
|
1818
|
-
#
|
1819
|
-
# Loads any converters requested during construction.
|
1820
|
-
#
|
1821
|
-
# If +field_name+ is set <tt>:converters</tt> (the default) field converters
|
1822
|
-
# are set. When +field_name+ is <tt>:header_converters</tt> header converters
|
1823
|
-
# are added instead.
|
1824
|
-
#
|
1825
|
-
# The <tt>:unconverted_fields</tt> option is also actived for
|
1826
|
-
# <tt>:converters</tt> calls, if requested.
|
1827
|
-
#
|
1828
|
-
def init_converters(options, field_name = :converters)
|
1829
|
-
if field_name == :converters
|
1830
|
-
@unconverted_fields = options.delete(:unconverted_fields)
|
1818
|
+
|
1819
|
+
# Pre-compiles parsers and stores them by name for access during reads.
|
1820
|
+
def init_parsers(options)
|
1821
|
+
# store the parser behaviors
|
1822
|
+
@skip_blanks = options.delete(:skip_blanks)
|
1823
|
+
@encoding = options.delete(:encoding) # nil will use $KCODE
|
1824
|
+
@field_size_limit = options.delete(:field_size_limit)
|
1825
|
+
|
1826
|
+
# prebuild Regexps for faster parsing
|
1827
|
+
esc_col_sep = Regexp.escape(@col_sep)
|
1828
|
+
esc_row_sep = Regexp.escape(@row_sep)
|
1829
|
+
esc_quote = Regexp.escape(@quote_char)
|
1830
|
+
@parsers = {
|
1831
|
+
:any_field => Regexp.new( "[^#{esc_col_sep}]+",
|
1832
|
+
Regexp::MULTILINE,
|
1833
|
+
@encoding ),
|
1834
|
+
:quoted_field => Regexp.new( "^#{esc_quote}(.*)#{esc_quote}$",
|
1835
|
+
Regexp::MULTILINE,
|
1836
|
+
@encoding ),
|
1837
|
+
:stray_quote => Regexp.new( "[^#{esc_quote}]#{esc_quote}[^#{esc_quote}]",
|
1838
|
+
Regexp::MULTILINE,
|
1839
|
+
@encoding ),
|
1840
|
+
# safer than chomp!()
|
1841
|
+
:line_end => Regexp.new("#{esc_row_sep}\\z", nil, @encoding)
|
1842
|
+
}
|
1831
1843
|
end
|
1832
1844
|
|
1833
|
-
|
1834
|
-
|
1835
|
-
#
|
1836
|
-
|
1837
|
-
|
1838
|
-
#
|
1839
|
-
|
1840
|
-
|
1841
|
-
|
1842
|
-
|
1843
|
-
|
1844
|
-
|
1845
|
-
|
1846
|
-
|
1847
|
-
|
1848
|
-
|
1849
|
-
|
1845
|
+
#
|
1846
|
+
# Loads any converters requested during construction.
|
1847
|
+
#
|
1848
|
+
# If +field_name+ is set <tt>:converters</tt> (the default) field converters
|
1849
|
+
# are set. When +field_name+ is <tt>:header_converters</tt> header converters
|
1850
|
+
# are added instead.
|
1851
|
+
#
|
1852
|
+
# The <tt>:unconverted_fields</tt> option is also actived for
|
1853
|
+
# <tt>:converters</tt> calls, if requested.
|
1854
|
+
#
|
1855
|
+
def init_converters(options, field_name = :converters)
|
1856
|
+
if field_name == :converters
|
1857
|
+
@unconverted_fields = options.delete(:unconverted_fields)
|
1858
|
+
end
|
1859
|
+
|
1860
|
+
instance_variable_set("@#{field_name}", Array.new)
|
1861
|
+
|
1862
|
+
# find the correct method to add the coverters
|
1863
|
+
convert = method(field_name.to_s.sub(/ers\Z/, ""))
|
1864
|
+
|
1865
|
+
# load converters
|
1866
|
+
unless options[field_name].nil?
|
1867
|
+
# allow a single converter not wrapped in an Array
|
1868
|
+
unless options[field_name].is_a? Array
|
1869
|
+
options[field_name] = [options[field_name]]
|
1870
|
+
end
|
1871
|
+
# load each converter...
|
1872
|
+
options[field_name].each do |converter|
|
1873
|
+
if converter.is_a? Proc # custom code block
|
1874
|
+
convert.call(&converter)
|
1875
|
+
else # by name
|
1876
|
+
convert.call(converter)
|
1877
|
+
end
|
1850
1878
|
end
|
1851
1879
|
end
|
1880
|
+
|
1881
|
+
options.delete(field_name)
|
1852
1882
|
end
|
1853
|
-
|
1854
|
-
|
1855
|
-
|
1856
|
-
|
1857
|
-
|
1858
|
-
|
1859
|
-
|
1860
|
-
|
1861
|
-
|
1862
|
-
|
1863
|
-
|
1864
|
-
|
1865
|
-
|
1866
|
-
|
1867
|
-
|
1868
|
-
|
1869
|
-
|
1870
|
-
|
1871
|
-
|
1872
|
-
|
1873
|
-
|
1874
|
-
|
1875
|
-
|
1876
|
-
#
|
1877
|
-
|
1878
|
-
|
1879
|
-
|
1880
|
-
|
1881
|
-
|
1882
|
-
|
1883
|
-
|
1884
|
-
|
1885
|
-
|
1886
|
-
|
1883
|
+
|
1884
|
+
# Stores header row settings and loads header converters, if needed.
|
1885
|
+
def init_headers(options)
|
1886
|
+
@use_headers = options.delete(:headers)
|
1887
|
+
@return_headers = options.delete(:return_headers)
|
1888
|
+
@write_headers = options.delete(:write_headers)
|
1889
|
+
|
1890
|
+
# headers must be delayed until shift(), in case they need a row of content
|
1891
|
+
@headers = nil
|
1892
|
+
|
1893
|
+
init_converters(options, :header_converters)
|
1894
|
+
end
|
1895
|
+
|
1896
|
+
#
|
1897
|
+
# The actual work method for adding converters, used by both
|
1898
|
+
# FasterCSV.convert() and FasterCSV.header_convert().
|
1899
|
+
#
|
1900
|
+
# This method requires the +var_name+ of the instance variable to place the
|
1901
|
+
# converters in, the +const+ Hash to lookup named converters in, and the
|
1902
|
+
# normal parameters of the FasterCSV.convert() and FasterCSV.header_convert()
|
1903
|
+
# methods.
|
1904
|
+
#
|
1905
|
+
def add_converter(var_name, const, name = nil, &converter)
|
1906
|
+
if name.nil? # custom converter
|
1907
|
+
instance_variable_get("@#{var_name}") << converter
|
1908
|
+
else # named converter
|
1909
|
+
combo = const[name]
|
1910
|
+
case combo
|
1911
|
+
when Array # combo converter
|
1912
|
+
combo.each do |converter_name|
|
1913
|
+
add_converter(var_name, const, converter_name)
|
1914
|
+
end
|
1915
|
+
else # individual named converter
|
1916
|
+
instance_variable_get("@#{var_name}") << combo
|
1887
1917
|
end
|
1888
|
-
else # individual named converter
|
1889
|
-
instance_variable_get("@#{var_name}") << combo
|
1890
1918
|
end
|
1891
1919
|
end
|
1892
|
-
|
1893
|
-
|
1894
|
-
|
1895
|
-
|
1896
|
-
|
1897
|
-
|
1898
|
-
|
1899
|
-
|
1900
|
-
|
1901
|
-
|
1902
|
-
|
1903
|
-
|
1904
|
-
|
1905
|
-
|
1906
|
-
|
1907
|
-
|
1908
|
-
converter
|
1909
|
-
|
1910
|
-
|
1911
|
-
|
1920
|
+
|
1921
|
+
#
|
1922
|
+
# Processes +fields+ with <tt>@converters</tt>, or <tt>@header_converters</tt>
|
1923
|
+
# if +headers+ is passed as +true+, returning the converted field set. Any
|
1924
|
+
# converter that changes the field into something other than a String halts
|
1925
|
+
# the pipeline of conversion for that field. This is primarily an efficiency
|
1926
|
+
# shortcut.
|
1927
|
+
#
|
1928
|
+
def convert_fields(fields, headers = false)
|
1929
|
+
# see if we are converting headers or fields
|
1930
|
+
converters = headers ? @header_converters : @converters
|
1931
|
+
|
1932
|
+
fields.enum_for(:each_with_index).map do |field, index| # map_with_index
|
1933
|
+
converters.each do |converter|
|
1934
|
+
field = if converter.arity == 1 # straight field converter
|
1935
|
+
converter[field]
|
1936
|
+
else # FieldInfo converter
|
1937
|
+
header = @use_headers && !headers ? @headers[index] : nil
|
1938
|
+
converter[field, FieldInfo.new(index, lineno, header)]
|
1939
|
+
end
|
1940
|
+
break unless field.is_a? String # short-curcuit pipeline for speed
|
1912
1941
|
end
|
1913
|
-
|
1942
|
+
field # return final state of each field, converted or original
|
1914
1943
|
end
|
1915
|
-
field # return final state of each field, converted or original
|
1916
1944
|
end
|
1917
|
-
|
1918
|
-
|
1919
|
-
|
1920
|
-
|
1921
|
-
|
1922
|
-
|
1923
|
-
|
1924
|
-
|
1925
|
-
|
1926
|
-
|
1927
|
-
|
1928
|
-
|
1929
|
-
|
1930
|
-
|
1931
|
-
|
1932
|
-
|
1933
|
-
|
1934
|
-
|
1935
|
-
|
1936
|
-
|
1937
|
-
|
1938
|
-
|
1939
|
-
|
1940
|
-
|
1941
|
-
|
1942
|
-
|
1943
|
-
|
1944
|
-
|
1945
|
-
|
1946
|
-
|
1947
|
-
|
1948
|
-
|
1949
|
-
|
1950
|
-
|
1951
|
-
|
1945
|
+
|
1946
|
+
#
|
1947
|
+
# This methods is used to turn a finished +row+ into a FasterCSV::Row. Header
|
1948
|
+
# rows are also dealt with here, either by returning a FasterCSV::Row with
|
1949
|
+
# identical headers and fields (save that the fields do not go through the
|
1950
|
+
# converters) or by reading past them to return a field row. Headers are also
|
1951
|
+
# saved in <tt>@headers</tt> for use in future rows.
|
1952
|
+
#
|
1953
|
+
# When +nil+, +row+ is assumed to be a header row not based on an actual row
|
1954
|
+
# of the stream.
|
1955
|
+
#
|
1956
|
+
def parse_headers(row = nil)
|
1957
|
+
if @headers.nil? # header row
|
1958
|
+
@headers = case @use_headers # save headers
|
1959
|
+
# Array of headers
|
1960
|
+
when Array then @use_headers
|
1961
|
+
# CSV header String
|
1962
|
+
when String
|
1963
|
+
self.class.parse_line( @use_headers,
|
1964
|
+
:col_sep => @col_sep,
|
1965
|
+
:row_sep => @row_sep,
|
1966
|
+
:quote_char => @quote_char )
|
1967
|
+
# first row is headers
|
1968
|
+
else row
|
1969
|
+
end
|
1970
|
+
|
1971
|
+
# prepare converted and unconverted copies
|
1972
|
+
row = @headers if row.nil?
|
1973
|
+
@headers = convert_fields(@headers, true)
|
1974
|
+
|
1975
|
+
if @return_headers # return headers
|
1976
|
+
return FasterCSV::Row.new(@headers, row, true)
|
1977
|
+
elsif not [Array, String].include? @use_headers.class # skip to field row
|
1978
|
+
return shift
|
1979
|
+
end
|
1952
1980
|
end
|
1981
|
+
|
1982
|
+
FasterCSV::Row.new(@headers, convert_fields(row)) # field row
|
1953
1983
|
end
|
1954
1984
|
|
1955
|
-
|
1956
|
-
|
1957
|
-
|
1958
|
-
|
1959
|
-
|
1960
|
-
|
1961
|
-
|
1962
|
-
|
1963
|
-
|
1964
|
-
|
1965
|
-
|
1985
|
+
#
|
1986
|
+
# Thiw methods injects an instance variable <tt>unconverted_fields</tt> into
|
1987
|
+
# +row+ and an accessor method for it called unconverted_fields(). The
|
1988
|
+
# variable is set to the contents of +fields+.
|
1989
|
+
#
|
1990
|
+
def add_unconverted_fields(row, fields)
|
1991
|
+
class << row
|
1992
|
+
attr_reader :unconverted_fields
|
1993
|
+
end
|
1994
|
+
row.instance_eval { @unconverted_fields = fields }
|
1995
|
+
row
|
1966
1996
|
end
|
1967
|
-
row.instance_eval { @unconverted_fields = fields }
|
1968
|
-
row
|
1969
1997
|
end
|
1970
1998
|
end
|
1971
1999
|
|