fastcsv 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,317 @@
1
+ #!/usr/bin/env ruby -w
2
+ # encoding: UTF-8
3
+
4
+ # tc_features.rb
5
+ #
6
+ # Created by James Edward Gray II on 2005-10-31.
7
+ # Copyright 2005 James Edward Gray II. You can redistribute or modify this code
8
+ # under the terms of Ruby's license.
9
+
10
+ begin
11
+ require "zlib"
12
+ rescue LoadError
13
+ end
14
+
15
+ require_relative "base"
16
+ require "tempfile"
17
+
18
+ class TestCSV::Features < TestCSV
19
+ extend DifferentOFS
20
+
21
+ TEST_CASES = [ [%Q{a,b}, ["a", "b"]],
22
+ [%Q{a,"""b"""}, ["a", "\"b\""]],
23
+ [%Q{a,"""b"}, ["a", "\"b"]],
24
+ [%Q{a,"b"""}, ["a", "b\""]],
25
+ [%Q{a,"\nb"""}, ["a", "\nb\""]],
26
+ [%Q{a,"""\nb"}, ["a", "\"\nb"]],
27
+ [%Q{a,"""\nb\n"""}, ["a", "\"\nb\n\""]],
28
+ [%Q{a,"""\nb\n""",\nc}, ["a", "\"\nb\n\"", nil]],
29
+ [%Q{a,,,}, ["a", nil, nil, nil]],
30
+ [%Q{,}, [nil, nil]],
31
+ [%Q{"",""}, ["", ""]],
32
+ [%Q{""""}, ["\""]],
33
+ [%Q{"""",""}, ["\"",""]],
34
+ [%Q{,""}, [nil,""]],
35
+ [%Q{,"\r"}, [nil,"\r"]],
36
+ [%Q{"\r\n,"}, ["\r\n,"]],
37
+ [%Q{"\r\n,",}, ["\r\n,", nil]] ]
38
+
39
+ def setup
40
+ super
41
+ @sample_data = <<-END_DATA.gsub(/^ +/, "")
42
+ line,1,abc
43
+ line,2,"def\nghi"
44
+
45
+ line,4,jkl
46
+ END_DATA
47
+ @csv = FastCSV.new(@sample_data)
48
+ end
49
+
50
+ # def test_col_sep
51
+ # [";", "\t"].each do |sep|
52
+ # TEST_CASES.each do |test_case|
53
+ # assert_equal( test_case.last.map { |t| t.tr(",", sep) unless t.nil? },
54
+ # FastCSV.parse_line( test_case.first.tr(",", sep),
55
+ # col_sep: sep ) )
56
+ # end
57
+ # end
58
+ # assert_equal([",,,", nil], FastCSV.parse_line(",,,;", col_sep: ";"))
59
+ # end
60
+
61
+ # def test_row_sep
62
+ # assert_raise(FastCSV::MalformedCSVError) do
63
+ # FastCSV.parse_line("1,2,3\n,4,5\r\n", row_sep: "\r\n")
64
+ # end
65
+ # assert_equal( ["1", "2", "3\n", "4", "5"],
66
+ # FastCSV.parse_line(%Q{1,2,"3\n",4,5\r\n}, row_sep: "\r\n"))
67
+ # end
68
+
69
+ # def test_quote_char
70
+ # TEST_CASES.each do |test_case|
71
+ # assert_equal( test_case.last.map { |t| t.tr('"', "'") unless t.nil? },
72
+ # FastCSV.parse_line( test_case.first.tr('"', "'"),
73
+ # quote_char: "'" ) )
74
+ # end
75
+ # end
76
+
77
+ def test_csv_char_readers
78
+ %w[col_sep row_sep quote_char].each do |reader|
79
+ csv = FastCSV.new("abc,def", reader.to_sym => "|")
80
+ assert_equal("|", csv.send(reader))
81
+ end
82
+ end
83
+
84
+ def test_row_sep_auto_discovery
85
+ ["\r\n", "\n", "\r"].each do |line_end|
86
+ data = "1,2,3#{line_end}4,5#{line_end}"
87
+ discovered = FastCSV.new(data).row_sep
88
+ assert_equal(line_end, discovered)
89
+ end
90
+
91
+ assert_equal("\n", FastCSV.new("\n\r\n\r").row_sep)
92
+
93
+ assert_equal($/, FastCSV.new("").row_sep)
94
+
95
+ assert_equal($/, FastCSV.new(STDERR).row_sep)
96
+ end
97
+
98
+ def test_lineno
99
+ assert_equal(5, @sample_data.lines.to_a.size)
100
+
101
+ 4.times do |line_count|
102
+ assert_equal(line_count, @csv.lineno)
103
+ assert_not_nil(@csv.shift)
104
+ assert_equal(line_count + 1, @csv.lineno)
105
+ end
106
+ assert_nil(@csv.shift)
107
+ end
108
+
109
+ def test_readline
110
+ test_lineno
111
+
112
+ @csv.rewind
113
+
114
+ test_lineno
115
+ end
116
+
117
+ def test_unknown_options
118
+ assert_raise(ArgumentError) { FastCSV.new(String.new, unknown: :error) }
119
+ end
120
+
121
+ def test_skip_blanks
122
+ assert_equal(4, @csv.to_a.size)
123
+
124
+ @csv = FastCSV.new(@sample_data, skip_blanks: true)
125
+
126
+ count = 0
127
+ @csv.each do |row|
128
+ count += 1
129
+ assert_equal("line", row.first)
130
+ end
131
+ assert_equal(3, count)
132
+ end
133
+
134
+ def test_csv_behavior_readers
135
+ %w[ unconverted_fields return_headers write_headers
136
+ skip_blanks force_quotes ].each do |behavior|
137
+ assert( !FastCSV.new("abc,def").send("#{behavior}?"),
138
+ "Behavior defaulted to on." )
139
+ csv = FastCSV.new("abc,def", behavior.to_sym => true)
140
+ assert(csv.send("#{behavior}?"), "Behavior change now registered.")
141
+ end
142
+ end
143
+
144
+ def test_converters_reader
145
+ # no change
146
+ assert_equal( [:integer],
147
+ FastCSV.new("abc,def", converters: [:integer]).converters )
148
+
149
+ # just one
150
+ assert_equal( [:integer],
151
+ FastCSV.new("abc,def", converters: :integer).converters )
152
+
153
+ # expanded
154
+ assert_equal( [:integer, :float],
155
+ FastCSV.new("abc,def", converters: :numeric).converters )
156
+
157
+ # custom
158
+ csv = FastCSV.new("abc,def", converters: [:integer, lambda { }])
159
+ assert_equal(2, csv.converters.size)
160
+ assert_equal(:integer, csv.converters.first)
161
+ assert_instance_of(Proc, csv.converters.last)
162
+ end
163
+
164
+ def test_header_converters_reader
165
+ # no change
166
+ hc = :header_converters
167
+ assert_equal([:downcase], FastCSV.new("abc,def", hc => [:downcase]).send(hc))
168
+
169
+ # just one
170
+ assert_equal([:downcase], FastCSV.new("abc,def", hc => :downcase).send(hc))
171
+
172
+ # custom
173
+ csv = FastCSV.new("abc,def", hc => [:symbol, lambda { }])
174
+ assert_equal(2, csv.send(hc).size)
175
+ assert_equal(:symbol, csv.send(hc).first)
176
+ assert_instance_of(Proc, csv.send(hc).last)
177
+ end
178
+
179
+ # reported by Kev Jackson
180
+ def test_failing_to_escape_col_sep_bug_fix
181
+ assert_nothing_raised(Exception) { FastCSV.new(String.new, col_sep: "|") }
182
+ end
183
+
184
+ # reported by Chris Roos
185
+ def test_failing_to_reset_headers_in_rewind_bug_fix
186
+ csv = FastCSV.new("forename,surname", headers: true, return_headers: true)
187
+ csv.each { |row| assert row.header_row? }
188
+ csv.rewind
189
+ csv.each { |row| assert row.header_row? }
190
+ end
191
+
192
+ # reported by Dave Burt
193
+ # def test_leading_empty_fields_with_multibyte_col_sep_bug_fix
194
+ # data = <<-END_DATA.gsub(/^\s+/, "")
195
+ # <=><=>A<=>B<=>C
196
+ # 1<=>2<=>3
197
+ # END_DATA
198
+ # parsed = FastCSV.parse(data, col_sep: "<=>")
199
+ # assert_equal([[nil, nil, "A", "B", "C"], ["1", "2", "3"]], parsed)
200
+ # end
201
+
202
+ def test_gzip_reader_bug_fix
203
+ zipped = nil
204
+ assert_nothing_raised(NoMethodError) do
205
+ zipped = FastCSV.new(
206
+ Zlib::GzipReader.open(
207
+ File.join(File.dirname(__FILE__), "line_endings.gz")
208
+ )
209
+ )
210
+ end
211
+ assert_equal("\r\n", zipped.row_sep)
212
+ end if defined?(Zlib::GzipReader)
213
+
214
+ def test_gzip_writer_bug_fix
215
+ Tempfile.create(%w"temp .gz") {|tempfile|
216
+ tempfile.close
217
+ file = tempfile.path
218
+ zipped = nil
219
+ assert_nothing_raised(NoMethodError) do
220
+ zipped = FastCSV.new(Zlib::GzipWriter.open(file))
221
+ end
222
+ zipped << %w[one two three]
223
+ zipped << [1, 2, 3]
224
+ zipped.close
225
+
226
+ assert( Zlib::GzipReader.open(file) { |f| f.read }.
227
+ include?($INPUT_RECORD_SEPARATOR),
228
+ "@row_sep did not default" )
229
+ }
230
+ end if defined?(Zlib::GzipWriter)
231
+
232
+ def test_inspect_is_smart_about_io_types
233
+ str = FastCSV.new("string,data").inspect
234
+ assert(str.include?("io_type:StringIO"), "IO type not detected.")
235
+
236
+ str = FastCSV.new($stderr).inspect
237
+ assert(str.include?("io_type:$stderr"), "IO type not detected.")
238
+
239
+ Tempfile.create(%w"temp .csv") {|tempfile|
240
+ tempfile.close
241
+ path = tempfile.path
242
+ File.open(path, "w") { |csv| csv << "one,two,three\n1,2,3\n" }
243
+ str = FastCSV.open(path) { |csv| csv.inspect }
244
+ assert(str.include?("io_type:File"), "IO type not detected.")
245
+ }
246
+ end
247
+
248
+ def test_inspect_shows_key_attributes
249
+ str = @csv.inspect
250
+ %w[lineno col_sep row_sep quote_char].each do |attr_name|
251
+ assert_match(/\b#{attr_name}:[^\s>]+/, str)
252
+ end
253
+ end
254
+
255
+ def test_inspect_shows_headers_when_available
256
+ FastCSV.new("one,two,three\n1,2,3\n", headers: true) do |csv|
257
+ assert(csv.inspect.include?("headers:true"), "Header hint not shown.")
258
+ csv.shift # load headers
259
+ assert_match(/headers:\[[^\]]+\]/, csv.inspect)
260
+ end
261
+ end
262
+
263
+ def test_inspect_encoding_is_ascii_compatible
264
+ FastCSV.new("one,two,three\n1,2,3\n".encode("UTF-16BE")) do |csv|
265
+ assert( Encoding.compatible?( Encoding.find("US-ASCII"),
266
+ csv.inspect.encoding ),
267
+ "inspect() was not ASCII compatible." )
268
+ end
269
+ end
270
+
271
+ def test_version
272
+ assert_not_nil(FastCSV::VERSION)
273
+ assert_instance_of(String, FastCSV::VERSION)
274
+ assert(FastCSV::VERSION.frozen?)
275
+ assert_match(/\A\d\.\d\.\d\Z/, FastCSV::VERSION)
276
+ end
277
+
278
+ def test_accepts_comment_skip_lines_option
279
+ assert_nothing_raised(ArgumentError) do
280
+ FastCSV.new nil, :skip_lines => /\A\s*#/
281
+ end
282
+ end
283
+
284
+ def test_accepts_comment_defaults_to_nil
285
+ c = FastCSV.new nil
286
+ assert_equal c.skip_lines, nil
287
+ end
288
+
289
+ class RegexStub
290
+ end
291
+
292
+ def test_requires_skip_lines_to_call_match
293
+ regex_stub = RegexStub.new
294
+ assert_raise(ArgumentError) do
295
+ FastCSV.new nil, :skip_lines => regex_stub
296
+ end
297
+ end
298
+
299
+ def test_comment_rows_are_ignored
300
+ sample_data = "line,1,a\n#not,a,line\nline,2,b\n #also,no,line"
301
+ c = FastCSV.new sample_data, :skip_lines => /\A\s*#/
302
+ assert_equal [["line", "1", "a"], ["line", "2", "b"]], c.each.to_a
303
+ end
304
+
305
+ def test_quoted_skip_line_markers_are_ignored
306
+ sample_data = "line,1,a\n\"#not\",a,line\nline,2,b"
307
+ c = FastCSV.new sample_data, :skip_lines => /\A\s*#/
308
+ assert_equal [["line", "1", "a"], ["#not", "a", "line"], ["line", "2", "b"]], c.each.to_a
309
+ end
310
+
311
+ def test_string_works_like_a_regexp
312
+ sample_data = "line,1,a\n#(not,a,line\nline,2,b\n also,#no,line"
313
+ c = FastCSV.new sample_data, :skip_lines => "#"
314
+ assert_equal [["line", "1", "a"], ["line", "2", "b"]], c.each.to_a
315
+ end
316
+
317
+ end
@@ -0,0 +1,289 @@
1
+ #!/usr/bin/env ruby -w
2
+ # encoding: UTF-8
3
+
4
+ # tc_headers.rb
5
+ #
6
+ # Created by James Edward Gray II on 2005-10-31.
7
+ # Copyright 2005 James Edward Gray II. You can redistribute or modify this code
8
+ # under the terms of Ruby's license.
9
+
10
+ require_relative "base"
11
+
12
+ class TestCSV::Headers < TestCSV
13
+ extend DifferentOFS
14
+
15
+ def setup
16
+ super
17
+ @data = <<-END_CSV.gsub(/^\s+/, "")
18
+ first,second,third
19
+ A,B,C
20
+ 1,2,3
21
+ END_CSV
22
+ end
23
+
24
+ def test_first_row
25
+ [:first_row, true].each do |setting| # two names for the same setting
26
+ # activate headers
27
+ csv = nil
28
+ assert_nothing_raised(Exception) do
29
+ csv = FastCSV.parse(@data, headers: setting)
30
+ end
31
+
32
+ # first data row - skipping headers
33
+ row = csv[0]
34
+ assert_not_nil(row)
35
+ assert_instance_of(FastCSV::Row, row)
36
+ assert_equal([%w{first A}, %w{second B}, %w{third C}], row.to_a)
37
+
38
+ # second data row
39
+ row = csv[1]
40
+ assert_not_nil(row)
41
+ assert_instance_of(FastCSV::Row, row)
42
+ assert_equal([%w{first 1}, %w{second 2}, %w{third 3}], row.to_a)
43
+
44
+ # empty
45
+ assert_nil(csv[2])
46
+ end
47
+ end
48
+
49
+ def test_array_of_headers
50
+ # activate headers
51
+ csv = nil
52
+ assert_nothing_raised(Exception) do
53
+ csv = FastCSV.parse(@data, headers: [:my, :new, :headers])
54
+ end
55
+
56
+ # first data row - skipping headers
57
+ row = csv[0]
58
+ assert_not_nil(row)
59
+ assert_instance_of(FastCSV::Row, row)
60
+ assert_equal( [[:my, "first"], [:new, "second"], [:headers, "third"]],
61
+ row.to_a )
62
+
63
+ # second data row
64
+ row = csv[1]
65
+ assert_not_nil(row)
66
+ assert_instance_of(FastCSV::Row, row)
67
+ assert_equal([[:my, "A"], [:new, "B"], [:headers, "C"]], row.to_a)
68
+
69
+ # third data row
70
+ row = csv[2]
71
+ assert_not_nil(row)
72
+ assert_instance_of(FastCSV::Row, row)
73
+ assert_equal([[:my, "1"], [:new, "2"], [:headers, "3"]], row.to_a)
74
+
75
+ # empty
76
+ assert_nil(csv[3])
77
+
78
+ # with return and convert
79
+ assert_nothing_raised(Exception) do
80
+ csv = FastCSV.parse( @data, headers: [:my, :new, :headers],
81
+ return_headers: true,
82
+ header_converters: lambda { |h| h.to_s } )
83
+ end
84
+ row = csv[0]
85
+ assert_not_nil(row)
86
+ assert_instance_of(FastCSV::Row, row)
87
+ assert_equal([["my", :my], ["new", :new], ["headers", :headers]], row.to_a)
88
+ assert(row.header_row?)
89
+ assert(!row.field_row?)
90
+ end
91
+
92
+ def test_csv_header_string
93
+ # activate headers
94
+ csv = nil
95
+ assert_nothing_raised(Exception) do
96
+ csv = FastCSV.parse(@data, headers: "my,new,headers")
97
+ end
98
+
99
+ # first data row - skipping headers
100
+ row = csv[0]
101
+ assert_not_nil(row)
102
+ assert_instance_of(FastCSV::Row, row)
103
+ assert_equal([%w{my first}, %w{new second}, %w{headers third}], row.to_a)
104
+
105
+ # second data row
106
+ row = csv[1]
107
+ assert_not_nil(row)
108
+ assert_instance_of(FastCSV::Row, row)
109
+ assert_equal([%w{my A}, %w{new B}, %w{headers C}], row.to_a)
110
+
111
+ # third data row
112
+ row = csv[2]
113
+ assert_not_nil(row)
114
+ assert_instance_of(FastCSV::Row, row)
115
+ assert_equal([%w{my 1}, %w{new 2}, %w{headers 3}], row.to_a)
116
+
117
+ # empty
118
+ assert_nil(csv[3])
119
+
120
+ # with return and convert
121
+ assert_nothing_raised(Exception) do
122
+ csv = FastCSV.parse( @data, headers: "my,new,headers",
123
+ return_headers: true,
124
+ header_converters: :symbol )
125
+ end
126
+ row = csv[0]
127
+ assert_not_nil(row)
128
+ assert_instance_of(FastCSV::Row, row)
129
+ assert_equal([[:my, "my"], [:new, "new"], [:headers, "headers"]], row.to_a)
130
+ assert(row.header_row?)
131
+ assert(!row.field_row?)
132
+ end
133
+
134
+ # def test_csv_header_string_inherits_separators
135
+ # # parse with custom col_sep
136
+ # csv = nil
137
+ # assert_nothing_raised(Exception) do
138
+ # csv = FastCSV.parse( @data.tr(",", "|"), col_sep: "|",
139
+ # headers: "my|new|headers" )
140
+ # end
141
+
142
+ # # verify headers were recognized
143
+ # row = csv[0]
144
+ # assert_not_nil(row)
145
+ # assert_instance_of(FastCSV::Row, row)
146
+ # assert_equal([%w{my first}, %w{new second}, %w{headers third}], row.to_a)
147
+ # end
148
+
149
+ def test_return_headers
150
+ # activate headers and request they are returned
151
+ csv = nil
152
+ assert_nothing_raised(Exception) do
153
+ csv = FastCSV.parse(@data, headers: true, return_headers: true)
154
+ end
155
+
156
+ # header row
157
+ row = csv[0]
158
+ assert_not_nil(row)
159
+ assert_instance_of(FastCSV::Row, row)
160
+ assert_equal( [%w{first first}, %w{second second}, %w{third third}],
161
+ row.to_a )
162
+ assert(row.header_row?)
163
+ assert(!row.field_row?)
164
+
165
+ # first data row - skipping headers
166
+ row = csv[1]
167
+ assert_not_nil(row)
168
+ assert_instance_of(FastCSV::Row, row)
169
+ assert_equal([%w{first A}, %w{second B}, %w{third C}], row.to_a)
170
+ assert(!row.header_row?)
171
+ assert(row.field_row?)
172
+
173
+ # second data row
174
+ row = csv[2]
175
+ assert_not_nil(row)
176
+ assert_instance_of(FastCSV::Row, row)
177
+ assert_equal([%w{first 1}, %w{second 2}, %w{third 3}], row.to_a)
178
+ assert(!row.header_row?)
179
+ assert(row.field_row?)
180
+
181
+ # empty
182
+ assert_nil(csv[3])
183
+ end
184
+
185
+ def test_converters
186
+ # create test data where headers and fields look alike
187
+ data = <<-END_MATCHING_CSV.gsub(/^\s+/, "")
188
+ 1,2,3
189
+ 1,2,3
190
+ END_MATCHING_CSV
191
+
192
+ # normal converters do not affect headers
193
+ csv = FastCSV.parse( data, headers: true,
194
+ return_headers: true,
195
+ converters: :numeric )
196
+ assert_equal([%w{1 1}, %w{2 2}, %w{3 3}], csv[0].to_a)
197
+ assert_equal([["1", 1], ["2", 2], ["3", 3]], csv[1].to_a)
198
+ assert_nil(csv[2])
199
+
200
+ # header converters do affect headers (only)
201
+ assert_nothing_raised(Exception) do
202
+ csv = FastCSV.parse( data, headers: true,
203
+ return_headers: true,
204
+ converters: :numeric,
205
+ header_converters: :symbol )
206
+ end
207
+ assert_equal([[:"1", "1"], [:"2", "2"], [:"3", "3"]], csv[0].to_a)
208
+ assert_equal([[:"1", 1], [:"2", 2], [:"3", 3]], csv[1].to_a)
209
+ assert_nil(csv[2])
210
+ end
211
+
212
+ def test_builtin_downcase_converter
213
+ csv = FastCSV.parse( "One,TWO Three", headers: true,
214
+ return_headers: true,
215
+ header_converters: :downcase )
216
+ assert_equal(%w{one two\ three}, csv.headers)
217
+ end
218
+
219
+ def test_builtin_symbol_converter
220
+ csv = FastCSV.parse( "One,TWO Three", headers: true,
221
+ return_headers: true,
222
+ header_converters: :symbol )
223
+ assert_equal([:one, :two_three], csv.headers)
224
+ end
225
+
226
+ def test_custom_converter
227
+ converter = lambda { |header| header.tr(" ", "_") }
228
+ csv = FastCSV.parse( "One,TWO Three",
229
+ headers: true,
230
+ return_headers: true,
231
+ header_converters: converter )
232
+ assert_equal(%w{One TWO_Three}, csv.headers)
233
+ end
234
+
235
+ def test_table_support
236
+ csv = nil
237
+ assert_nothing_raised(Exception) do
238
+ csv = FastCSV.parse(@data, headers: true)
239
+ end
240
+
241
+ assert_instance_of(FastCSV::Table, csv)
242
+ end
243
+
244
+ def test_skip_blanks
245
+ @data = <<-END_CSV.gsub(/^ +/, "")
246
+
247
+
248
+ A,B,C
249
+
250
+ 1,2,3
251
+
252
+
253
+
254
+ END_CSV
255
+
256
+ expected = [%w[1 2 3]]
257
+ FastCSV.parse(@data, headers: true, skip_blanks: true) do |row|
258
+ assert_equal(expected.shift, row.fields)
259
+ end
260
+
261
+ expected = [%w[A B C], %w[1 2 3]]
262
+ FastCSV.parse( @data,
263
+ headers: true,
264
+ return_headers: true,
265
+ skip_blanks: true ) do |row|
266
+ assert_equal(expected.shift, row.fields)
267
+ end
268
+ end
269
+
270
+ def test_headers_reader
271
+ # no headers
272
+ assert_nil(FastCSV.new(@data).headers)
273
+
274
+ # headers
275
+ csv = FastCSV.new(@data, headers: true)
276
+ assert_equal(true, csv.headers) # before headers are read
277
+ csv.shift # set headers
278
+ assert_equal(%w[first second third], csv.headers) # after headers are read
279
+ end
280
+
281
+ def test_blank_row_bug_fix
282
+ @data += "\n#{@data}" # add a blank row
283
+
284
+ # ensure that everything returned is a Row object
285
+ FastCSV.parse(@data, headers: true) do |row|
286
+ assert_instance_of(FastCSV::Row, row)
287
+ end
288
+ end
289
+ end