csvlint 0.4.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/dependabot.yml +11 -0
- data/.github/workflows/push.yml +35 -0
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/.standard_todo.yml +43 -0
- data/CHANGELOG.md +38 -0
- data/Dockerfile +16 -0
- data/Gemfile +2 -2
- data/README.md +13 -10
- data/Rakefile +7 -7
- data/bin/create_schema +2 -2
- data/csvlint.gemspec +19 -22
- data/docker_notes_for_windows.txt +20 -0
- data/features/step_definitions/cli_steps.rb +11 -11
- data/features/step_definitions/information_steps.rb +4 -4
- data/features/step_definitions/parse_csv_steps.rb +11 -11
- data/features/step_definitions/schema_validation_steps.rb +10 -10
- data/features/step_definitions/sources_steps.rb +1 -1
- data/features/step_definitions/validation_errors_steps.rb +19 -19
- data/features/step_definitions/validation_info_steps.rb +9 -9
- data/features/step_definitions/validation_warnings_steps.rb +11 -11
- data/features/support/aruba.rb +10 -9
- data/features/support/earl_formatter.rb +39 -39
- data/features/support/env.rb +10 -11
- data/features/support/load_tests.rb +109 -105
- data/features/support/webmock.rb +3 -1
- data/lib/csvlint/cli.rb +136 -142
- data/lib/csvlint/csvw/column.rb +279 -280
- data/lib/csvlint/csvw/date_format.rb +90 -92
- data/lib/csvlint/csvw/metadata_error.rb +1 -3
- data/lib/csvlint/csvw/number_format.rb +40 -32
- data/lib/csvlint/csvw/property_checker.rb +714 -717
- data/lib/csvlint/csvw/table.rb +49 -52
- data/lib/csvlint/csvw/table_group.rb +24 -23
- data/lib/csvlint/error_collector.rb +2 -0
- data/lib/csvlint/error_message.rb +0 -1
- data/lib/csvlint/field.rb +153 -141
- data/lib/csvlint/schema.rb +35 -43
- data/lib/csvlint/validate.rb +173 -151
- data/lib/csvlint/version.rb +1 -1
- data/lib/csvlint.rb +22 -23
- data/spec/csvw/column_spec.rb +15 -16
- data/spec/csvw/date_format_spec.rb +5 -7
- data/spec/csvw/number_format_spec.rb +2 -4
- data/spec/csvw/table_group_spec.rb +103 -105
- data/spec/csvw/table_spec.rb +71 -73
- data/spec/field_spec.rb +116 -121
- data/spec/schema_spec.rb +131 -141
- data/spec/spec_helper.rb +6 -6
- data/spec/validator_spec.rb +167 -203
- metadata +41 -85
- data/.travis.yml +0 -37
data/spec/validator_spec.rb
CHANGED
@@ -1,49 +1,45 @@
|
|
1
|
-
require
|
1
|
+
require "spec_helper"
|
2
2
|
|
3
3
|
describe Csvlint::Validator do
|
4
|
-
|
5
4
|
before do
|
6
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
7
|
-
stub_request(:get, "http://example.com/.well-known/csvm").to_return(:
|
8
|
-
stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(:
|
9
|
-
stub_request(:get, "http://example.com/csv-metadata.json").to_return(:
|
5
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, body: "")
|
6
|
+
stub_request(:get, "http://example.com/.well-known/csvm").to_return(status: 404)
|
7
|
+
stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(status: 404)
|
8
|
+
stub_request(:get, "http://example.com/csv-metadata.json").to_return(status: 404)
|
10
9
|
end
|
11
10
|
|
12
11
|
it "should validate from a URL" do
|
13
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
12
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
14
13
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
15
14
|
|
16
15
|
expect(validator.valid?).to eql(true)
|
17
|
-
expect(validator.instance_variable_get(
|
18
|
-
expect(validator.instance_variable_get(
|
16
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
17
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
|
19
18
|
expect(validator.data.size).to eql(3)
|
20
19
|
end
|
21
20
|
|
22
21
|
it "should validate from a file path" do
|
23
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
22
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
24
23
|
|
25
24
|
expect(validator.valid?).to eql(true)
|
26
|
-
expect(validator.instance_variable_get(
|
27
|
-
expect(validator.instance_variable_get(
|
25
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
26
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
|
28
27
|
expect(validator.data.size).to eql(3)
|
29
28
|
end
|
30
29
|
|
31
30
|
it "should validate from a file path including whitespace" do
|
32
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
31
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "white space in filename.csv")))
|
33
32
|
|
34
33
|
expect(validator.valid?).to eql(true)
|
35
34
|
end
|
36
35
|
|
37
36
|
context "multi line CSV validation with included schema" do
|
38
|
-
|
39
37
|
end
|
40
38
|
|
41
39
|
context "single line row validation with included schema" do
|
42
|
-
|
43
40
|
end
|
44
41
|
|
45
42
|
context "validation with multiple lines: " do
|
46
|
-
|
47
43
|
# TODO multiple lines permits testing of warnings
|
48
44
|
# TODO need more assertions in each test IE @formats
|
49
45
|
# TODO the phrasing of col_counts if only consulting specs might be confusing
|
@@ -52,22 +48,21 @@ describe Csvlint::Validator do
|
|
52
48
|
|
53
49
|
it ".each() -> parse_contents method validates a well formed CSV" do
|
54
50
|
# when invoking parse contents
|
55
|
-
data = StringIO.new(%
|
51
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
|
56
52
|
|
57
53
|
validator = Csvlint::Validator.new(data)
|
58
54
|
|
59
55
|
expect(validator.valid?).to eql(true)
|
60
56
|
# TODO would be beneficial to know how formats functions WRT to headers - check_format.feature:17 returns 3 rows total
|
61
57
|
# TODO in its formats object but is provided with 5 rows (with one nil row) [uses validation_warnings_steps.rb]
|
62
|
-
expect(validator.instance_variable_get(
|
63
|
-
expect(validator.instance_variable_get(
|
58
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
59
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
|
64
60
|
expect(validator.data.size).to eql(4)
|
65
|
-
|
66
61
|
end
|
67
62
|
|
68
63
|
it ".each() -> `parse_contents` parses malformed CSV and catches unclosed quote" do
|
69
64
|
# doesn't build warnings because check_consistency isn't invoked
|
70
|
-
data = StringIO.new(%
|
65
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1))
|
71
66
|
|
72
67
|
validator = Csvlint::Validator.new(data)
|
73
68
|
|
@@ -76,23 +71,10 @@ describe Csvlint::Validator do
|
|
76
71
|
expect(validator.errors.first.type).to eql(:unclosed_quote)
|
77
72
|
end
|
78
73
|
|
79
|
-
it ".each() -> `parse_contents` parses malformed CSV and catches stray quote" do
|
80
|
-
pending "cannot make Ruby generate a stray quote error"
|
81
|
-
# doesn't build warnings because check_consistency isn't invoked
|
82
|
-
# TODO below is trailing whitespace but is interpreted as a stray quote
|
83
|
-
data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1""})
|
84
|
-
|
85
|
-
validator = Csvlint::Validator.new(data)
|
86
|
-
|
87
|
-
expect(validator.valid?).to eql(false)
|
88
|
-
expect(validator.errors.first.type).to eql(:stray_quote)
|
89
|
-
expect(validator.errors.count).to eql(1)
|
90
|
-
end
|
91
|
-
|
92
74
|
it ".each() -> `parse_contents` parses malformed CSV and catches whitespace and edge case" do
|
93
75
|
# when this data gets passed the header it rescues a whitespace error, resulting in the header row being discarded
|
94
76
|
# TODO - check if this is an edge case, currently passing because it requires advice on how to specify
|
95
|
-
data = StringIO.new(%
|
77
|
+
data = StringIO.new(%( "Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1" ))
|
96
78
|
|
97
79
|
validator = Csvlint::Validator.new(data)
|
98
80
|
|
@@ -102,13 +84,13 @@ describe Csvlint::Validator do
|
|
102
84
|
end
|
103
85
|
|
104
86
|
it "handles line breaks within a cell" do
|
105
|
-
data = StringIO.new(%
|
87
|
+
data = StringIO.new(%("a","b","c"\r\n"d","e","this is\r\nvalid"\r\n"a","b","c"))
|
106
88
|
validator = Csvlint::Validator.new(data)
|
107
89
|
expect(validator.valid?).to eql(true)
|
108
90
|
end
|
109
91
|
|
110
92
|
it "handles multiple line breaks within a cell" do
|
111
|
-
data = StringIO.new(%
|
93
|
+
data = StringIO.new(%("a","b","c"\r\n"d","this is\r\n valid","as is this\r\n too"))
|
112
94
|
validator = Csvlint::Validator.new(data)
|
113
95
|
expect(validator.valid?).to eql(true)
|
114
96
|
end
|
@@ -117,50 +99,50 @@ describe Csvlint::Validator do
|
|
117
99
|
context "csv dialect" do
|
118
100
|
it "should provide sensible defaults for CSV parsing" do
|
119
101
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
120
|
-
opts = validator.instance_variable_get(
|
102
|
+
opts = validator.instance_variable_get(:@csv_options)
|
121
103
|
expect(opts).to include({
|
122
|
-
:
|
123
|
-
:
|
124
|
-
:
|
125
|
-
:
|
104
|
+
col_sep: ",",
|
105
|
+
row_sep: :auto,
|
106
|
+
quote_char: '"',
|
107
|
+
skip_blanks: false
|
126
108
|
})
|
127
109
|
end
|
128
110
|
|
129
111
|
it "should map CSV DDF to correct values" do
|
130
112
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
131
|
-
opts = validator.dialect_to_csv_options(
|
113
|
+
opts = validator.dialect_to_csv_options({
|
132
114
|
"lineTerminator" => "\n",
|
133
115
|
"delimiter" => "\t",
|
134
116
|
"quoteChar" => "'"
|
135
117
|
})
|
136
118
|
expect(opts).to include({
|
137
|
-
:
|
138
|
-
:
|
139
|
-
:
|
140
|
-
:
|
119
|
+
col_sep: "\t",
|
120
|
+
row_sep: "\n",
|
121
|
+
quote_char: "'",
|
122
|
+
skip_blanks: false
|
141
123
|
})
|
142
124
|
end
|
143
125
|
|
144
126
|
it ".each() -> `validate` to pass input in streaming fashion" do
|
145
127
|
# warnings are built when validate is used to call all three methods
|
146
|
-
data = StringIO.new(%
|
128
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
|
147
129
|
validator = Csvlint::Validator.new(data)
|
148
130
|
|
149
131
|
expect(validator.valid?).to eql(true)
|
150
|
-
expect(validator.instance_variable_get(
|
151
|
-
expect(validator.instance_variable_get(
|
132
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
133
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
|
152
134
|
expect(validator.data.size).to eql(4)
|
153
135
|
expect(validator.info_messages.count).to eql(1)
|
154
136
|
end
|
155
137
|
|
156
138
|
it ".each() -> `validate` parses malformed CSV, populates errors, warnings & info_msgs,invokes finish()" do
|
157
|
-
data = StringIO.new(%
|
139
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"1","two","3"\r\n"3","2", "1"))
|
158
140
|
|
159
141
|
validator = Csvlint::Validator.new(data)
|
160
142
|
|
161
143
|
expect(validator.valid?).to eql(false)
|
162
|
-
expect(validator.instance_variable_get(
|
163
|
-
expect(validator.instance_variable_get(
|
144
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
145
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
|
164
146
|
expect(validator.data.size).to eql(5)
|
165
147
|
expect(validator.info_messages.count).to eql(1)
|
166
148
|
expect(validator.errors.count).to eql(1)
|
@@ -170,7 +152,7 @@ describe Csvlint::Validator do
|
|
170
152
|
end
|
171
153
|
|
172
154
|
it "File.open.each_line -> `validate` passes a valid csv" do
|
173
|
-
filename =
|
155
|
+
filename = "valid_many_rows.csv"
|
174
156
|
file = File.join(File.expand_path(Dir.pwd), "features", "fixtures", filename)
|
175
157
|
validator = Csvlint::Validator.new(File.new(file))
|
176
158
|
|
@@ -179,11 +161,9 @@ describe Csvlint::Validator do
|
|
179
161
|
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
180
162
|
expect(validator.info_messages.first.category).to eql(:structure)
|
181
163
|
end
|
182
|
-
|
183
164
|
end
|
184
165
|
|
185
166
|
context "with a single row" do
|
186
|
-
|
187
167
|
it "validates correctly" do
|
188
168
|
stream = "\"a\",\"b\",\"c\"\r\n"
|
189
169
|
validator = Csvlint::Validator.new(StringIO.new(stream), "header" => false)
|
@@ -217,10 +197,10 @@ describe Csvlint::Validator do
|
|
217
197
|
stream = "1,2,3\r\n"
|
218
198
|
validator = Csvlint::Validator.new(StringIO.new(stream))
|
219
199
|
|
220
|
-
expect(
|
221
|
-
expect(
|
222
|
-
expect(
|
223
|
-
expect(
|
200
|
+
expect(validator.valid?).to eql(true)
|
201
|
+
expect(validator.info_messages.size).to eql(1)
|
202
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
203
|
+
expect(validator.info_messages.first.category).to eql(:structure)
|
224
204
|
end
|
225
205
|
|
226
206
|
it "should evaluate the row as 'row 2' when stipulated" do
|
@@ -230,11 +210,9 @@ describe Csvlint::Validator do
|
|
230
210
|
expect(validator.valid?).to eql(true)
|
231
211
|
expect(validator.info_messages.size).to eql(0)
|
232
212
|
end
|
233
|
-
|
234
213
|
end
|
235
214
|
|
236
215
|
context "it returns the correct error from ERROR_MATCHES" do
|
237
|
-
|
238
216
|
it "checks for unclosed quotes" do
|
239
217
|
stream = "\"a,\"b\",\"c\"\n"
|
240
218
|
validator = Csvlint::Validator.new(StringIO.new(stream))
|
@@ -243,7 +221,6 @@ describe Csvlint::Validator do
|
|
243
221
|
expect(validator.errors.first.type).to eql(:unclosed_quote)
|
244
222
|
end
|
245
223
|
|
246
|
-
|
247
224
|
# TODO stray quotes is not covered in any spec in this library
|
248
225
|
# it "checks for stray quotes" do
|
249
226
|
# stream = "\"a\",“b“,\"c\"" "\r\n"
|
@@ -271,64 +248,61 @@ describe Csvlint::Validator do
|
|
271
248
|
expect(validator.errors.count).to eq(1)
|
272
249
|
expect(validator.errors.first.type).to eql(:line_breaks)
|
273
250
|
end
|
274
|
-
|
275
251
|
end
|
276
252
|
|
277
253
|
context "when validating headers" do
|
278
|
-
|
279
254
|
it "should warn if column names aren't unique" do
|
280
|
-
data = StringIO.new(
|
255
|
+
data = StringIO.new("minimum, minimum")
|
281
256
|
validator = Csvlint::Validator.new(data)
|
282
257
|
validator.reset
|
283
|
-
expect(
|
284
|
-
expect(
|
285
|
-
expect(
|
286
|
-
expect(
|
258
|
+
expect(validator.validate_header(["minimum", "minimum"])).to eql(true)
|
259
|
+
expect(validator.warnings.size).to eql(1)
|
260
|
+
expect(validator.warnings.first.type).to eql(:duplicate_column_name)
|
261
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
287
262
|
end
|
288
263
|
|
289
264
|
it "should warn if column names are blank" do
|
290
|
-
data = StringIO.new(
|
265
|
+
data = StringIO.new("minimum,")
|
291
266
|
validator = Csvlint::Validator.new(data)
|
292
267
|
|
293
|
-
expect(
|
294
|
-
expect(
|
295
|
-
expect(
|
296
|
-
expect(
|
268
|
+
expect(validator.validate_header(["minimum", ""])).to eql(true)
|
269
|
+
expect(validator.warnings.size).to eql(1)
|
270
|
+
expect(validator.warnings.first.type).to eql(:empty_column_name)
|
271
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
297
272
|
end
|
298
273
|
|
299
274
|
it "should include info message about missing header when we have assumed a header" do
|
300
|
-
data = StringIO.new(
|
275
|
+
data = StringIO.new("1,2,3\r\n")
|
301
276
|
validator = Csvlint::Validator.new(data)
|
302
|
-
expect(
|
303
|
-
expect(
|
304
|
-
expect(
|
305
|
-
expect(
|
277
|
+
expect(validator.valid?).to eql(true)
|
278
|
+
expect(validator.info_messages.size).to eql(1)
|
279
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
280
|
+
expect(validator.info_messages.first.category).to eql(:structure)
|
306
281
|
end
|
307
282
|
|
308
283
|
it "should not include info message about missing header when we are told about the header" do
|
309
|
-
data = StringIO.new(
|
284
|
+
data = StringIO.new("1,2,3\r\n")
|
310
285
|
validator = Csvlint::Validator.new(data, "header" => false)
|
311
|
-
expect(
|
312
|
-
expect(
|
286
|
+
expect(validator.valid?).to eql(true)
|
287
|
+
expect(validator.info_messages.size).to eql(0)
|
313
288
|
end
|
314
289
|
end
|
315
290
|
|
316
291
|
context "build_formats" do
|
317
|
-
|
318
292
|
{
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
293
|
+
string: "foo",
|
294
|
+
numeric: "1",
|
295
|
+
uri: "http://www.example.com",
|
296
|
+
dateTime_iso8601: "2013-01-01T13:00:00Z",
|
297
|
+
date_db: "2013-01-01",
|
298
|
+
dateTime_hms: "13:00:00"
|
325
299
|
}.each do |type, content|
|
326
300
|
it "should return the format of #{type} correctly" do
|
327
301
|
row = [content]
|
328
302
|
|
329
303
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
330
304
|
validator.build_formats(row)
|
331
|
-
formats = validator.instance_variable_get(
|
305
|
+
formats = validator.instance_variable_get(:@formats)
|
332
306
|
|
333
307
|
expect(formats[0].keys.first).to eql type
|
334
308
|
end
|
@@ -339,7 +313,7 @@ describe Csvlint::Validator do
|
|
339
313
|
|
340
314
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
341
315
|
validator.build_formats(row)
|
342
|
-
formats = validator.instance_variable_get(
|
316
|
+
formats = validator.instance_variable_get(:@formats)
|
343
317
|
|
344
318
|
expect(formats[0].keys.first).to eql :numeric
|
345
319
|
expect(formats[1].keys.first).to eql :numeric
|
@@ -351,15 +325,15 @@ describe Csvlint::Validator do
|
|
351
325
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
352
326
|
validator.build_formats(row)
|
353
327
|
|
354
|
-
formats = validator.instance_variable_get(
|
328
|
+
formats = validator.instance_variable_get(:@formats)
|
355
329
|
expect(formats).to eql []
|
356
330
|
end
|
357
331
|
|
358
332
|
it "should work correctly for single columns" do
|
359
333
|
rows = [
|
360
|
-
|
361
|
-
|
362
|
-
|
334
|
+
["foo"],
|
335
|
+
["bar"],
|
336
|
+
["baz"]
|
363
337
|
]
|
364
338
|
|
365
339
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
@@ -368,14 +342,14 @@ describe Csvlint::Validator do
|
|
368
342
|
validator.build_formats(row)
|
369
343
|
end
|
370
344
|
|
371
|
-
formats = validator.instance_variable_get(
|
372
|
-
expect(formats).to eql [{:
|
345
|
+
formats = validator.instance_variable_get(:@formats)
|
346
|
+
expect(formats).to eql [{string: 3}]
|
373
347
|
end
|
374
348
|
|
375
349
|
it "should return formats correctly if a row is blank" do
|
376
350
|
rows = [
|
377
|
-
|
378
|
-
|
351
|
+
[],
|
352
|
+
["foo", "1", "$2345"]
|
379
353
|
]
|
380
354
|
|
381
355
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
@@ -384,72 +358,68 @@ describe Csvlint::Validator do
|
|
384
358
|
validator.build_formats(row)
|
385
359
|
end
|
386
360
|
|
387
|
-
formats = validator.instance_variable_get(
|
361
|
+
formats = validator.instance_variable_get(:@formats)
|
388
362
|
|
389
363
|
expect(formats).to eql [
|
390
|
-
{:
|
391
|
-
{:
|
392
|
-
{:
|
364
|
+
{string: 1},
|
365
|
+
{numeric: 1},
|
366
|
+
{string: 1}
|
393
367
|
]
|
394
368
|
end
|
395
|
-
|
396
369
|
end
|
397
370
|
|
398
371
|
context "csv dialect" do
|
399
372
|
it "should provide sensible defaults for CSV parsing" do
|
400
373
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
401
|
-
opts = validator.instance_variable_get(
|
374
|
+
opts = validator.instance_variable_get(:@csv_options)
|
402
375
|
expect(opts).to include({
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
376
|
+
col_sep: ",",
|
377
|
+
row_sep: :auto,
|
378
|
+
quote_char: '"',
|
379
|
+
skip_blanks: false
|
380
|
+
})
|
408
381
|
end
|
409
382
|
|
410
383
|
it "should map CSV DDF to correct values" do
|
411
384
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
412
385
|
opts = validator.dialect_to_csv_options({
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
386
|
+
"lineTerminator" => "\n",
|
387
|
+
"delimiter" => "\t",
|
388
|
+
"quoteChar" => "'"
|
389
|
+
})
|
417
390
|
expect(opts).to include({
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
391
|
+
col_sep: "\t",
|
392
|
+
row_sep: "\n",
|
393
|
+
quote_char: "'",
|
394
|
+
skip_blanks: false
|
395
|
+
})
|
423
396
|
end
|
424
|
-
|
425
397
|
end
|
426
398
|
|
427
399
|
context "check_consistency" do
|
428
|
-
|
429
400
|
it "should return a warning if columns have inconsistent values" do
|
430
401
|
formats = [
|
431
|
-
|
432
|
-
|
433
|
-
|
402
|
+
{string: 3},
|
403
|
+
{string: 2, numeric: 1},
|
404
|
+
{numeric: 3}
|
434
405
|
]
|
435
406
|
|
436
407
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
437
|
-
validator.instance_variable_set(
|
408
|
+
validator.instance_variable_set(:@formats, formats)
|
438
409
|
validator.check_consistency
|
439
410
|
|
440
|
-
warnings = validator.instance_variable_get(
|
411
|
+
warnings = validator.instance_variable_get(:@warnings)
|
441
412
|
warnings.delete_if { |h| h.type != :inconsistent_values }
|
442
413
|
|
443
414
|
expect(warnings.count).to eql 1
|
444
415
|
end
|
445
|
-
|
446
416
|
end
|
447
417
|
|
448
|
-
#TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
|
418
|
+
# TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
|
449
419
|
context "when detecting headers" do
|
450
420
|
it "should default to expecting a header" do
|
451
421
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
452
|
-
expect(
|
422
|
+
expect(validator.header?).to eql(true)
|
453
423
|
end
|
454
424
|
|
455
425
|
it "should look in CSV options to detect header" do
|
@@ -457,171 +427,165 @@ describe Csvlint::Validator do
|
|
457
427
|
"header" => true
|
458
428
|
}
|
459
429
|
validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
|
460
|
-
expect(
|
430
|
+
expect(validator.header?).to eql(true)
|
461
431
|
opts = {
|
462
432
|
"header" => false
|
463
433
|
}
|
464
434
|
validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
|
465
|
-
expect(
|
435
|
+
expect(validator.header?).to eql(false)
|
466
436
|
end
|
467
437
|
|
468
438
|
it "should look in content-type for header=absent" do
|
469
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
439
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=absent"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
470
440
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
471
|
-
expect(
|
472
|
-
expect(
|
441
|
+
expect(validator.header?).to eql(false)
|
442
|
+
expect(validator.errors.size).to eql(0)
|
473
443
|
end
|
474
444
|
|
475
445
|
it "should look in content-type for header=present" do
|
476
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
446
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=present"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
477
447
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
478
|
-
expect(
|
479
|
-
expect(
|
448
|
+
expect(validator.header?).to eql(true)
|
449
|
+
expect(validator.errors.size).to eql(0)
|
480
450
|
end
|
481
451
|
|
482
452
|
it "assume header present if not specified in content type" do
|
483
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
453
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
484
454
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
485
|
-
expect(
|
486
|
-
expect(
|
487
|
-
expect(
|
488
|
-
expect(
|
455
|
+
expect(validator.header?).to eql(true)
|
456
|
+
expect(validator.errors.size).to eql(0)
|
457
|
+
expect(validator.info_messages.size).to eql(1)
|
458
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
489
459
|
end
|
490
460
|
|
491
461
|
it "give wrong content type error if content type is wrong" do
|
492
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
462
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
493
463
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
494
|
-
expect(
|
495
|
-
expect(
|
496
|
-
expect(
|
464
|
+
expect(validator.header?).to eql(true)
|
465
|
+
expect(validator.errors.size).to eql(1)
|
466
|
+
expect(validator.errors[0].type).to eql(:wrong_content_type)
|
497
467
|
end
|
498
|
-
|
499
468
|
end
|
500
469
|
|
501
470
|
context "when validating headers" do
|
502
471
|
it "should warn if column names aren't unique" do
|
503
|
-
data = StringIO.new(
|
472
|
+
data = StringIO.new("minimum, minimum")
|
504
473
|
validator = Csvlint::Validator.new(data)
|
505
|
-
expect(
|
506
|
-
expect(
|
507
|
-
expect(
|
474
|
+
expect(validator.warnings.size).to eql(1)
|
475
|
+
expect(validator.warnings.first.type).to eql(:duplicate_column_name)
|
476
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
508
477
|
end
|
509
478
|
|
510
479
|
it "should warn if column names are blank" do
|
511
|
-
data = StringIO.new(
|
480
|
+
data = StringIO.new("minimum,")
|
512
481
|
validator = Csvlint::Validator.new(data)
|
513
482
|
|
514
|
-
expect(
|
515
|
-
expect(
|
516
|
-
expect(
|
517
|
-
expect(
|
483
|
+
expect(validator.validate_header(["minimum", ""])).to eql(true)
|
484
|
+
expect(validator.warnings.size).to eql(1)
|
485
|
+
expect(validator.warnings.first.type).to eql(:empty_column_name)
|
486
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
518
487
|
end
|
519
488
|
|
520
489
|
it "should include info message about missing header when we have assumed a header" do
|
521
|
-
data = StringIO.new(
|
490
|
+
data = StringIO.new("1,2,3\r\n")
|
522
491
|
validator = Csvlint::Validator.new(data)
|
523
492
|
|
524
|
-
expect(
|
525
|
-
expect(
|
526
|
-
expect(
|
527
|
-
expect(
|
493
|
+
expect(validator.valid?).to eql(true)
|
494
|
+
expect(validator.info_messages.size).to eql(1)
|
495
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
496
|
+
expect(validator.info_messages.first.category).to eql(:structure)
|
528
497
|
end
|
529
498
|
|
530
499
|
it "should not include info message about missing header when we are told about the header" do
|
531
|
-
data = StringIO.new(
|
532
|
-
validator = Csvlint::Validator.new(data, "header"=>false)
|
533
|
-
expect(
|
534
|
-
expect(
|
500
|
+
data = StringIO.new("1,2,3\r\n")
|
501
|
+
validator = Csvlint::Validator.new(data, "header" => false)
|
502
|
+
expect(validator.valid?).to eql(true)
|
503
|
+
expect(validator.info_messages.size).to eql(0)
|
535
504
|
end
|
536
505
|
|
537
506
|
it "should not be an error if we have assumed a header, there is no dialect and content-type doesn't declare header, as we assume header=present" do
|
538
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
507
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
539
508
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
540
|
-
expect(
|
509
|
+
expect(validator.valid?).to eql(true)
|
541
510
|
end
|
542
511
|
|
543
512
|
it "should be valid if we have a dialect and the data is from the web" do
|
544
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
545
|
-
#header defaults to true in csv dialect, so this is valid
|
513
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
514
|
+
# header defaults to true in csv dialect, so this is valid
|
546
515
|
validator = Csvlint::Validator.new("http://example.com/example.csv", {})
|
547
|
-
expect(
|
516
|
+
expect(validator.valid?).to eql(true)
|
548
517
|
|
549
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
550
|
-
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>true})
|
551
|
-
expect(
|
518
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
519
|
+
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => true})
|
520
|
+
expect(validator.valid?).to eql(true)
|
552
521
|
|
553
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
554
|
-
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>false})
|
555
|
-
expect(
|
522
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
523
|
+
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => false})
|
524
|
+
expect(validator.valid?).to eql(true)
|
556
525
|
end
|
557
|
-
|
558
526
|
end
|
559
527
|
|
560
528
|
context "accessing metadata" do
|
561
|
-
|
562
529
|
before :all do
|
563
|
-
stub_request(:get, "http://example.com/crlf.csv").to_return(:
|
564
|
-
stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(:
|
530
|
+
stub_request(:get, "http://example.com/crlf.csv").to_return(status: 200, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "windows-line-endings.csv")))
|
531
|
+
stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(status: 404)
|
565
532
|
end
|
566
533
|
|
567
534
|
it "can get line break symbol" do
|
568
535
|
validator = Csvlint::Validator.new("http://example.com/crlf.csv")
|
569
536
|
expect(validator.line_breaks).to eql "\r\n"
|
570
537
|
end
|
571
|
-
|
572
538
|
end
|
573
539
|
|
574
540
|
it "should give access to the complete CSV data file" do
|
575
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
576
|
-
|
577
|
-
|
541
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
|
542
|
+
headers: {"Content-Type" => "text/csv; header=present"},
|
543
|
+
body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
578
544
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
579
|
-
expect(
|
545
|
+
expect(validator.valid?).to eql(true)
|
580
546
|
data = validator.data
|
581
547
|
|
582
|
-
expect(
|
583
|
-
expect(
|
584
|
-
expect(
|
548
|
+
expect(data.count).to eql 3
|
549
|
+
expect(data[0]).to eql ["Foo", "Bar", "Baz"]
|
550
|
+
expect(data[2]).to eql ["3", "2", "1"]
|
585
551
|
end
|
586
552
|
|
587
553
|
it "should count the total number of rows read" do
|
588
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
589
|
-
|
590
|
-
|
554
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
|
555
|
+
headers: {"Content-Type" => "text/csv; header=present"},
|
556
|
+
body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
591
557
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
592
558
|
expect(validator.row_count).to eq(3)
|
593
559
|
end
|
594
560
|
|
595
561
|
it "should limit number of lines read" do
|
596
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
597
|
-
|
598
|
-
|
562
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
|
563
|
+
headers: {"Content-Type" => "text/csv; header=present"},
|
564
|
+
body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
599
565
|
validator = Csvlint::Validator.new("http://example.com/example.csv", {}, nil, limit_lines: 2)
|
600
|
-
expect(
|
566
|
+
expect(validator.valid?).to eql(true)
|
601
567
|
data = validator.data
|
602
|
-
expect(
|
603
|
-
expect(
|
568
|
+
expect(data.count).to eql 2
|
569
|
+
expect(data[0]).to eql ["Foo", "Bar", "Baz"]
|
604
570
|
end
|
605
571
|
|
606
572
|
context "with a lambda" do
|
607
|
-
|
608
573
|
it "should call a lambda for each line" do
|
609
574
|
@count = 0
|
610
|
-
mylambda = lambda { |row| @count
|
611
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
575
|
+
mylambda = lambda { |row| @count += 1 }
|
576
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
|
612
577
|
expect(@count).to eq(3)
|
613
578
|
end
|
614
579
|
|
615
580
|
it "reports back the status of each line" do
|
616
581
|
@results = []
|
617
582
|
mylambda = lambda { |row| @results << row.current_line }
|
618
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
583
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
|
619
584
|
expect(@results.count).to eq(3)
|
620
585
|
expect(@results[0]).to eq(1)
|
621
586
|
expect(@results[1]).to eq(2)
|
622
587
|
expect(@results[2]).to eq(3)
|
623
588
|
end
|
624
|
-
|
625
589
|
end
|
626
590
|
|
627
591
|
# Commented out because there is currently no way to mock redirects with Typhoeus and WebMock - see https://github.com/bblimke/webmock/issues/237
|