csvlint 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +4 -0
- data/.github/workflows/push.yml +14 -2
- data/.ruby-version +1 -1
- data/.standard_todo.yml +43 -0
- data/Dockerfile +16 -0
- data/Gemfile +2 -2
- data/README.md +9 -9
- data/Rakefile +7 -7
- data/csvlint.gemspec +14 -16
- data/docker_notes_for_windows.txt +20 -0
- data/features/step_definitions/cli_steps.rb +11 -11
- data/features/step_definitions/information_steps.rb +4 -4
- data/features/step_definitions/parse_csv_steps.rb +11 -11
- data/features/step_definitions/schema_validation_steps.rb +10 -10
- data/features/step_definitions/sources_steps.rb +1 -1
- data/features/step_definitions/validation_errors_steps.rb +19 -19
- data/features/step_definitions/validation_info_steps.rb +9 -9
- data/features/step_definitions/validation_warnings_steps.rb +11 -11
- data/features/support/aruba.rb +6 -6
- data/features/support/earl_formatter.rb +39 -39
- data/features/support/env.rb +10 -11
- data/features/support/load_tests.rb +107 -103
- data/features/support/webmock.rb +2 -2
- data/lib/csvlint/cli.rb +133 -130
- data/lib/csvlint/csvw/column.rb +279 -280
- data/lib/csvlint/csvw/date_format.rb +90 -92
- data/lib/csvlint/csvw/metadata_error.rb +1 -3
- data/lib/csvlint/csvw/number_format.rb +40 -32
- data/lib/csvlint/csvw/property_checker.rb +714 -717
- data/lib/csvlint/csvw/table.rb +49 -52
- data/lib/csvlint/csvw/table_group.rb +24 -23
- data/lib/csvlint/error_collector.rb +2 -0
- data/lib/csvlint/error_message.rb +0 -1
- data/lib/csvlint/field.rb +153 -141
- data/lib/csvlint/schema.rb +34 -42
- data/lib/csvlint/validate.rb +161 -143
- data/lib/csvlint/version.rb +1 -1
- data/lib/csvlint.rb +22 -23
- data/spec/csvw/column_spec.rb +15 -16
- data/spec/csvw/date_format_spec.rb +5 -7
- data/spec/csvw/number_format_spec.rb +2 -4
- data/spec/csvw/table_group_spec.rb +103 -105
- data/spec/csvw/table_spec.rb +71 -73
- data/spec/field_spec.rb +116 -121
- data/spec/schema_spec.rb +129 -139
- data/spec/spec_helper.rb +6 -6
- data/spec/validator_spec.rb +167 -190
- metadata +22 -55
data/spec/validator_spec.rb
CHANGED
@@ -1,49 +1,45 @@
|
|
1
|
-
require
|
1
|
+
require "spec_helper"
|
2
2
|
|
3
3
|
describe Csvlint::Validator do
|
4
|
-
|
5
4
|
before do
|
6
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
7
|
-
stub_request(:get, "http://example.com/.well-known/csvm").to_return(:
|
8
|
-
stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(:
|
9
|
-
stub_request(:get, "http://example.com/csv-metadata.json").to_return(:
|
5
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, body: "")
|
6
|
+
stub_request(:get, "http://example.com/.well-known/csvm").to_return(status: 404)
|
7
|
+
stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(status: 404)
|
8
|
+
stub_request(:get, "http://example.com/csv-metadata.json").to_return(status: 404)
|
10
9
|
end
|
11
10
|
|
12
11
|
it "should validate from a URL" do
|
13
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
12
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
14
13
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
15
14
|
|
16
15
|
expect(validator.valid?).to eql(true)
|
17
|
-
expect(validator.instance_variable_get(
|
18
|
-
expect(validator.instance_variable_get(
|
16
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
17
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
|
19
18
|
expect(validator.data.size).to eql(3)
|
20
19
|
end
|
21
20
|
|
22
21
|
it "should validate from a file path" do
|
23
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
22
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
24
23
|
|
25
24
|
expect(validator.valid?).to eql(true)
|
26
|
-
expect(validator.instance_variable_get(
|
27
|
-
expect(validator.instance_variable_get(
|
25
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
26
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
|
28
27
|
expect(validator.data.size).to eql(3)
|
29
28
|
end
|
30
29
|
|
31
30
|
it "should validate from a file path including whitespace" do
|
32
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
31
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "white space in filename.csv")))
|
33
32
|
|
34
33
|
expect(validator.valid?).to eql(true)
|
35
34
|
end
|
36
35
|
|
37
36
|
context "multi line CSV validation with included schema" do
|
38
|
-
|
39
37
|
end
|
40
38
|
|
41
39
|
context "single line row validation with included schema" do
|
42
|
-
|
43
40
|
end
|
44
41
|
|
45
42
|
context "validation with multiple lines: " do
|
46
|
-
|
47
43
|
# TODO multiple lines permits testing of warnings
|
48
44
|
# TODO need more assertions in each test IE @formats
|
49
45
|
# TODO the phrasing of col_counts if only consulting specs might be confusing
|
@@ -52,22 +48,21 @@ describe Csvlint::Validator do
|
|
52
48
|
|
53
49
|
it ".each() -> parse_contents method validates a well formed CSV" do
|
54
50
|
# when invoking parse contents
|
55
|
-
data = StringIO.new(%
|
51
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
|
56
52
|
|
57
53
|
validator = Csvlint::Validator.new(data)
|
58
54
|
|
59
55
|
expect(validator.valid?).to eql(true)
|
60
56
|
# TODO would be beneficial to know how formats functions WRT to headers - check_format.feature:17 returns 3 rows total
|
61
57
|
# TODO in its formats object but is provided with 5 rows (with one nil row) [uses validation_warnings_steps.rb]
|
62
|
-
expect(validator.instance_variable_get(
|
63
|
-
expect(validator.instance_variable_get(
|
58
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
59
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
|
64
60
|
expect(validator.data.size).to eql(4)
|
65
|
-
|
66
61
|
end
|
67
62
|
|
68
63
|
it ".each() -> `parse_contents` parses malformed CSV and catches unclosed quote" do
|
69
64
|
# doesn't build warnings because check_consistency isn't invoked
|
70
|
-
data = StringIO.new(%
|
65
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1))
|
71
66
|
|
72
67
|
validator = Csvlint::Validator.new(data)
|
73
68
|
|
@@ -79,7 +74,7 @@ describe Csvlint::Validator do
|
|
79
74
|
it ".each() -> `parse_contents` parses malformed CSV and catches whitespace and edge case" do
|
80
75
|
# when this data gets passed the header it rescues a whitespace error, resulting in the header row being discarded
|
81
76
|
# TODO - check if this is an edge case, currently passing because it requires advice on how to specify
|
82
|
-
data = StringIO.new(%
|
77
|
+
data = StringIO.new(%( "Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1" ))
|
83
78
|
|
84
79
|
validator = Csvlint::Validator.new(data)
|
85
80
|
|
@@ -89,13 +84,13 @@ describe Csvlint::Validator do
|
|
89
84
|
end
|
90
85
|
|
91
86
|
it "handles line breaks within a cell" do
|
92
|
-
data = StringIO.new(%
|
87
|
+
data = StringIO.new(%("a","b","c"\r\n"d","e","this is\r\nvalid"\r\n"a","b","c"))
|
93
88
|
validator = Csvlint::Validator.new(data)
|
94
89
|
expect(validator.valid?).to eql(true)
|
95
90
|
end
|
96
91
|
|
97
92
|
it "handles multiple line breaks within a cell" do
|
98
|
-
data = StringIO.new(%
|
93
|
+
data = StringIO.new(%("a","b","c"\r\n"d","this is\r\n valid","as is this\r\n too"))
|
99
94
|
validator = Csvlint::Validator.new(data)
|
100
95
|
expect(validator.valid?).to eql(true)
|
101
96
|
end
|
@@ -104,50 +99,50 @@ describe Csvlint::Validator do
|
|
104
99
|
context "csv dialect" do
|
105
100
|
it "should provide sensible defaults for CSV parsing" do
|
106
101
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
107
|
-
opts = validator.instance_variable_get(
|
102
|
+
opts = validator.instance_variable_get(:@csv_options)
|
108
103
|
expect(opts).to include({
|
109
|
-
:
|
110
|
-
:
|
111
|
-
:
|
112
|
-
:
|
104
|
+
col_sep: ",",
|
105
|
+
row_sep: :auto,
|
106
|
+
quote_char: '"',
|
107
|
+
skip_blanks: false
|
113
108
|
})
|
114
109
|
end
|
115
110
|
|
116
111
|
it "should map CSV DDF to correct values" do
|
117
112
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
118
|
-
opts = validator.dialect_to_csv_options(
|
113
|
+
opts = validator.dialect_to_csv_options({
|
119
114
|
"lineTerminator" => "\n",
|
120
115
|
"delimiter" => "\t",
|
121
116
|
"quoteChar" => "'"
|
122
117
|
})
|
123
118
|
expect(opts).to include({
|
124
|
-
:
|
125
|
-
:
|
126
|
-
:
|
127
|
-
:
|
119
|
+
col_sep: "\t",
|
120
|
+
row_sep: "\n",
|
121
|
+
quote_char: "'",
|
122
|
+
skip_blanks: false
|
128
123
|
})
|
129
124
|
end
|
130
125
|
|
131
126
|
it ".each() -> `validate` to pass input in streaming fashion" do
|
132
127
|
# warnings are built when validate is used to call all three methods
|
133
|
-
data = StringIO.new(%
|
128
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
|
134
129
|
validator = Csvlint::Validator.new(data)
|
135
130
|
|
136
131
|
expect(validator.valid?).to eql(true)
|
137
|
-
expect(validator.instance_variable_get(
|
138
|
-
expect(validator.instance_variable_get(
|
132
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
133
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
|
139
134
|
expect(validator.data.size).to eql(4)
|
140
135
|
expect(validator.info_messages.count).to eql(1)
|
141
136
|
end
|
142
137
|
|
143
138
|
it ".each() -> `validate` parses malformed CSV, populates errors, warnings & info_msgs,invokes finish()" do
|
144
|
-
data = StringIO.new(%
|
139
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"1","two","3"\r\n"3","2", "1"))
|
145
140
|
|
146
141
|
validator = Csvlint::Validator.new(data)
|
147
142
|
|
148
143
|
expect(validator.valid?).to eql(false)
|
149
|
-
expect(validator.instance_variable_get(
|
150
|
-
expect(validator.instance_variable_get(
|
144
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
145
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
|
151
146
|
expect(validator.data.size).to eql(5)
|
152
147
|
expect(validator.info_messages.count).to eql(1)
|
153
148
|
expect(validator.errors.count).to eql(1)
|
@@ -157,7 +152,7 @@ describe Csvlint::Validator do
|
|
157
152
|
end
|
158
153
|
|
159
154
|
it "File.open.each_line -> `validate` passes a valid csv" do
|
160
|
-
filename =
|
155
|
+
filename = "valid_many_rows.csv"
|
161
156
|
file = File.join(File.expand_path(Dir.pwd), "features", "fixtures", filename)
|
162
157
|
validator = Csvlint::Validator.new(File.new(file))
|
163
158
|
|
@@ -166,11 +161,9 @@ describe Csvlint::Validator do
|
|
166
161
|
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
167
162
|
expect(validator.info_messages.first.category).to eql(:structure)
|
168
163
|
end
|
169
|
-
|
170
164
|
end
|
171
165
|
|
172
166
|
context "with a single row" do
|
173
|
-
|
174
167
|
it "validates correctly" do
|
175
168
|
stream = "\"a\",\"b\",\"c\"\r\n"
|
176
169
|
validator = Csvlint::Validator.new(StringIO.new(stream), "header" => false)
|
@@ -204,10 +197,10 @@ describe Csvlint::Validator do
|
|
204
197
|
stream = "1,2,3\r\n"
|
205
198
|
validator = Csvlint::Validator.new(StringIO.new(stream))
|
206
199
|
|
207
|
-
expect(
|
208
|
-
expect(
|
209
|
-
expect(
|
210
|
-
expect(
|
200
|
+
expect(validator.valid?).to eql(true)
|
201
|
+
expect(validator.info_messages.size).to eql(1)
|
202
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
203
|
+
expect(validator.info_messages.first.category).to eql(:structure)
|
211
204
|
end
|
212
205
|
|
213
206
|
it "should evaluate the row as 'row 2' when stipulated" do
|
@@ -217,11 +210,9 @@ describe Csvlint::Validator do
|
|
217
210
|
expect(validator.valid?).to eql(true)
|
218
211
|
expect(validator.info_messages.size).to eql(0)
|
219
212
|
end
|
220
|
-
|
221
213
|
end
|
222
214
|
|
223
215
|
context "it returns the correct error from ERROR_MATCHES" do
|
224
|
-
|
225
216
|
it "checks for unclosed quotes" do
|
226
217
|
stream = "\"a,\"b\",\"c\"\n"
|
227
218
|
validator = Csvlint::Validator.new(StringIO.new(stream))
|
@@ -230,7 +221,6 @@ describe Csvlint::Validator do
|
|
230
221
|
expect(validator.errors.first.type).to eql(:unclosed_quote)
|
231
222
|
end
|
232
223
|
|
233
|
-
|
234
224
|
# TODO stray quotes is not covered in any spec in this library
|
235
225
|
# it "checks for stray quotes" do
|
236
226
|
# stream = "\"a\",“b“,\"c\"" "\r\n"
|
@@ -258,64 +248,61 @@ describe Csvlint::Validator do
|
|
258
248
|
expect(validator.errors.count).to eq(1)
|
259
249
|
expect(validator.errors.first.type).to eql(:line_breaks)
|
260
250
|
end
|
261
|
-
|
262
251
|
end
|
263
252
|
|
264
253
|
context "when validating headers" do
|
265
|
-
|
266
254
|
it "should warn if column names aren't unique" do
|
267
|
-
data = StringIO.new(
|
255
|
+
data = StringIO.new("minimum, minimum")
|
268
256
|
validator = Csvlint::Validator.new(data)
|
269
257
|
validator.reset
|
270
|
-
expect(
|
271
|
-
expect(
|
272
|
-
expect(
|
273
|
-
expect(
|
258
|
+
expect(validator.validate_header(["minimum", "minimum"])).to eql(true)
|
259
|
+
expect(validator.warnings.size).to eql(1)
|
260
|
+
expect(validator.warnings.first.type).to eql(:duplicate_column_name)
|
261
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
274
262
|
end
|
275
263
|
|
276
264
|
it "should warn if column names are blank" do
|
277
|
-
data = StringIO.new(
|
265
|
+
data = StringIO.new("minimum,")
|
278
266
|
validator = Csvlint::Validator.new(data)
|
279
267
|
|
280
|
-
expect(
|
281
|
-
expect(
|
282
|
-
expect(
|
283
|
-
expect(
|
268
|
+
expect(validator.validate_header(["minimum", ""])).to eql(true)
|
269
|
+
expect(validator.warnings.size).to eql(1)
|
270
|
+
expect(validator.warnings.first.type).to eql(:empty_column_name)
|
271
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
284
272
|
end
|
285
273
|
|
286
274
|
it "should include info message about missing header when we have assumed a header" do
|
287
|
-
data = StringIO.new(
|
275
|
+
data = StringIO.new("1,2,3\r\n")
|
288
276
|
validator = Csvlint::Validator.new(data)
|
289
|
-
expect(
|
290
|
-
expect(
|
291
|
-
expect(
|
292
|
-
expect(
|
277
|
+
expect(validator.valid?).to eql(true)
|
278
|
+
expect(validator.info_messages.size).to eql(1)
|
279
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
280
|
+
expect(validator.info_messages.first.category).to eql(:structure)
|
293
281
|
end
|
294
282
|
|
295
283
|
it "should not include info message about missing header when we are told about the header" do
|
296
|
-
data = StringIO.new(
|
284
|
+
data = StringIO.new("1,2,3\r\n")
|
297
285
|
validator = Csvlint::Validator.new(data, "header" => false)
|
298
|
-
expect(
|
299
|
-
expect(
|
286
|
+
expect(validator.valid?).to eql(true)
|
287
|
+
expect(validator.info_messages.size).to eql(0)
|
300
288
|
end
|
301
289
|
end
|
302
290
|
|
303
291
|
context "build_formats" do
|
304
|
-
|
305
292
|
{
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
293
|
+
string: "foo",
|
294
|
+
numeric: "1",
|
295
|
+
uri: "http://www.example.com",
|
296
|
+
dateTime_iso8601: "2013-01-01T13:00:00Z",
|
297
|
+
date_db: "2013-01-01",
|
298
|
+
dateTime_hms: "13:00:00"
|
312
299
|
}.each do |type, content|
|
313
300
|
it "should return the format of #{type} correctly" do
|
314
301
|
row = [content]
|
315
302
|
|
316
303
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
317
304
|
validator.build_formats(row)
|
318
|
-
formats = validator.instance_variable_get(
|
305
|
+
formats = validator.instance_variable_get(:@formats)
|
319
306
|
|
320
307
|
expect(formats[0].keys.first).to eql type
|
321
308
|
end
|
@@ -326,7 +313,7 @@ describe Csvlint::Validator do
|
|
326
313
|
|
327
314
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
328
315
|
validator.build_formats(row)
|
329
|
-
formats = validator.instance_variable_get(
|
316
|
+
formats = validator.instance_variable_get(:@formats)
|
330
317
|
|
331
318
|
expect(formats[0].keys.first).to eql :numeric
|
332
319
|
expect(formats[1].keys.first).to eql :numeric
|
@@ -338,15 +325,15 @@ describe Csvlint::Validator do
|
|
338
325
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
339
326
|
validator.build_formats(row)
|
340
327
|
|
341
|
-
formats = validator.instance_variable_get(
|
328
|
+
formats = validator.instance_variable_get(:@formats)
|
342
329
|
expect(formats).to eql []
|
343
330
|
end
|
344
331
|
|
345
332
|
it "should work correctly for single columns" do
|
346
333
|
rows = [
|
347
|
-
|
348
|
-
|
349
|
-
|
334
|
+
["foo"],
|
335
|
+
["bar"],
|
336
|
+
["baz"]
|
350
337
|
]
|
351
338
|
|
352
339
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
@@ -355,14 +342,14 @@ describe Csvlint::Validator do
|
|
355
342
|
validator.build_formats(row)
|
356
343
|
end
|
357
344
|
|
358
|
-
formats = validator.instance_variable_get(
|
359
|
-
expect(formats).to eql [{:
|
345
|
+
formats = validator.instance_variable_get(:@formats)
|
346
|
+
expect(formats).to eql [{string: 3}]
|
360
347
|
end
|
361
348
|
|
362
349
|
it "should return formats correctly if a row is blank" do
|
363
350
|
rows = [
|
364
|
-
|
365
|
-
|
351
|
+
[],
|
352
|
+
["foo", "1", "$2345"]
|
366
353
|
]
|
367
354
|
|
368
355
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
@@ -371,72 +358,68 @@ describe Csvlint::Validator do
|
|
371
358
|
validator.build_formats(row)
|
372
359
|
end
|
373
360
|
|
374
|
-
formats = validator.instance_variable_get(
|
361
|
+
formats = validator.instance_variable_get(:@formats)
|
375
362
|
|
376
363
|
expect(formats).to eql [
|
377
|
-
{:
|
378
|
-
{:
|
379
|
-
{:
|
364
|
+
{string: 1},
|
365
|
+
{numeric: 1},
|
366
|
+
{string: 1}
|
380
367
|
]
|
381
368
|
end
|
382
|
-
|
383
369
|
end
|
384
370
|
|
385
371
|
context "csv dialect" do
|
386
372
|
it "should provide sensible defaults for CSV parsing" do
|
387
373
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
388
|
-
opts = validator.instance_variable_get(
|
374
|
+
opts = validator.instance_variable_get(:@csv_options)
|
389
375
|
expect(opts).to include({
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
376
|
+
col_sep: ",",
|
377
|
+
row_sep: :auto,
|
378
|
+
quote_char: '"',
|
379
|
+
skip_blanks: false
|
380
|
+
})
|
395
381
|
end
|
396
382
|
|
397
383
|
it "should map CSV DDF to correct values" do
|
398
384
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
399
385
|
opts = validator.dialect_to_csv_options({
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
386
|
+
"lineTerminator" => "\n",
|
387
|
+
"delimiter" => "\t",
|
388
|
+
"quoteChar" => "'"
|
389
|
+
})
|
404
390
|
expect(opts).to include({
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
391
|
+
col_sep: "\t",
|
392
|
+
row_sep: "\n",
|
393
|
+
quote_char: "'",
|
394
|
+
skip_blanks: false
|
395
|
+
})
|
410
396
|
end
|
411
|
-
|
412
397
|
end
|
413
398
|
|
414
399
|
context "check_consistency" do
|
415
|
-
|
416
400
|
it "should return a warning if columns have inconsistent values" do
|
417
401
|
formats = [
|
418
|
-
|
419
|
-
|
420
|
-
|
402
|
+
{string: 3},
|
403
|
+
{string: 2, numeric: 1},
|
404
|
+
{numeric: 3}
|
421
405
|
]
|
422
406
|
|
423
407
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
424
|
-
validator.instance_variable_set(
|
408
|
+
validator.instance_variable_set(:@formats, formats)
|
425
409
|
validator.check_consistency
|
426
410
|
|
427
|
-
warnings = validator.instance_variable_get(
|
411
|
+
warnings = validator.instance_variable_get(:@warnings)
|
428
412
|
warnings.delete_if { |h| h.type != :inconsistent_values }
|
429
413
|
|
430
414
|
expect(warnings.count).to eql 1
|
431
415
|
end
|
432
|
-
|
433
416
|
end
|
434
417
|
|
435
|
-
#TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
|
418
|
+
# TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
|
436
419
|
context "when detecting headers" do
|
437
420
|
it "should default to expecting a header" do
|
438
421
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
439
|
-
expect(
|
422
|
+
expect(validator.header?).to eql(true)
|
440
423
|
end
|
441
424
|
|
442
425
|
it "should look in CSV options to detect header" do
|
@@ -444,171 +427,165 @@ describe Csvlint::Validator do
|
|
444
427
|
"header" => true
|
445
428
|
}
|
446
429
|
validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
|
447
|
-
expect(
|
430
|
+
expect(validator.header?).to eql(true)
|
448
431
|
opts = {
|
449
432
|
"header" => false
|
450
433
|
}
|
451
434
|
validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
|
452
|
-
expect(
|
435
|
+
expect(validator.header?).to eql(false)
|
453
436
|
end
|
454
437
|
|
455
438
|
it "should look in content-type for header=absent" do
|
456
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
439
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=absent"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
457
440
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
458
|
-
expect(
|
459
|
-
expect(
|
441
|
+
expect(validator.header?).to eql(false)
|
442
|
+
expect(validator.errors.size).to eql(0)
|
460
443
|
end
|
461
444
|
|
462
445
|
it "should look in content-type for header=present" do
|
463
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
446
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=present"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
464
447
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
465
|
-
expect(
|
466
|
-
expect(
|
448
|
+
expect(validator.header?).to eql(true)
|
449
|
+
expect(validator.errors.size).to eql(0)
|
467
450
|
end
|
468
451
|
|
469
452
|
it "assume header present if not specified in content type" do
|
470
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
453
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
471
454
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
472
|
-
expect(
|
473
|
-
expect(
|
474
|
-
expect(
|
475
|
-
expect(
|
455
|
+
expect(validator.header?).to eql(true)
|
456
|
+
expect(validator.errors.size).to eql(0)
|
457
|
+
expect(validator.info_messages.size).to eql(1)
|
458
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
476
459
|
end
|
477
460
|
|
478
461
|
it "give wrong content type error if content type is wrong" do
|
479
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
462
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
480
463
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
481
|
-
expect(
|
482
|
-
expect(
|
483
|
-
expect(
|
464
|
+
expect(validator.header?).to eql(true)
|
465
|
+
expect(validator.errors.size).to eql(1)
|
466
|
+
expect(validator.errors[0].type).to eql(:wrong_content_type)
|
484
467
|
end
|
485
|
-
|
486
468
|
end
|
487
469
|
|
488
470
|
context "when validating headers" do
|
489
471
|
it "should warn if column names aren't unique" do
|
490
|
-
data = StringIO.new(
|
472
|
+
data = StringIO.new("minimum, minimum")
|
491
473
|
validator = Csvlint::Validator.new(data)
|
492
|
-
expect(
|
493
|
-
expect(
|
494
|
-
expect(
|
474
|
+
expect(validator.warnings.size).to eql(1)
|
475
|
+
expect(validator.warnings.first.type).to eql(:duplicate_column_name)
|
476
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
495
477
|
end
|
496
478
|
|
497
479
|
it "should warn if column names are blank" do
|
498
|
-
data = StringIO.new(
|
480
|
+
data = StringIO.new("minimum,")
|
499
481
|
validator = Csvlint::Validator.new(data)
|
500
482
|
|
501
|
-
expect(
|
502
|
-
expect(
|
503
|
-
expect(
|
504
|
-
expect(
|
483
|
+
expect(validator.validate_header(["minimum", ""])).to eql(true)
|
484
|
+
expect(validator.warnings.size).to eql(1)
|
485
|
+
expect(validator.warnings.first.type).to eql(:empty_column_name)
|
486
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
505
487
|
end
|
506
488
|
|
507
489
|
it "should include info message about missing header when we have assumed a header" do
|
508
|
-
data = StringIO.new(
|
490
|
+
data = StringIO.new("1,2,3\r\n")
|
509
491
|
validator = Csvlint::Validator.new(data)
|
510
492
|
|
511
|
-
expect(
|
512
|
-
expect(
|
513
|
-
expect(
|
514
|
-
expect(
|
493
|
+
expect(validator.valid?).to eql(true)
|
494
|
+
expect(validator.info_messages.size).to eql(1)
|
495
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
496
|
+
expect(validator.info_messages.first.category).to eql(:structure)
|
515
497
|
end
|
516
498
|
|
517
499
|
it "should not include info message about missing header when we are told about the header" do
|
518
|
-
data = StringIO.new(
|
519
|
-
validator = Csvlint::Validator.new(data, "header"=>false)
|
520
|
-
expect(
|
521
|
-
expect(
|
500
|
+
data = StringIO.new("1,2,3\r\n")
|
501
|
+
validator = Csvlint::Validator.new(data, "header" => false)
|
502
|
+
expect(validator.valid?).to eql(true)
|
503
|
+
expect(validator.info_messages.size).to eql(0)
|
522
504
|
end
|
523
505
|
|
524
506
|
it "should not be an error if we have assumed a header, there is no dialect and content-type doesn't declare header, as we assume header=present" do
|
525
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
507
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
526
508
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
527
|
-
expect(
|
509
|
+
expect(validator.valid?).to eql(true)
|
528
510
|
end
|
529
511
|
|
530
512
|
it "should be valid if we have a dialect and the data is from the web" do
|
531
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
532
|
-
#header defaults to true in csv dialect, so this is valid
|
513
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
514
|
+
# header defaults to true in csv dialect, so this is valid
|
533
515
|
validator = Csvlint::Validator.new("http://example.com/example.csv", {})
|
534
|
-
expect(
|
516
|
+
expect(validator.valid?).to eql(true)
|
535
517
|
|
536
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
537
|
-
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>true})
|
538
|
-
expect(
|
518
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
519
|
+
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => true})
|
520
|
+
expect(validator.valid?).to eql(true)
|
539
521
|
|
540
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
541
|
-
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>false})
|
542
|
-
expect(
|
522
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
523
|
+
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => false})
|
524
|
+
expect(validator.valid?).to eql(true)
|
543
525
|
end
|
544
|
-
|
545
526
|
end
|
546
527
|
|
547
528
|
context "accessing metadata" do
|
548
|
-
|
549
529
|
before :all do
|
550
|
-
stub_request(:get, "http://example.com/crlf.csv").to_return(:
|
551
|
-
stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(:
|
530
|
+
stub_request(:get, "http://example.com/crlf.csv").to_return(status: 200, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "windows-line-endings.csv")))
|
531
|
+
stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(status: 404)
|
552
532
|
end
|
553
533
|
|
554
534
|
it "can get line break symbol" do
|
555
535
|
validator = Csvlint::Validator.new("http://example.com/crlf.csv")
|
556
536
|
expect(validator.line_breaks).to eql "\r\n"
|
557
537
|
end
|
558
|
-
|
559
538
|
end
|
560
539
|
|
561
540
|
it "should give access to the complete CSV data file" do
|
562
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
563
|
-
|
564
|
-
|
541
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
|
542
|
+
headers: {"Content-Type" => "text/csv; header=present"},
|
543
|
+
body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
565
544
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
566
|
-
expect(
|
545
|
+
expect(validator.valid?).to eql(true)
|
567
546
|
data = validator.data
|
568
547
|
|
569
|
-
expect(
|
570
|
-
expect(
|
571
|
-
expect(
|
548
|
+
expect(data.count).to eql 3
|
549
|
+
expect(data[0]).to eql ["Foo", "Bar", "Baz"]
|
550
|
+
expect(data[2]).to eql ["3", "2", "1"]
|
572
551
|
end
|
573
552
|
|
574
553
|
it "should count the total number of rows read" do
|
575
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
576
|
-
|
577
|
-
|
554
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
|
555
|
+
headers: {"Content-Type" => "text/csv; header=present"},
|
556
|
+
body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
578
557
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
579
558
|
expect(validator.row_count).to eq(3)
|
580
559
|
end
|
581
560
|
|
582
561
|
it "should limit number of lines read" do
|
583
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
584
|
-
|
585
|
-
|
562
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
|
563
|
+
headers: {"Content-Type" => "text/csv; header=present"},
|
564
|
+
body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
586
565
|
validator = Csvlint::Validator.new("http://example.com/example.csv", {}, nil, limit_lines: 2)
|
587
|
-
expect(
|
566
|
+
expect(validator.valid?).to eql(true)
|
588
567
|
data = validator.data
|
589
|
-
expect(
|
590
|
-
expect(
|
568
|
+
expect(data.count).to eql 2
|
569
|
+
expect(data[0]).to eql ["Foo", "Bar", "Baz"]
|
591
570
|
end
|
592
571
|
|
593
572
|
context "with a lambda" do
|
594
|
-
|
595
573
|
it "should call a lambda for each line" do
|
596
574
|
@count = 0
|
597
|
-
mylambda = lambda { |row| @count
|
598
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
575
|
+
mylambda = lambda { |row| @count += 1 }
|
576
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
|
599
577
|
expect(@count).to eq(3)
|
600
578
|
end
|
601
579
|
|
602
580
|
it "reports back the status of each line" do
|
603
581
|
@results = []
|
604
582
|
mylambda = lambda { |row| @results << row.current_line }
|
605
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
583
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
|
606
584
|
expect(@results.count).to eq(3)
|
607
585
|
expect(@results[0]).to eq(1)
|
608
586
|
expect(@results[1]).to eq(2)
|
609
587
|
expect(@results[2]).to eq(3)
|
610
588
|
end
|
611
|
-
|
612
589
|
end
|
613
590
|
|
614
591
|
# Commented out because there is currently no way to mock redirects with Typhoeus and WebMock - see https://github.com/bblimke/webmock/issues/237
|