csvlint 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +4 -0
- data/.github/workflows/push.yml +14 -2
- data/.pre-commit-hooks.yaml +5 -0
- data/.ruby-version +1 -1
- data/.standard_todo.yml +43 -0
- data/CHANGELOG.md +84 -32
- data/Dockerfile +16 -0
- data/Gemfile +2 -2
- data/README.md +30 -9
- data/Rakefile +7 -7
- data/csvlint.gemspec +14 -16
- data/docker_notes_for_windows.txt +20 -0
- data/features/step_definitions/cli_steps.rb +11 -11
- data/features/step_definitions/information_steps.rb +4 -4
- data/features/step_definitions/parse_csv_steps.rb +11 -11
- data/features/step_definitions/schema_validation_steps.rb +10 -10
- data/features/step_definitions/sources_steps.rb +1 -1
- data/features/step_definitions/validation_errors_steps.rb +19 -19
- data/features/step_definitions/validation_info_steps.rb +9 -9
- data/features/step_definitions/validation_warnings_steps.rb +11 -11
- data/features/support/aruba.rb +6 -6
- data/features/support/earl_formatter.rb +39 -39
- data/features/support/env.rb +10 -11
- data/features/support/load_tests.rb +107 -103
- data/features/support/webmock.rb +2 -2
- data/lib/csvlint/cli.rb +133 -130
- data/lib/csvlint/csvw/column.rb +279 -280
- data/lib/csvlint/csvw/date_format.rb +90 -92
- data/lib/csvlint/csvw/metadata_error.rb +1 -3
- data/lib/csvlint/csvw/number_format.rb +40 -32
- data/lib/csvlint/csvw/property_checker.rb +714 -717
- data/lib/csvlint/csvw/table.rb +49 -52
- data/lib/csvlint/csvw/table_group.rb +24 -23
- data/lib/csvlint/error_collector.rb +2 -0
- data/lib/csvlint/error_message.rb +0 -1
- data/lib/csvlint/field.rb +153 -141
- data/lib/csvlint/schema.rb +34 -42
- data/lib/csvlint/validate.rb +161 -143
- data/lib/csvlint/version.rb +1 -1
- data/lib/csvlint.rb +22 -23
- data/spec/csvw/column_spec.rb +15 -16
- data/spec/csvw/date_format_spec.rb +5 -7
- data/spec/csvw/number_format_spec.rb +2 -4
- data/spec/csvw/table_group_spec.rb +103 -105
- data/spec/csvw/table_spec.rb +71 -73
- data/spec/field_spec.rb +116 -121
- data/spec/schema_spec.rb +129 -139
- data/spec/spec_helper.rb +6 -6
- data/spec/validator_spec.rb +167 -190
- metadata +23 -55
data/spec/validator_spec.rb
CHANGED
@@ -1,49 +1,45 @@
|
|
1
|
-
require
|
1
|
+
require "spec_helper"
|
2
2
|
|
3
3
|
describe Csvlint::Validator do
|
4
|
-
|
5
4
|
before do
|
6
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
7
|
-
stub_request(:get, "http://example.com/.well-known/csvm").to_return(:
|
8
|
-
stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(:
|
9
|
-
stub_request(:get, "http://example.com/csv-metadata.json").to_return(:
|
5
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, body: "")
|
6
|
+
stub_request(:get, "http://example.com/.well-known/csvm").to_return(status: 404)
|
7
|
+
stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(status: 404)
|
8
|
+
stub_request(:get, "http://example.com/csv-metadata.json").to_return(status: 404)
|
10
9
|
end
|
11
10
|
|
12
11
|
it "should validate from a URL" do
|
13
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
12
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
14
13
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
15
14
|
|
16
15
|
expect(validator.valid?).to eql(true)
|
17
|
-
expect(validator.instance_variable_get(
|
18
|
-
expect(validator.instance_variable_get(
|
16
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
17
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
|
19
18
|
expect(validator.data.size).to eql(3)
|
20
19
|
end
|
21
20
|
|
22
21
|
it "should validate from a file path" do
|
23
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
22
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
24
23
|
|
25
24
|
expect(validator.valid?).to eql(true)
|
26
|
-
expect(validator.instance_variable_get(
|
27
|
-
expect(validator.instance_variable_get(
|
25
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
26
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
|
28
27
|
expect(validator.data.size).to eql(3)
|
29
28
|
end
|
30
29
|
|
31
30
|
it "should validate from a file path including whitespace" do
|
32
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
31
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "white space in filename.csv")))
|
33
32
|
|
34
33
|
expect(validator.valid?).to eql(true)
|
35
34
|
end
|
36
35
|
|
37
36
|
context "multi line CSV validation with included schema" do
|
38
|
-
|
39
37
|
end
|
40
38
|
|
41
39
|
context "single line row validation with included schema" do
|
42
|
-
|
43
40
|
end
|
44
41
|
|
45
42
|
context "validation with multiple lines: " do
|
46
|
-
|
47
43
|
# TODO multiple lines permits testing of warnings
|
48
44
|
# TODO need more assertions in each test IE @formats
|
49
45
|
# TODO the phrasing of col_counts if only consulting specs might be confusing
|
@@ -52,22 +48,21 @@ describe Csvlint::Validator do
|
|
52
48
|
|
53
49
|
it ".each() -> parse_contents method validates a well formed CSV" do
|
54
50
|
# when invoking parse contents
|
55
|
-
data = StringIO.new(%
|
51
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
|
56
52
|
|
57
53
|
validator = Csvlint::Validator.new(data)
|
58
54
|
|
59
55
|
expect(validator.valid?).to eql(true)
|
60
56
|
# TODO would be beneficial to know how formats functions WRT to headers - check_format.feature:17 returns 3 rows total
|
61
57
|
# TODO in its formats object but is provided with 5 rows (with one nil row) [uses validation_warnings_steps.rb]
|
62
|
-
expect(validator.instance_variable_get(
|
63
|
-
expect(validator.instance_variable_get(
|
58
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
59
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
|
64
60
|
expect(validator.data.size).to eql(4)
|
65
|
-
|
66
61
|
end
|
67
62
|
|
68
63
|
it ".each() -> `parse_contents` parses malformed CSV and catches unclosed quote" do
|
69
64
|
# doesn't build warnings because check_consistency isn't invoked
|
70
|
-
data = StringIO.new(%
|
65
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1))
|
71
66
|
|
72
67
|
validator = Csvlint::Validator.new(data)
|
73
68
|
|
@@ -79,7 +74,7 @@ describe Csvlint::Validator do
|
|
79
74
|
it ".each() -> `parse_contents` parses malformed CSV and catches whitespace and edge case" do
|
80
75
|
# when this data gets passed the header it rescues a whitespace error, resulting in the header row being discarded
|
81
76
|
# TODO - check if this is an edge case, currently passing because it requires advice on how to specify
|
82
|
-
data = StringIO.new(%
|
77
|
+
data = StringIO.new(%( "Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1" ))
|
83
78
|
|
84
79
|
validator = Csvlint::Validator.new(data)
|
85
80
|
|
@@ -89,13 +84,13 @@ describe Csvlint::Validator do
|
|
89
84
|
end
|
90
85
|
|
91
86
|
it "handles line breaks within a cell" do
|
92
|
-
data = StringIO.new(%
|
87
|
+
data = StringIO.new(%("a","b","c"\r\n"d","e","this is\r\nvalid"\r\n"a","b","c"))
|
93
88
|
validator = Csvlint::Validator.new(data)
|
94
89
|
expect(validator.valid?).to eql(true)
|
95
90
|
end
|
96
91
|
|
97
92
|
it "handles multiple line breaks within a cell" do
|
98
|
-
data = StringIO.new(%
|
93
|
+
data = StringIO.new(%("a","b","c"\r\n"d","this is\r\n valid","as is this\r\n too"))
|
99
94
|
validator = Csvlint::Validator.new(data)
|
100
95
|
expect(validator.valid?).to eql(true)
|
101
96
|
end
|
@@ -104,50 +99,50 @@ describe Csvlint::Validator do
|
|
104
99
|
context "csv dialect" do
|
105
100
|
it "should provide sensible defaults for CSV parsing" do
|
106
101
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
107
|
-
opts = validator.instance_variable_get(
|
102
|
+
opts = validator.instance_variable_get(:@csv_options)
|
108
103
|
expect(opts).to include({
|
109
|
-
:
|
110
|
-
:
|
111
|
-
:
|
112
|
-
:
|
104
|
+
col_sep: ",",
|
105
|
+
row_sep: :auto,
|
106
|
+
quote_char: '"',
|
107
|
+
skip_blanks: false
|
113
108
|
})
|
114
109
|
end
|
115
110
|
|
116
111
|
it "should map CSV DDF to correct values" do
|
117
112
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
118
|
-
opts = validator.dialect_to_csv_options(
|
113
|
+
opts = validator.dialect_to_csv_options({
|
119
114
|
"lineTerminator" => "\n",
|
120
115
|
"delimiter" => "\t",
|
121
116
|
"quoteChar" => "'"
|
122
117
|
})
|
123
118
|
expect(opts).to include({
|
124
|
-
:
|
125
|
-
:
|
126
|
-
:
|
127
|
-
:
|
119
|
+
col_sep: "\t",
|
120
|
+
row_sep: "\n",
|
121
|
+
quote_char: "'",
|
122
|
+
skip_blanks: false
|
128
123
|
})
|
129
124
|
end
|
130
125
|
|
131
126
|
it ".each() -> `validate` to pass input in streaming fashion" do
|
132
127
|
# warnings are built when validate is used to call all three methods
|
133
|
-
data = StringIO.new(%
|
128
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
|
134
129
|
validator = Csvlint::Validator.new(data)
|
135
130
|
|
136
131
|
expect(validator.valid?).to eql(true)
|
137
|
-
expect(validator.instance_variable_get(
|
138
|
-
expect(validator.instance_variable_get(
|
132
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
133
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
|
139
134
|
expect(validator.data.size).to eql(4)
|
140
135
|
expect(validator.info_messages.count).to eql(1)
|
141
136
|
end
|
142
137
|
|
143
138
|
it ".each() -> `validate` parses malformed CSV, populates errors, warnings & info_msgs,invokes finish()" do
|
144
|
-
data = StringIO.new(%
|
139
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"1","two","3"\r\n"3","2", "1"))
|
145
140
|
|
146
141
|
validator = Csvlint::Validator.new(data)
|
147
142
|
|
148
143
|
expect(validator.valid?).to eql(false)
|
149
|
-
expect(validator.instance_variable_get(
|
150
|
-
expect(validator.instance_variable_get(
|
144
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
145
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
|
151
146
|
expect(validator.data.size).to eql(5)
|
152
147
|
expect(validator.info_messages.count).to eql(1)
|
153
148
|
expect(validator.errors.count).to eql(1)
|
@@ -157,7 +152,7 @@ describe Csvlint::Validator do
|
|
157
152
|
end
|
158
153
|
|
159
154
|
it "File.open.each_line -> `validate` passes a valid csv" do
|
160
|
-
filename =
|
155
|
+
filename = "valid_many_rows.csv"
|
161
156
|
file = File.join(File.expand_path(Dir.pwd), "features", "fixtures", filename)
|
162
157
|
validator = Csvlint::Validator.new(File.new(file))
|
163
158
|
|
@@ -166,11 +161,9 @@ describe Csvlint::Validator do
|
|
166
161
|
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
167
162
|
expect(validator.info_messages.first.category).to eql(:structure)
|
168
163
|
end
|
169
|
-
|
170
164
|
end
|
171
165
|
|
172
166
|
context "with a single row" do
|
173
|
-
|
174
167
|
it "validates correctly" do
|
175
168
|
stream = "\"a\",\"b\",\"c\"\r\n"
|
176
169
|
validator = Csvlint::Validator.new(StringIO.new(stream), "header" => false)
|
@@ -204,10 +197,10 @@ describe Csvlint::Validator do
|
|
204
197
|
stream = "1,2,3\r\n"
|
205
198
|
validator = Csvlint::Validator.new(StringIO.new(stream))
|
206
199
|
|
207
|
-
expect(
|
208
|
-
expect(
|
209
|
-
expect(
|
210
|
-
expect(
|
200
|
+
expect(validator.valid?).to eql(true)
|
201
|
+
expect(validator.info_messages.size).to eql(1)
|
202
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
203
|
+
expect(validator.info_messages.first.category).to eql(:structure)
|
211
204
|
end
|
212
205
|
|
213
206
|
it "should evaluate the row as 'row 2' when stipulated" do
|
@@ -217,11 +210,9 @@ describe Csvlint::Validator do
|
|
217
210
|
expect(validator.valid?).to eql(true)
|
218
211
|
expect(validator.info_messages.size).to eql(0)
|
219
212
|
end
|
220
|
-
|
221
213
|
end
|
222
214
|
|
223
215
|
context "it returns the correct error from ERROR_MATCHES" do
|
224
|
-
|
225
216
|
it "checks for unclosed quotes" do
|
226
217
|
stream = "\"a,\"b\",\"c\"\n"
|
227
218
|
validator = Csvlint::Validator.new(StringIO.new(stream))
|
@@ -230,7 +221,6 @@ describe Csvlint::Validator do
|
|
230
221
|
expect(validator.errors.first.type).to eql(:unclosed_quote)
|
231
222
|
end
|
232
223
|
|
233
|
-
|
234
224
|
# TODO stray quotes is not covered in any spec in this library
|
235
225
|
# it "checks for stray quotes" do
|
236
226
|
# stream = "\"a\",“b“,\"c\"" "\r\n"
|
@@ -258,64 +248,61 @@ describe Csvlint::Validator do
|
|
258
248
|
expect(validator.errors.count).to eq(1)
|
259
249
|
expect(validator.errors.first.type).to eql(:line_breaks)
|
260
250
|
end
|
261
|
-
|
262
251
|
end
|
263
252
|
|
264
253
|
context "when validating headers" do
|
265
|
-
|
266
254
|
it "should warn if column names aren't unique" do
|
267
|
-
data = StringIO.new(
|
255
|
+
data = StringIO.new("minimum, minimum")
|
268
256
|
validator = Csvlint::Validator.new(data)
|
269
257
|
validator.reset
|
270
|
-
expect(
|
271
|
-
expect(
|
272
|
-
expect(
|
273
|
-
expect(
|
258
|
+
expect(validator.validate_header(["minimum", "minimum"])).to eql(true)
|
259
|
+
expect(validator.warnings.size).to eql(1)
|
260
|
+
expect(validator.warnings.first.type).to eql(:duplicate_column_name)
|
261
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
274
262
|
end
|
275
263
|
|
276
264
|
it "should warn if column names are blank" do
|
277
|
-
data = StringIO.new(
|
265
|
+
data = StringIO.new("minimum,")
|
278
266
|
validator = Csvlint::Validator.new(data)
|
279
267
|
|
280
|
-
expect(
|
281
|
-
expect(
|
282
|
-
expect(
|
283
|
-
expect(
|
268
|
+
expect(validator.validate_header(["minimum", ""])).to eql(true)
|
269
|
+
expect(validator.warnings.size).to eql(1)
|
270
|
+
expect(validator.warnings.first.type).to eql(:empty_column_name)
|
271
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
284
272
|
end
|
285
273
|
|
286
274
|
it "should include info message about missing header when we have assumed a header" do
|
287
|
-
data = StringIO.new(
|
275
|
+
data = StringIO.new("1,2,3\r\n")
|
288
276
|
validator = Csvlint::Validator.new(data)
|
289
|
-
expect(
|
290
|
-
expect(
|
291
|
-
expect(
|
292
|
-
expect(
|
277
|
+
expect(validator.valid?).to eql(true)
|
278
|
+
expect(validator.info_messages.size).to eql(1)
|
279
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
280
|
+
expect(validator.info_messages.first.category).to eql(:structure)
|
293
281
|
end
|
294
282
|
|
295
283
|
it "should not include info message about missing header when we are told about the header" do
|
296
|
-
data = StringIO.new(
|
284
|
+
data = StringIO.new("1,2,3\r\n")
|
297
285
|
validator = Csvlint::Validator.new(data, "header" => false)
|
298
|
-
expect(
|
299
|
-
expect(
|
286
|
+
expect(validator.valid?).to eql(true)
|
287
|
+
expect(validator.info_messages.size).to eql(0)
|
300
288
|
end
|
301
289
|
end
|
302
290
|
|
303
291
|
context "build_formats" do
|
304
|
-
|
305
292
|
{
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
293
|
+
string: "foo",
|
294
|
+
numeric: "1",
|
295
|
+
uri: "http://www.example.com",
|
296
|
+
dateTime_iso8601: "2013-01-01T13:00:00Z",
|
297
|
+
date_db: "2013-01-01",
|
298
|
+
dateTime_hms: "13:00:00"
|
312
299
|
}.each do |type, content|
|
313
300
|
it "should return the format of #{type} correctly" do
|
314
301
|
row = [content]
|
315
302
|
|
316
303
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
317
304
|
validator.build_formats(row)
|
318
|
-
formats = validator.instance_variable_get(
|
305
|
+
formats = validator.instance_variable_get(:@formats)
|
319
306
|
|
320
307
|
expect(formats[0].keys.first).to eql type
|
321
308
|
end
|
@@ -326,7 +313,7 @@ describe Csvlint::Validator do
|
|
326
313
|
|
327
314
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
328
315
|
validator.build_formats(row)
|
329
|
-
formats = validator.instance_variable_get(
|
316
|
+
formats = validator.instance_variable_get(:@formats)
|
330
317
|
|
331
318
|
expect(formats[0].keys.first).to eql :numeric
|
332
319
|
expect(formats[1].keys.first).to eql :numeric
|
@@ -338,15 +325,15 @@ describe Csvlint::Validator do
|
|
338
325
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
339
326
|
validator.build_formats(row)
|
340
327
|
|
341
|
-
formats = validator.instance_variable_get(
|
328
|
+
formats = validator.instance_variable_get(:@formats)
|
342
329
|
expect(formats).to eql []
|
343
330
|
end
|
344
331
|
|
345
332
|
it "should work correctly for single columns" do
|
346
333
|
rows = [
|
347
|
-
|
348
|
-
|
349
|
-
|
334
|
+
["foo"],
|
335
|
+
["bar"],
|
336
|
+
["baz"]
|
350
337
|
]
|
351
338
|
|
352
339
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
@@ -355,14 +342,14 @@ describe Csvlint::Validator do
|
|
355
342
|
validator.build_formats(row)
|
356
343
|
end
|
357
344
|
|
358
|
-
formats = validator.instance_variable_get(
|
359
|
-
expect(formats).to eql [{:
|
345
|
+
formats = validator.instance_variable_get(:@formats)
|
346
|
+
expect(formats).to eql [{string: 3}]
|
360
347
|
end
|
361
348
|
|
362
349
|
it "should return formats correctly if a row is blank" do
|
363
350
|
rows = [
|
364
|
-
|
365
|
-
|
351
|
+
[],
|
352
|
+
["foo", "1", "$2345"]
|
366
353
|
]
|
367
354
|
|
368
355
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
@@ -371,72 +358,68 @@ describe Csvlint::Validator do
|
|
371
358
|
validator.build_formats(row)
|
372
359
|
end
|
373
360
|
|
374
|
-
formats = validator.instance_variable_get(
|
361
|
+
formats = validator.instance_variable_get(:@formats)
|
375
362
|
|
376
363
|
expect(formats).to eql [
|
377
|
-
{:
|
378
|
-
{:
|
379
|
-
{:
|
364
|
+
{string: 1},
|
365
|
+
{numeric: 1},
|
366
|
+
{string: 1}
|
380
367
|
]
|
381
368
|
end
|
382
|
-
|
383
369
|
end
|
384
370
|
|
385
371
|
context "csv dialect" do
|
386
372
|
it "should provide sensible defaults for CSV parsing" do
|
387
373
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
388
|
-
opts = validator.instance_variable_get(
|
374
|
+
opts = validator.instance_variable_get(:@csv_options)
|
389
375
|
expect(opts).to include({
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
376
|
+
col_sep: ",",
|
377
|
+
row_sep: :auto,
|
378
|
+
quote_char: '"',
|
379
|
+
skip_blanks: false
|
380
|
+
})
|
395
381
|
end
|
396
382
|
|
397
383
|
it "should map CSV DDF to correct values" do
|
398
384
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
399
385
|
opts = validator.dialect_to_csv_options({
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
386
|
+
"lineTerminator" => "\n",
|
387
|
+
"delimiter" => "\t",
|
388
|
+
"quoteChar" => "'"
|
389
|
+
})
|
404
390
|
expect(opts).to include({
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
391
|
+
col_sep: "\t",
|
392
|
+
row_sep: "\n",
|
393
|
+
quote_char: "'",
|
394
|
+
skip_blanks: false
|
395
|
+
})
|
410
396
|
end
|
411
|
-
|
412
397
|
end
|
413
398
|
|
414
399
|
context "check_consistency" do
|
415
|
-
|
416
400
|
it "should return a warning if columns have inconsistent values" do
|
417
401
|
formats = [
|
418
|
-
|
419
|
-
|
420
|
-
|
402
|
+
{string: 3},
|
403
|
+
{string: 2, numeric: 1},
|
404
|
+
{numeric: 3}
|
421
405
|
]
|
422
406
|
|
423
407
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
424
|
-
validator.instance_variable_set(
|
408
|
+
validator.instance_variable_set(:@formats, formats)
|
425
409
|
validator.check_consistency
|
426
410
|
|
427
|
-
warnings = validator.instance_variable_get(
|
411
|
+
warnings = validator.instance_variable_get(:@warnings)
|
428
412
|
warnings.delete_if { |h| h.type != :inconsistent_values }
|
429
413
|
|
430
414
|
expect(warnings.count).to eql 1
|
431
415
|
end
|
432
|
-
|
433
416
|
end
|
434
417
|
|
435
|
-
#TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
|
418
|
+
# TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
|
436
419
|
context "when detecting headers" do
|
437
420
|
it "should default to expecting a header" do
|
438
421
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
439
|
-
expect(
|
422
|
+
expect(validator.header?).to eql(true)
|
440
423
|
end
|
441
424
|
|
442
425
|
it "should look in CSV options to detect header" do
|
@@ -444,171 +427,165 @@ describe Csvlint::Validator do
|
|
444
427
|
"header" => true
|
445
428
|
}
|
446
429
|
validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
|
447
|
-
expect(
|
430
|
+
expect(validator.header?).to eql(true)
|
448
431
|
opts = {
|
449
432
|
"header" => false
|
450
433
|
}
|
451
434
|
validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
|
452
|
-
expect(
|
435
|
+
expect(validator.header?).to eql(false)
|
453
436
|
end
|
454
437
|
|
455
438
|
it "should look in content-type for header=absent" do
|
456
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
439
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=absent"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
457
440
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
458
|
-
expect(
|
459
|
-
expect(
|
441
|
+
expect(validator.header?).to eql(false)
|
442
|
+
expect(validator.errors.size).to eql(0)
|
460
443
|
end
|
461
444
|
|
462
445
|
it "should look in content-type for header=present" do
|
463
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
446
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=present"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
464
447
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
465
|
-
expect(
|
466
|
-
expect(
|
448
|
+
expect(validator.header?).to eql(true)
|
449
|
+
expect(validator.errors.size).to eql(0)
|
467
450
|
end
|
468
451
|
|
469
452
|
it "assume header present if not specified in content type" do
|
470
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
453
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
471
454
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
472
|
-
expect(
|
473
|
-
expect(
|
474
|
-
expect(
|
475
|
-
expect(
|
455
|
+
expect(validator.header?).to eql(true)
|
456
|
+
expect(validator.errors.size).to eql(0)
|
457
|
+
expect(validator.info_messages.size).to eql(1)
|
458
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
476
459
|
end
|
477
460
|
|
478
461
|
it "give wrong content type error if content type is wrong" do
|
479
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
462
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
480
463
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
481
|
-
expect(
|
482
|
-
expect(
|
483
|
-
expect(
|
464
|
+
expect(validator.header?).to eql(true)
|
465
|
+
expect(validator.errors.size).to eql(1)
|
466
|
+
expect(validator.errors[0].type).to eql(:wrong_content_type)
|
484
467
|
end
|
485
|
-
|
486
468
|
end
|
487
469
|
|
488
470
|
context "when validating headers" do
|
489
471
|
it "should warn if column names aren't unique" do
|
490
|
-
data = StringIO.new(
|
472
|
+
data = StringIO.new("minimum, minimum")
|
491
473
|
validator = Csvlint::Validator.new(data)
|
492
|
-
expect(
|
493
|
-
expect(
|
494
|
-
expect(
|
474
|
+
expect(validator.warnings.size).to eql(1)
|
475
|
+
expect(validator.warnings.first.type).to eql(:duplicate_column_name)
|
476
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
495
477
|
end
|
496
478
|
|
497
479
|
it "should warn if column names are blank" do
|
498
|
-
data = StringIO.new(
|
480
|
+
data = StringIO.new("minimum,")
|
499
481
|
validator = Csvlint::Validator.new(data)
|
500
482
|
|
501
|
-
expect(
|
502
|
-
expect(
|
503
|
-
expect(
|
504
|
-
expect(
|
483
|
+
expect(validator.validate_header(["minimum", ""])).to eql(true)
|
484
|
+
expect(validator.warnings.size).to eql(1)
|
485
|
+
expect(validator.warnings.first.type).to eql(:empty_column_name)
|
486
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
505
487
|
end
|
506
488
|
|
507
489
|
it "should include info message about missing header when we have assumed a header" do
|
508
|
-
data = StringIO.new(
|
490
|
+
data = StringIO.new("1,2,3\r\n")
|
509
491
|
validator = Csvlint::Validator.new(data)
|
510
492
|
|
511
|
-
expect(
|
512
|
-
expect(
|
513
|
-
expect(
|
514
|
-
expect(
|
493
|
+
expect(validator.valid?).to eql(true)
|
494
|
+
expect(validator.info_messages.size).to eql(1)
|
495
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
496
|
+
expect(validator.info_messages.first.category).to eql(:structure)
|
515
497
|
end
|
516
498
|
|
517
499
|
it "should not include info message about missing header when we are told about the header" do
|
518
|
-
data = StringIO.new(
|
519
|
-
validator = Csvlint::Validator.new(data, "header"=>false)
|
520
|
-
expect(
|
521
|
-
expect(
|
500
|
+
data = StringIO.new("1,2,3\r\n")
|
501
|
+
validator = Csvlint::Validator.new(data, "header" => false)
|
502
|
+
expect(validator.valid?).to eql(true)
|
503
|
+
expect(validator.info_messages.size).to eql(0)
|
522
504
|
end
|
523
505
|
|
524
506
|
it "should not be an error if we have assumed a header, there is no dialect and content-type doesn't declare header, as we assume header=present" do
|
525
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
507
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
526
508
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
527
|
-
expect(
|
509
|
+
expect(validator.valid?).to eql(true)
|
528
510
|
end
|
529
511
|
|
530
512
|
it "should be valid if we have a dialect and the data is from the web" do
|
531
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
532
|
-
#header defaults to true in csv dialect, so this is valid
|
513
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
514
|
+
# header defaults to true in csv dialect, so this is valid
|
533
515
|
validator = Csvlint::Validator.new("http://example.com/example.csv", {})
|
534
|
-
expect(
|
516
|
+
expect(validator.valid?).to eql(true)
|
535
517
|
|
536
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
537
|
-
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>true})
|
538
|
-
expect(
|
518
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
519
|
+
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => true})
|
520
|
+
expect(validator.valid?).to eql(true)
|
539
521
|
|
540
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
541
|
-
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>false})
|
542
|
-
expect(
|
522
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
523
|
+
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => false})
|
524
|
+
expect(validator.valid?).to eql(true)
|
543
525
|
end
|
544
|
-
|
545
526
|
end
|
546
527
|
|
547
528
|
context "accessing metadata" do
|
548
|
-
|
549
529
|
before :all do
|
550
|
-
stub_request(:get, "http://example.com/crlf.csv").to_return(:
|
551
|
-
stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(:
|
530
|
+
stub_request(:get, "http://example.com/crlf.csv").to_return(status: 200, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "windows-line-endings.csv")))
|
531
|
+
stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(status: 404)
|
552
532
|
end
|
553
533
|
|
554
534
|
it "can get line break symbol" do
|
555
535
|
validator = Csvlint::Validator.new("http://example.com/crlf.csv")
|
556
536
|
expect(validator.line_breaks).to eql "\r\n"
|
557
537
|
end
|
558
|
-
|
559
538
|
end
|
560
539
|
|
561
540
|
it "should give access to the complete CSV data file" do
|
562
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
563
|
-
|
564
|
-
|
541
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
|
542
|
+
headers: {"Content-Type" => "text/csv; header=present"},
|
543
|
+
body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
565
544
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
566
|
-
expect(
|
545
|
+
expect(validator.valid?).to eql(true)
|
567
546
|
data = validator.data
|
568
547
|
|
569
|
-
expect(
|
570
|
-
expect(
|
571
|
-
expect(
|
548
|
+
expect(data.count).to eql 3
|
549
|
+
expect(data[0]).to eql ["Foo", "Bar", "Baz"]
|
550
|
+
expect(data[2]).to eql ["3", "2", "1"]
|
572
551
|
end
|
573
552
|
|
574
553
|
it "should count the total number of rows read" do
|
575
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
576
|
-
|
577
|
-
|
554
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
|
555
|
+
headers: {"Content-Type" => "text/csv; header=present"},
|
556
|
+
body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
578
557
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
579
558
|
expect(validator.row_count).to eq(3)
|
580
559
|
end
|
581
560
|
|
582
561
|
it "should limit number of lines read" do
|
583
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
584
|
-
|
585
|
-
|
562
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
|
563
|
+
headers: {"Content-Type" => "text/csv; header=present"},
|
564
|
+
body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
586
565
|
validator = Csvlint::Validator.new("http://example.com/example.csv", {}, nil, limit_lines: 2)
|
587
|
-
expect(
|
566
|
+
expect(validator.valid?).to eql(true)
|
588
567
|
data = validator.data
|
589
|
-
expect(
|
590
|
-
expect(
|
568
|
+
expect(data.count).to eql 2
|
569
|
+
expect(data[0]).to eql ["Foo", "Bar", "Baz"]
|
591
570
|
end
|
592
571
|
|
593
572
|
context "with a lambda" do
|
594
|
-
|
595
573
|
it "should call a lambda for each line" do
|
596
574
|
@count = 0
|
597
|
-
mylambda = lambda { |row| @count
|
598
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
575
|
+
mylambda = lambda { |row| @count += 1 }
|
576
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
|
599
577
|
expect(@count).to eq(3)
|
600
578
|
end
|
601
579
|
|
602
580
|
it "reports back the status of each line" do
|
603
581
|
@results = []
|
604
582
|
mylambda = lambda { |row| @results << row.current_line }
|
605
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
583
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
|
606
584
|
expect(@results.count).to eq(3)
|
607
585
|
expect(@results[0]).to eq(1)
|
608
586
|
expect(@results[1]).to eq(2)
|
609
587
|
expect(@results[2]).to eq(3)
|
610
588
|
end
|
611
|
-
|
612
589
|
end
|
613
590
|
|
614
591
|
# Commented out because there is currently no way to mock redirects with Typhoeus and WebMock - see https://github.com/bblimke/webmock/issues/237
|