csvlint 0.4.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/dependabot.yml +11 -0
- data/.github/workflows/push.yml +35 -0
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/.standard_todo.yml +43 -0
- data/CHANGELOG.md +38 -0
- data/Dockerfile +16 -0
- data/Gemfile +2 -2
- data/README.md +13 -10
- data/Rakefile +7 -7
- data/bin/create_schema +2 -2
- data/csvlint.gemspec +19 -22
- data/docker_notes_for_windows.txt +20 -0
- data/features/step_definitions/cli_steps.rb +11 -11
- data/features/step_definitions/information_steps.rb +4 -4
- data/features/step_definitions/parse_csv_steps.rb +11 -11
- data/features/step_definitions/schema_validation_steps.rb +10 -10
- data/features/step_definitions/sources_steps.rb +1 -1
- data/features/step_definitions/validation_errors_steps.rb +19 -19
- data/features/step_definitions/validation_info_steps.rb +9 -9
- data/features/step_definitions/validation_warnings_steps.rb +11 -11
- data/features/support/aruba.rb +10 -9
- data/features/support/earl_formatter.rb +39 -39
- data/features/support/env.rb +10 -11
- data/features/support/load_tests.rb +109 -105
- data/features/support/webmock.rb +3 -1
- data/lib/csvlint/cli.rb +136 -142
- data/lib/csvlint/csvw/column.rb +279 -280
- data/lib/csvlint/csvw/date_format.rb +90 -92
- data/lib/csvlint/csvw/metadata_error.rb +1 -3
- data/lib/csvlint/csvw/number_format.rb +40 -32
- data/lib/csvlint/csvw/property_checker.rb +714 -717
- data/lib/csvlint/csvw/table.rb +49 -52
- data/lib/csvlint/csvw/table_group.rb +24 -23
- data/lib/csvlint/error_collector.rb +2 -0
- data/lib/csvlint/error_message.rb +0 -1
- data/lib/csvlint/field.rb +153 -141
- data/lib/csvlint/schema.rb +35 -43
- data/lib/csvlint/validate.rb +173 -151
- data/lib/csvlint/version.rb +1 -1
- data/lib/csvlint.rb +22 -23
- data/spec/csvw/column_spec.rb +15 -16
- data/spec/csvw/date_format_spec.rb +5 -7
- data/spec/csvw/number_format_spec.rb +2 -4
- data/spec/csvw/table_group_spec.rb +103 -105
- data/spec/csvw/table_spec.rb +71 -73
- data/spec/field_spec.rb +116 -121
- data/spec/schema_spec.rb +131 -141
- data/spec/spec_helper.rb +6 -6
- data/spec/validator_spec.rb +167 -203
- metadata +41 -85
- data/.travis.yml +0 -37
data/spec/validator_spec.rb
CHANGED
@@ -1,49 +1,45 @@
|
|
1
|
-
require
|
1
|
+
require "spec_helper"
|
2
2
|
|
3
3
|
describe Csvlint::Validator do
|
4
|
-
|
5
4
|
before do
|
6
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
7
|
-
stub_request(:get, "http://example.com/.well-known/csvm").to_return(:
|
8
|
-
stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(:
|
9
|
-
stub_request(:get, "http://example.com/csv-metadata.json").to_return(:
|
5
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, body: "")
|
6
|
+
stub_request(:get, "http://example.com/.well-known/csvm").to_return(status: 404)
|
7
|
+
stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(status: 404)
|
8
|
+
stub_request(:get, "http://example.com/csv-metadata.json").to_return(status: 404)
|
10
9
|
end
|
11
10
|
|
12
11
|
it "should validate from a URL" do
|
13
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
12
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
14
13
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
15
14
|
|
16
15
|
expect(validator.valid?).to eql(true)
|
17
|
-
expect(validator.instance_variable_get(
|
18
|
-
expect(validator.instance_variable_get(
|
16
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
17
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
|
19
18
|
expect(validator.data.size).to eql(3)
|
20
19
|
end
|
21
20
|
|
22
21
|
it "should validate from a file path" do
|
23
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
22
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
24
23
|
|
25
24
|
expect(validator.valid?).to eql(true)
|
26
|
-
expect(validator.instance_variable_get(
|
27
|
-
expect(validator.instance_variable_get(
|
25
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
26
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
|
28
27
|
expect(validator.data.size).to eql(3)
|
29
28
|
end
|
30
29
|
|
31
30
|
it "should validate from a file path including whitespace" do
|
32
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
31
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "white space in filename.csv")))
|
33
32
|
|
34
33
|
expect(validator.valid?).to eql(true)
|
35
34
|
end
|
36
35
|
|
37
36
|
context "multi line CSV validation with included schema" do
|
38
|
-
|
39
37
|
end
|
40
38
|
|
41
39
|
context "single line row validation with included schema" do
|
42
|
-
|
43
40
|
end
|
44
41
|
|
45
42
|
context "validation with multiple lines: " do
|
46
|
-
|
47
43
|
# TODO multiple lines permits testing of warnings
|
48
44
|
# TODO need more assertions in each test IE @formats
|
49
45
|
# TODO the phrasing of col_counts if only consulting specs might be confusing
|
@@ -52,22 +48,21 @@ describe Csvlint::Validator do
|
|
52
48
|
|
53
49
|
it ".each() -> parse_contents method validates a well formed CSV" do
|
54
50
|
# when invoking parse contents
|
55
|
-
data = StringIO.new(%
|
51
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
|
56
52
|
|
57
53
|
validator = Csvlint::Validator.new(data)
|
58
54
|
|
59
55
|
expect(validator.valid?).to eql(true)
|
60
56
|
# TODO would be beneficial to know how formats functions WRT to headers - check_format.feature:17 returns 3 rows total
|
61
57
|
# TODO in its formats object but is provided with 5 rows (with one nil row) [uses validation_warnings_steps.rb]
|
62
|
-
expect(validator.instance_variable_get(
|
63
|
-
expect(validator.instance_variable_get(
|
58
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
59
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
|
64
60
|
expect(validator.data.size).to eql(4)
|
65
|
-
|
66
61
|
end
|
67
62
|
|
68
63
|
it ".each() -> `parse_contents` parses malformed CSV and catches unclosed quote" do
|
69
64
|
# doesn't build warnings because check_consistency isn't invoked
|
70
|
-
data = StringIO.new(%
|
65
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1))
|
71
66
|
|
72
67
|
validator = Csvlint::Validator.new(data)
|
73
68
|
|
@@ -76,23 +71,10 @@ describe Csvlint::Validator do
|
|
76
71
|
expect(validator.errors.first.type).to eql(:unclosed_quote)
|
77
72
|
end
|
78
73
|
|
79
|
-
it ".each() -> `parse_contents` parses malformed CSV and catches stray quote" do
|
80
|
-
pending "cannot make Ruby generate a stray quote error"
|
81
|
-
# doesn't build warnings because check_consistency isn't invoked
|
82
|
-
# TODO below is trailing whitespace but is interpreted as a stray quote
|
83
|
-
data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1""})
|
84
|
-
|
85
|
-
validator = Csvlint::Validator.new(data)
|
86
|
-
|
87
|
-
expect(validator.valid?).to eql(false)
|
88
|
-
expect(validator.errors.first.type).to eql(:stray_quote)
|
89
|
-
expect(validator.errors.count).to eql(1)
|
90
|
-
end
|
91
|
-
|
92
74
|
it ".each() -> `parse_contents` parses malformed CSV and catches whitespace and edge case" do
|
93
75
|
# when this data gets passed the header it rescues a whitespace error, resulting in the header row being discarded
|
94
76
|
# TODO - check if this is an edge case, currently passing because it requires advice on how to specify
|
95
|
-
data = StringIO.new(%
|
77
|
+
data = StringIO.new(%( "Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1" ))
|
96
78
|
|
97
79
|
validator = Csvlint::Validator.new(data)
|
98
80
|
|
@@ -102,13 +84,13 @@ describe Csvlint::Validator do
|
|
102
84
|
end
|
103
85
|
|
104
86
|
it "handles line breaks within a cell" do
|
105
|
-
data = StringIO.new(%
|
87
|
+
data = StringIO.new(%("a","b","c"\r\n"d","e","this is\r\nvalid"\r\n"a","b","c"))
|
106
88
|
validator = Csvlint::Validator.new(data)
|
107
89
|
expect(validator.valid?).to eql(true)
|
108
90
|
end
|
109
91
|
|
110
92
|
it "handles multiple line breaks within a cell" do
|
111
|
-
data = StringIO.new(%
|
93
|
+
data = StringIO.new(%("a","b","c"\r\n"d","this is\r\n valid","as is this\r\n too"))
|
112
94
|
validator = Csvlint::Validator.new(data)
|
113
95
|
expect(validator.valid?).to eql(true)
|
114
96
|
end
|
@@ -117,50 +99,50 @@ describe Csvlint::Validator do
|
|
117
99
|
context "csv dialect" do
|
118
100
|
it "should provide sensible defaults for CSV parsing" do
|
119
101
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
120
|
-
opts = validator.instance_variable_get(
|
102
|
+
opts = validator.instance_variable_get(:@csv_options)
|
121
103
|
expect(opts).to include({
|
122
|
-
:
|
123
|
-
:
|
124
|
-
:
|
125
|
-
:
|
104
|
+
col_sep: ",",
|
105
|
+
row_sep: :auto,
|
106
|
+
quote_char: '"',
|
107
|
+
skip_blanks: false
|
126
108
|
})
|
127
109
|
end
|
128
110
|
|
129
111
|
it "should map CSV DDF to correct values" do
|
130
112
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
131
|
-
opts = validator.dialect_to_csv_options(
|
113
|
+
opts = validator.dialect_to_csv_options({
|
132
114
|
"lineTerminator" => "\n",
|
133
115
|
"delimiter" => "\t",
|
134
116
|
"quoteChar" => "'"
|
135
117
|
})
|
136
118
|
expect(opts).to include({
|
137
|
-
:
|
138
|
-
:
|
139
|
-
:
|
140
|
-
:
|
119
|
+
col_sep: "\t",
|
120
|
+
row_sep: "\n",
|
121
|
+
quote_char: "'",
|
122
|
+
skip_blanks: false
|
141
123
|
})
|
142
124
|
end
|
143
125
|
|
144
126
|
it ".each() -> `validate` to pass input in streaming fashion" do
|
145
127
|
# warnings are built when validate is used to call all three methods
|
146
|
-
data = StringIO.new(%
|
128
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
|
147
129
|
validator = Csvlint::Validator.new(data)
|
148
130
|
|
149
131
|
expect(validator.valid?).to eql(true)
|
150
|
-
expect(validator.instance_variable_get(
|
151
|
-
expect(validator.instance_variable_get(
|
132
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
133
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
|
152
134
|
expect(validator.data.size).to eql(4)
|
153
135
|
expect(validator.info_messages.count).to eql(1)
|
154
136
|
end
|
155
137
|
|
156
138
|
it ".each() -> `validate` parses malformed CSV, populates errors, warnings & info_msgs,invokes finish()" do
|
157
|
-
data = StringIO.new(%
|
139
|
+
data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"1","two","3"\r\n"3","2", "1"))
|
158
140
|
|
159
141
|
validator = Csvlint::Validator.new(data)
|
160
142
|
|
161
143
|
expect(validator.valid?).to eql(false)
|
162
|
-
expect(validator.instance_variable_get(
|
163
|
-
expect(validator.instance_variable_get(
|
144
|
+
expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
|
145
|
+
expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
|
164
146
|
expect(validator.data.size).to eql(5)
|
165
147
|
expect(validator.info_messages.count).to eql(1)
|
166
148
|
expect(validator.errors.count).to eql(1)
|
@@ -170,7 +152,7 @@ describe Csvlint::Validator do
|
|
170
152
|
end
|
171
153
|
|
172
154
|
it "File.open.each_line -> `validate` passes a valid csv" do
|
173
|
-
filename =
|
155
|
+
filename = "valid_many_rows.csv"
|
174
156
|
file = File.join(File.expand_path(Dir.pwd), "features", "fixtures", filename)
|
175
157
|
validator = Csvlint::Validator.new(File.new(file))
|
176
158
|
|
@@ -179,11 +161,9 @@ describe Csvlint::Validator do
|
|
179
161
|
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
180
162
|
expect(validator.info_messages.first.category).to eql(:structure)
|
181
163
|
end
|
182
|
-
|
183
164
|
end
|
184
165
|
|
185
166
|
context "with a single row" do
|
186
|
-
|
187
167
|
it "validates correctly" do
|
188
168
|
stream = "\"a\",\"b\",\"c\"\r\n"
|
189
169
|
validator = Csvlint::Validator.new(StringIO.new(stream), "header" => false)
|
@@ -217,10 +197,10 @@ describe Csvlint::Validator do
|
|
217
197
|
stream = "1,2,3\r\n"
|
218
198
|
validator = Csvlint::Validator.new(StringIO.new(stream))
|
219
199
|
|
220
|
-
expect(
|
221
|
-
expect(
|
222
|
-
expect(
|
223
|
-
expect(
|
200
|
+
expect(validator.valid?).to eql(true)
|
201
|
+
expect(validator.info_messages.size).to eql(1)
|
202
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
203
|
+
expect(validator.info_messages.first.category).to eql(:structure)
|
224
204
|
end
|
225
205
|
|
226
206
|
it "should evaluate the row as 'row 2' when stipulated" do
|
@@ -230,11 +210,9 @@ describe Csvlint::Validator do
|
|
230
210
|
expect(validator.valid?).to eql(true)
|
231
211
|
expect(validator.info_messages.size).to eql(0)
|
232
212
|
end
|
233
|
-
|
234
213
|
end
|
235
214
|
|
236
215
|
context "it returns the correct error from ERROR_MATCHES" do
|
237
|
-
|
238
216
|
it "checks for unclosed quotes" do
|
239
217
|
stream = "\"a,\"b\",\"c\"\n"
|
240
218
|
validator = Csvlint::Validator.new(StringIO.new(stream))
|
@@ -243,7 +221,6 @@ describe Csvlint::Validator do
|
|
243
221
|
expect(validator.errors.first.type).to eql(:unclosed_quote)
|
244
222
|
end
|
245
223
|
|
246
|
-
|
247
224
|
# TODO stray quotes is not covered in any spec in this library
|
248
225
|
# it "checks for stray quotes" do
|
249
226
|
# stream = "\"a\",“b“,\"c\"" "\r\n"
|
@@ -271,64 +248,61 @@ describe Csvlint::Validator do
|
|
271
248
|
expect(validator.errors.count).to eq(1)
|
272
249
|
expect(validator.errors.first.type).to eql(:line_breaks)
|
273
250
|
end
|
274
|
-
|
275
251
|
end
|
276
252
|
|
277
253
|
context "when validating headers" do
|
278
|
-
|
279
254
|
it "should warn if column names aren't unique" do
|
280
|
-
data = StringIO.new(
|
255
|
+
data = StringIO.new("minimum, minimum")
|
281
256
|
validator = Csvlint::Validator.new(data)
|
282
257
|
validator.reset
|
283
|
-
expect(
|
284
|
-
expect(
|
285
|
-
expect(
|
286
|
-
expect(
|
258
|
+
expect(validator.validate_header(["minimum", "minimum"])).to eql(true)
|
259
|
+
expect(validator.warnings.size).to eql(1)
|
260
|
+
expect(validator.warnings.first.type).to eql(:duplicate_column_name)
|
261
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
287
262
|
end
|
288
263
|
|
289
264
|
it "should warn if column names are blank" do
|
290
|
-
data = StringIO.new(
|
265
|
+
data = StringIO.new("minimum,")
|
291
266
|
validator = Csvlint::Validator.new(data)
|
292
267
|
|
293
|
-
expect(
|
294
|
-
expect(
|
295
|
-
expect(
|
296
|
-
expect(
|
268
|
+
expect(validator.validate_header(["minimum", ""])).to eql(true)
|
269
|
+
expect(validator.warnings.size).to eql(1)
|
270
|
+
expect(validator.warnings.first.type).to eql(:empty_column_name)
|
271
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
297
272
|
end
|
298
273
|
|
299
274
|
it "should include info message about missing header when we have assumed a header" do
|
300
|
-
data = StringIO.new(
|
275
|
+
data = StringIO.new("1,2,3\r\n")
|
301
276
|
validator = Csvlint::Validator.new(data)
|
302
|
-
expect(
|
303
|
-
expect(
|
304
|
-
expect(
|
305
|
-
expect(
|
277
|
+
expect(validator.valid?).to eql(true)
|
278
|
+
expect(validator.info_messages.size).to eql(1)
|
279
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
280
|
+
expect(validator.info_messages.first.category).to eql(:structure)
|
306
281
|
end
|
307
282
|
|
308
283
|
it "should not include info message about missing header when we are told about the header" do
|
309
|
-
data = StringIO.new(
|
284
|
+
data = StringIO.new("1,2,3\r\n")
|
310
285
|
validator = Csvlint::Validator.new(data, "header" => false)
|
311
|
-
expect(
|
312
|
-
expect(
|
286
|
+
expect(validator.valid?).to eql(true)
|
287
|
+
expect(validator.info_messages.size).to eql(0)
|
313
288
|
end
|
314
289
|
end
|
315
290
|
|
316
291
|
context "build_formats" do
|
317
|
-
|
318
292
|
{
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
293
|
+
string: "foo",
|
294
|
+
numeric: "1",
|
295
|
+
uri: "http://www.example.com",
|
296
|
+
dateTime_iso8601: "2013-01-01T13:00:00Z",
|
297
|
+
date_db: "2013-01-01",
|
298
|
+
dateTime_hms: "13:00:00"
|
325
299
|
}.each do |type, content|
|
326
300
|
it "should return the format of #{type} correctly" do
|
327
301
|
row = [content]
|
328
302
|
|
329
303
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
330
304
|
validator.build_formats(row)
|
331
|
-
formats = validator.instance_variable_get(
|
305
|
+
formats = validator.instance_variable_get(:@formats)
|
332
306
|
|
333
307
|
expect(formats[0].keys.first).to eql type
|
334
308
|
end
|
@@ -339,7 +313,7 @@ describe Csvlint::Validator do
|
|
339
313
|
|
340
314
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
341
315
|
validator.build_formats(row)
|
342
|
-
formats = validator.instance_variable_get(
|
316
|
+
formats = validator.instance_variable_get(:@formats)
|
343
317
|
|
344
318
|
expect(formats[0].keys.first).to eql :numeric
|
345
319
|
expect(formats[1].keys.first).to eql :numeric
|
@@ -351,15 +325,15 @@ describe Csvlint::Validator do
|
|
351
325
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
352
326
|
validator.build_formats(row)
|
353
327
|
|
354
|
-
formats = validator.instance_variable_get(
|
328
|
+
formats = validator.instance_variable_get(:@formats)
|
355
329
|
expect(formats).to eql []
|
356
330
|
end
|
357
331
|
|
358
332
|
it "should work correctly for single columns" do
|
359
333
|
rows = [
|
360
|
-
|
361
|
-
|
362
|
-
|
334
|
+
["foo"],
|
335
|
+
["bar"],
|
336
|
+
["baz"]
|
363
337
|
]
|
364
338
|
|
365
339
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
@@ -368,14 +342,14 @@ describe Csvlint::Validator do
|
|
368
342
|
validator.build_formats(row)
|
369
343
|
end
|
370
344
|
|
371
|
-
formats = validator.instance_variable_get(
|
372
|
-
expect(formats).to eql [{:
|
345
|
+
formats = validator.instance_variable_get(:@formats)
|
346
|
+
expect(formats).to eql [{string: 3}]
|
373
347
|
end
|
374
348
|
|
375
349
|
it "should return formats correctly if a row is blank" do
|
376
350
|
rows = [
|
377
|
-
|
378
|
-
|
351
|
+
[],
|
352
|
+
["foo", "1", "$2345"]
|
379
353
|
]
|
380
354
|
|
381
355
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
@@ -384,72 +358,68 @@ describe Csvlint::Validator do
|
|
384
358
|
validator.build_formats(row)
|
385
359
|
end
|
386
360
|
|
387
|
-
formats = validator.instance_variable_get(
|
361
|
+
formats = validator.instance_variable_get(:@formats)
|
388
362
|
|
389
363
|
expect(formats).to eql [
|
390
|
-
{:
|
391
|
-
{:
|
392
|
-
{:
|
364
|
+
{string: 1},
|
365
|
+
{numeric: 1},
|
366
|
+
{string: 1}
|
393
367
|
]
|
394
368
|
end
|
395
|
-
|
396
369
|
end
|
397
370
|
|
398
371
|
context "csv dialect" do
|
399
372
|
it "should provide sensible defaults for CSV parsing" do
|
400
373
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
401
|
-
opts = validator.instance_variable_get(
|
374
|
+
opts = validator.instance_variable_get(:@csv_options)
|
402
375
|
expect(opts).to include({
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
376
|
+
col_sep: ",",
|
377
|
+
row_sep: :auto,
|
378
|
+
quote_char: '"',
|
379
|
+
skip_blanks: false
|
380
|
+
})
|
408
381
|
end
|
409
382
|
|
410
383
|
it "should map CSV DDF to correct values" do
|
411
384
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
412
385
|
opts = validator.dialect_to_csv_options({
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
386
|
+
"lineTerminator" => "\n",
|
387
|
+
"delimiter" => "\t",
|
388
|
+
"quoteChar" => "'"
|
389
|
+
})
|
417
390
|
expect(opts).to include({
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
391
|
+
col_sep: "\t",
|
392
|
+
row_sep: "\n",
|
393
|
+
quote_char: "'",
|
394
|
+
skip_blanks: false
|
395
|
+
})
|
423
396
|
end
|
424
|
-
|
425
397
|
end
|
426
398
|
|
427
399
|
context "check_consistency" do
|
428
|
-
|
429
400
|
it "should return a warning if columns have inconsistent values" do
|
430
401
|
formats = [
|
431
|
-
|
432
|
-
|
433
|
-
|
402
|
+
{string: 3},
|
403
|
+
{string: 2, numeric: 1},
|
404
|
+
{numeric: 3}
|
434
405
|
]
|
435
406
|
|
436
407
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
437
|
-
validator.instance_variable_set(
|
408
|
+
validator.instance_variable_set(:@formats, formats)
|
438
409
|
validator.check_consistency
|
439
410
|
|
440
|
-
warnings = validator.instance_variable_get(
|
411
|
+
warnings = validator.instance_variable_get(:@warnings)
|
441
412
|
warnings.delete_if { |h| h.type != :inconsistent_values }
|
442
413
|
|
443
414
|
expect(warnings.count).to eql 1
|
444
415
|
end
|
445
|
-
|
446
416
|
end
|
447
417
|
|
448
|
-
#TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
|
418
|
+
# TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
|
449
419
|
context "when detecting headers" do
|
450
420
|
it "should default to expecting a header" do
|
451
421
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
452
|
-
expect(
|
422
|
+
expect(validator.header?).to eql(true)
|
453
423
|
end
|
454
424
|
|
455
425
|
it "should look in CSV options to detect header" do
|
@@ -457,171 +427,165 @@ describe Csvlint::Validator do
|
|
457
427
|
"header" => true
|
458
428
|
}
|
459
429
|
validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
|
460
|
-
expect(
|
430
|
+
expect(validator.header?).to eql(true)
|
461
431
|
opts = {
|
462
432
|
"header" => false
|
463
433
|
}
|
464
434
|
validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
|
465
|
-
expect(
|
435
|
+
expect(validator.header?).to eql(false)
|
466
436
|
end
|
467
437
|
|
468
438
|
it "should look in content-type for header=absent" do
|
469
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
439
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=absent"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
470
440
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
471
|
-
expect(
|
472
|
-
expect(
|
441
|
+
expect(validator.header?).to eql(false)
|
442
|
+
expect(validator.errors.size).to eql(0)
|
473
443
|
end
|
474
444
|
|
475
445
|
it "should look in content-type for header=present" do
|
476
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
446
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=present"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
477
447
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
478
|
-
expect(
|
479
|
-
expect(
|
448
|
+
expect(validator.header?).to eql(true)
|
449
|
+
expect(validator.errors.size).to eql(0)
|
480
450
|
end
|
481
451
|
|
482
452
|
it "assume header present if not specified in content type" do
|
483
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
453
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
484
454
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
485
|
-
expect(
|
486
|
-
expect(
|
487
|
-
expect(
|
488
|
-
expect(
|
455
|
+
expect(validator.header?).to eql(true)
|
456
|
+
expect(validator.errors.size).to eql(0)
|
457
|
+
expect(validator.info_messages.size).to eql(1)
|
458
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
489
459
|
end
|
490
460
|
|
491
461
|
it "give wrong content type error if content type is wrong" do
|
492
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
462
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
493
463
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
494
|
-
expect(
|
495
|
-
expect(
|
496
|
-
expect(
|
464
|
+
expect(validator.header?).to eql(true)
|
465
|
+
expect(validator.errors.size).to eql(1)
|
466
|
+
expect(validator.errors[0].type).to eql(:wrong_content_type)
|
497
467
|
end
|
498
|
-
|
499
468
|
end
|
500
469
|
|
501
470
|
context "when validating headers" do
|
502
471
|
it "should warn if column names aren't unique" do
|
503
|
-
data = StringIO.new(
|
472
|
+
data = StringIO.new("minimum, minimum")
|
504
473
|
validator = Csvlint::Validator.new(data)
|
505
|
-
expect(
|
506
|
-
expect(
|
507
|
-
expect(
|
474
|
+
expect(validator.warnings.size).to eql(1)
|
475
|
+
expect(validator.warnings.first.type).to eql(:duplicate_column_name)
|
476
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
508
477
|
end
|
509
478
|
|
510
479
|
it "should warn if column names are blank" do
|
511
|
-
data = StringIO.new(
|
480
|
+
data = StringIO.new("minimum,")
|
512
481
|
validator = Csvlint::Validator.new(data)
|
513
482
|
|
514
|
-
expect(
|
515
|
-
expect(
|
516
|
-
expect(
|
517
|
-
expect(
|
483
|
+
expect(validator.validate_header(["minimum", ""])).to eql(true)
|
484
|
+
expect(validator.warnings.size).to eql(1)
|
485
|
+
expect(validator.warnings.first.type).to eql(:empty_column_name)
|
486
|
+
expect(validator.warnings.first.category).to eql(:schema)
|
518
487
|
end
|
519
488
|
|
520
489
|
it "should include info message about missing header when we have assumed a header" do
|
521
|
-
data = StringIO.new(
|
490
|
+
data = StringIO.new("1,2,3\r\n")
|
522
491
|
validator = Csvlint::Validator.new(data)
|
523
492
|
|
524
|
-
expect(
|
525
|
-
expect(
|
526
|
-
expect(
|
527
|
-
expect(
|
493
|
+
expect(validator.valid?).to eql(true)
|
494
|
+
expect(validator.info_messages.size).to eql(1)
|
495
|
+
expect(validator.info_messages.first.type).to eql(:assumed_header)
|
496
|
+
expect(validator.info_messages.first.category).to eql(:structure)
|
528
497
|
end
|
529
498
|
|
530
499
|
it "should not include info message about missing header when we are told about the header" do
|
531
|
-
data = StringIO.new(
|
532
|
-
validator = Csvlint::Validator.new(data, "header"=>false)
|
533
|
-
expect(
|
534
|
-
expect(
|
500
|
+
data = StringIO.new("1,2,3\r\n")
|
501
|
+
validator = Csvlint::Validator.new(data, "header" => false)
|
502
|
+
expect(validator.valid?).to eql(true)
|
503
|
+
expect(validator.info_messages.size).to eql(0)
|
535
504
|
end
|
536
505
|
|
537
506
|
it "should not be an error if we have assumed a header, there is no dialect and content-type doesn't declare header, as we assume header=present" do
|
538
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
507
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
539
508
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
540
|
-
expect(
|
509
|
+
expect(validator.valid?).to eql(true)
|
541
510
|
end
|
542
511
|
|
543
512
|
it "should be valid if we have a dialect and the data is from the web" do
|
544
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
545
|
-
#header defaults to true in csv dialect, so this is valid
|
513
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
514
|
+
# header defaults to true in csv dialect, so this is valid
|
546
515
|
validator = Csvlint::Validator.new("http://example.com/example.csv", {})
|
547
|
-
expect(
|
516
|
+
expect(validator.valid?).to eql(true)
|
548
517
|
|
549
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
550
|
-
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>true})
|
551
|
-
expect(
|
518
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
519
|
+
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => true})
|
520
|
+
expect(validator.valid?).to eql(true)
|
552
521
|
|
553
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
554
|
-
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>false})
|
555
|
-
expect(
|
522
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
523
|
+
validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => false})
|
524
|
+
expect(validator.valid?).to eql(true)
|
556
525
|
end
|
557
|
-
|
558
526
|
end
|
559
527
|
|
560
528
|
context "accessing metadata" do
|
561
|
-
|
562
529
|
before :all do
|
563
|
-
stub_request(:get, "http://example.com/crlf.csv").to_return(:
|
564
|
-
stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(:
|
530
|
+
stub_request(:get, "http://example.com/crlf.csv").to_return(status: 200, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "windows-line-endings.csv")))
|
531
|
+
stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(status: 404)
|
565
532
|
end
|
566
533
|
|
567
534
|
it "can get line break symbol" do
|
568
535
|
validator = Csvlint::Validator.new("http://example.com/crlf.csv")
|
569
536
|
expect(validator.line_breaks).to eql "\r\n"
|
570
537
|
end
|
571
|
-
|
572
538
|
end
|
573
539
|
|
574
540
|
it "should give access to the complete CSV data file" do
|
575
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
576
|
-
|
577
|
-
|
541
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
|
542
|
+
headers: {"Content-Type" => "text/csv; header=present"},
|
543
|
+
body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
578
544
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
579
|
-
expect(
|
545
|
+
expect(validator.valid?).to eql(true)
|
580
546
|
data = validator.data
|
581
547
|
|
582
|
-
expect(
|
583
|
-
expect(
|
584
|
-
expect(
|
548
|
+
expect(data.count).to eql 3
|
549
|
+
expect(data[0]).to eql ["Foo", "Bar", "Baz"]
|
550
|
+
expect(data[2]).to eql ["3", "2", "1"]
|
585
551
|
end
|
586
552
|
|
587
553
|
it "should count the total number of rows read" do
|
588
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
589
|
-
|
590
|
-
|
554
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
|
555
|
+
headers: {"Content-Type" => "text/csv; header=present"},
|
556
|
+
body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
591
557
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
592
558
|
expect(validator.row_count).to eq(3)
|
593
559
|
end
|
594
560
|
|
595
561
|
it "should limit number of lines read" do
|
596
|
-
stub_request(:get, "http://example.com/example.csv").to_return(:
|
597
|
-
|
598
|
-
|
562
|
+
stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
|
563
|
+
headers: {"Content-Type" => "text/csv; header=present"},
|
564
|
+
body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
|
599
565
|
validator = Csvlint::Validator.new("http://example.com/example.csv", {}, nil, limit_lines: 2)
|
600
|
-
expect(
|
566
|
+
expect(validator.valid?).to eql(true)
|
601
567
|
data = validator.data
|
602
|
-
expect(
|
603
|
-
expect(
|
568
|
+
expect(data.count).to eql 2
|
569
|
+
expect(data[0]).to eql ["Foo", "Bar", "Baz"]
|
604
570
|
end
|
605
571
|
|
606
572
|
context "with a lambda" do
|
607
|
-
|
608
573
|
it "should call a lambda for each line" do
|
609
574
|
@count = 0
|
610
|
-
mylambda = lambda { |row| @count
|
611
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
575
|
+
mylambda = lambda { |row| @count += 1 }
|
576
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
|
612
577
|
expect(@count).to eq(3)
|
613
578
|
end
|
614
579
|
|
615
580
|
it "reports back the status of each line" do
|
616
581
|
@results = []
|
617
582
|
mylambda = lambda { |row| @results << row.current_line }
|
618
|
-
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),
|
583
|
+
validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
|
619
584
|
expect(@results.count).to eq(3)
|
620
585
|
expect(@results[0]).to eq(1)
|
621
586
|
expect(@results[1]).to eq(2)
|
622
587
|
expect(@results[2]).to eq(3)
|
623
588
|
end
|
624
|
-
|
625
589
|
end
|
626
590
|
|
627
591
|
# Commented out because there is currently no way to mock redirects with Typhoeus and WebMock - see https://github.com/bblimke/webmock/issues/237
|